线上Nginx频繁502的排查过程与解决方案

tail -f /var/log/nginx/error.log

upstream timed out (110: connection timed out) while connecting to upstream
upstream prematurely closed connection while reading response header

# 查看后端服务进程
ps aux | grep java

# 查看端口监听
ss -tlnp | grep 8080

# 直接测试后端
curl -i http://127.0.0.1:8080/health

# 查看nginx到后端的连接数
ss -ant | grep 8080 | wc -l

# 查看连接状态分布
ss -ant | grep 8080 | awk '{print $1}' | sort | uniq -c

850 established
   120 time_wait
    50 syn_sent

# 查看后端服务的accept队列 
ss -lnt | grep 8080

state    recv-q   send-q   local address:port
listen   129      128      0.0.0.0:8080

客户端 → syn → 服务端（半连接队列）
服务端 → syn+ack → 客户端
客户端 → ack → 服务端（全连接队列/accept队列）
应用程序 accept() → 取出连接

server:
  tomcat:
    accept-count: 100  # tomcat的accept队列

# 查看当前值
sysctl net.core.somaxconn

# 临时修改
sysctl -w net.core.somaxconn=65535

# 永久修改
echo "net.core.somaxconn = 65535" >> /etc/sysctl.conf
sysctl -p

server:
  tomcat:
    accept-count: 1000      # accept队列大小
    max-connections: 10000  # 最大连接数
    threads:
      max: 500              # 最大工作线程数

upstream backend {
    server 127.0.0.1:8080 max_fails=3 fail_timeout=30s;
    
    keepalive 100;  # 保持连接数，减少新建连接
}

server {
    location / {
        proxy_pass http://backend;
        
        proxy_connect_timeout 5s;      # 连接超时
        proxy_read_timeout 60s;        # 读取超时
        proxy_send_timeout 60s;        # 发送超时
        
        proxy_http_version 1.1;        # 使用http/1.1
        proxy_set_header connection ""; # 配合keepalive
    }
}

upstream backend {
    least_conn;  # 最少连接数策略
    
    server 127.0.0.1:8080 weight=1;
    server 127.0.0.1:8081 weight=1;
    server 127.0.0.1:8082 weight=1;
    
    keepalive 100;
}

net.core.somaxconn = 65535
net.ipv4.tcp_max_syn_backlog = 65535
net.core.netdev_max_backlog = 65535

server:
  tomcat:
    accept-count: 2000
    max-connections: 20000
    threads:
      max: 500
      min-spare: 50

upstream backend {
    server 127.0.0.1:8080;
    keepalive 200;
}

server {
    location / {
        proxy_pass http://backend;
        proxy_http_version 1.1;
        proxy_set_header connection "";
        proxy_connect_timeout 5s;
        proxy_read_timeout 60s;
    }
}

# 查看nginx错误日志
tail -f /var/log/nginx/error.log

# 查看连接状态
ss -ant | grep <端口>

# 查看监听队列
ss -lnt | grep <端口>

# 查看队列溢出统计
netstat -s | grep -i listen

# 查看系统参数
sysctl net.core.somaxconn

# 实时监控连接数
watch -n 1 'ss -ant | grep <端口> | wc -l'