看懂网络命令输出 —— 把每一个数字讲清楚

$ ip addr | grep inet
inet 127.0.0.1/8 scope host lo
inet 192.168.1.5/24 scope global eth0
inet 169.254.1.5/16 scope link eth0       # link-local（没 DHCP 时自动）

0.0.0.0:22         # IPv4 所有地址的 22 端口（最常见）
127.0.0.1:6443     # 仅本机回环（apiserver 只允许本机访问场景）
192.168.1.5:80     # 仅这个具体 IP（多 IP 机器选 binding）
[::]:22            # IPv6 所有地址
[::1]:6443         # IPv6 回环
127.0.0.53%lo:53   # %lo 表示绑定到 lo 接口（systemd-resolved）
*:54321            # 旧式写法，等价 0.0.0.0

$ ss -tn
State        Recv-Q  Send-Q  Local Address:Port   Peer Address:Port
ESTAB        0       0       10.0.24.28:43210     10.0.24.29:6443
ESTAB        100     0       10.0.24.28:50001     10.244.1.5:8080
ESTAB        0       50000   10.0.24.28:55555     10.0.24.30:9090
TIME-WAIT    0       0       10.0.24.28:42345     10.0.24.29:6443
CLOSE-WAIT   200     0       10.0.24.28:60000     10.0.24.31:80

# 1. 数有多少
$ ss -tn state close-wait | wc -l
500

# 2. 看是哪个进程
$ ss -tnp state close-wait | head
ESTAB ... users:(("my-app",pid=1234,fd=23))
        ↑
        定位到具体进程

$ iptables -L INPUT -n -v --line-numbers | grep 30080
3     0     0     ACCEPT     tcp  ...  tcp dpt:30080
                  ^^^^^^^
                  pkts = 0 → 规则没被命中

# 重置 counter
iptables -Z
# 触发一次流量
curl http://node:30080
# 再看
iptables -L INPUT -n -v --line-numbers | grep 30080
# pkts 应该 ≥ 1

$ iptables -t nat -L -n | grep "^Chain"
Chain PREROUTING                    # 包刚进、未路由前
Chain INPUT
Chain OUTPUT
Chain POSTROUTING                   # 包出、已路由后
Chain DOCKER
Chain DOCKER-USER

Chain KUBE-SERVICES                 # 所有 Service 入口
Chain KUBE-NODEPORTS                # NodePort 服务
Chain KUBE-POSTROUTING              # 出口 SNAT
Chain KUBE-MARK-MASQ                # 给包打 mark 等 POSTROUTING 时 SNAT
Chain KUBE-SVC-XXXXX                # 每个 Service 一条
Chain KUBE-SEP-XXXXX                # 每个 Endpoint 一条

$ iptables -t nat -L KUBE-SERVICES -n -v | grep 10.96.1.5
   45 2700 KUBE-SVC-MYAPP123  tcp  --  ...  10.96.1.5  /* default/my-svc */ tcp dpt:80
       ↑   ↑^^^^^^^^^^^^^^^^                                                    ↑
       (1) (2)                                                                  (3)

$ iptables -t nat -L KUBE-SVC-MYAPP123 -n -v
 pkts bytes target           prot opt ...
   23  1380 KUBE-SEP-AAA      all  --  ...  /* probability 0.33 */ statistic mode random probability 0.33
   15   900 KUBE-SEP-BBB      all  --  ...  /* probability 0.50 */ statistic mode random probability 0.50
    7   420 KUBE-SEP-CCC      all  --  ...  /* */

$ iptables -t nat -L KUBE-SEP-AAA -n -v
 pkts bytes target  prot opt ...
   23  1380 DNAT    tcp  ...  to:10.244.0.5:8080

$ tcpdump -i any -nn -c 5 'port 80'
14:30:00.123456 IP 192.168.1.5.43210 > 93.184.215.14.80: Flags [S], seq 1234567890, win 64240, options [mss 1460,sackOK,TS val 12345 ecr 0,nop,wscale 7], length 0
14:30:00.234567 IP 93.184.215.14.80 > 192.168.1.5.43210: Flags [S.], seq 9876543210, ack 1234567891, win 65535, options [mss 1460,sackOK,TS val 67890 ecr 12345,nop,wscale 7], length 0
14:30:00.234780 IP 192.168.1.5.43210 > 93.184.215.14.80: Flags [.], ack 1, win 502, options [nop,nop,TS val 12346 ecr 67890], length 0
14:30:00.234890 IP 192.168.1.5.43210 > 93.184.215.14.80: Flags [P.], seq 1:79, ack 1, win 502, options [nop,nop,TS val 12346 ecr 67890], length 78: HTTP: GET / HTTP/1.1
14:30:00.500000 IP 93.184.215.14.80 > 192.168.1.5.43210: Flags [P.], seq 1:1200, ack 79, win 65535, options [nop,nop,TS val 67891 ecr 12346], length 1199: HTTP: HTTP/1.1 200 OK

[R]      RST           对端拒连 / 应用 crash
[R.]     RST-ACK       同上
重复 [S]  SYN 重传      包丢失 / server 没回
重复 seq  TCP 重传       中间丢包

$ tcpdump -i any -nn 'host <server> and port <port>'
# 看到 [R] = server 主动拒（端口没监听 / 防火墙）

# 看到大量重传（同 seq 出现多次）
$ tcpdump -i any -nn 'host <server>' | head
14:30:00.001 IP A > B: Flags [P.], seq 100:200, length 100
14:30:00.500 IP A > B: Flags [P.], seq 100:200, length 100   ← 重传！
14:30:01.500 IP A > B: Flags [P.], seq 100:200, length 100   ← 又重传

# 终端 1：源节点 m1
ssh m1 'tcpdump -i any -nn host 10.244.1.5'

# 终端 2：目的节点 m2
ssh m2 'tcpdump -i any -nn host 10.244.0.5'

# 终端 3：触发流量
kubectl exec pod-a -- curl http://10.244.1.5:8080

$ tcpdump -i any -nn -A -s 0 'port 80'
                  ^^ ^^^^
                  |  s 0 = 不截断（默认只抓前 96 字节）
                  -A = 显示 ASCII

# 输出含完整 HTTP 内容
14:30:00.123456 IP 1.2.3.4.43210 > 5.6.7.8.80: Flags [P.] ... length 78: HTTP: GET / HTTP/1.1
GET / HTTP/1.1
Host: example.com
User-Agent: curl/8.4.0
Accept: */*

# 看 DNS 内容
$ tcpdump -i any -nn 'port 53' -A
14:30:00.123456 IP 1.2.3.4.54321 > 10.96.0.10.53: 12345+ A? kubernetes.default.svc.cluster.local. (54)
                                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
                                                  query: 12345 是 ID、A 是记录类型、问 kubernetes.default...

$ nc -zv 10.96.0.1 443
Connection to 10.96.0.1 443 port [tcp/*] succeeded!

$ nc -zv 10.96.0.99 443
nc: connect to 10.96.0.99 port 443 (tcp) failed: Connection refused

$ nc -zv 10.0.0.99 443
nc: connect to 10.0.0.99 port 443 (tcp) failed: Operation timed out

$ dig kubernetes.default.svc.cluster.local @10.96.0.10
; <<>> DiG 9.18.18 <<>> kubernetes.default.svc.cluster.local @10.96.0.10
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 12345
;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 0
;;
;; QUESTION SECTION:
;kubernetes.default.svc.cluster.local. IN A
;;
;; ANSWER SECTION:
kubernetes.default.svc.cluster.local. 30 IN A 10.96.0.1
                                       ↑    ↑ ↑
                                      (1)  (2)(3)
;;
;; Query time: 5 msec
;; SERVER: 10.96.0.10#53(10.96.0.10) (UDP)
;; WHEN: Mon May 27 14:30:00 UTC 2026
;; MSG SIZE  rcvd: 99

$ dig +stats kubernetes.default
;; Query time: 5234 msec      ← 5 秒！太慢

$ mtr -rn -c 30 8.8.8.8
HOST: m1                                Loss%   Snt   Last   Avg  Best  Wrst StDev
  1.|-- 192.168.1.1                       0.0%    30    0.5   0.6   0.4   1.0   0.2
  2.|-- 10.0.0.1                           0.0%    30    2.1   2.5   1.8   5.2   0.7
  3.|-- ???                              100.0%    30    0.0   0.0   0.0   0.0   0.0
  4.|-- 203.0.113.1                        0.0%    30   12.0  12.5  11.8  20.1   1.5
  5.|-- 8.8.8.8                            0.0%    30   25.0  26.0  24.5  35.0   2.0

$ mtr -rn -T -P 443 -c 30 example.com    # 用 TCP 443 探测

$ journalctl -u kubelet --since "10 min ago" | grep -iE 'network|cni|cidr|route'

$ kubectl get pod my-pod -o wide
NAME      STATUS    IP            NODE
my-pod    Running   10.244.1.5    m2

# 别的 pod 访问 my-pod 失败
$ kubectl exec test-pod -- curl http://10.244.1.5:8080
# timeout

# 1. L4 端口连通？
$ kubectl exec test-pod -- nc -zv 10.244.1.5 8080
# refused / timeout?

# 2. 在 m2 上看 pod 是不是真在监听
$ ssh m2
$ PID=$(crictl inspect $(crictl ps -q --name my-pod) | jq '.info.pid')
$ nsenter -t $PID -n ss -lntp
# 看到 0.0.0.0:8080 LISTEN 了？

# 3. 看节点路由
$ ip route get 10.244.1.5
# via cni0 dev cni0   ← 直连 OK

# 4. 看 m2 上抓包
$ tcpdump -i any -nn host 10.244.0.5 and port 8080
# 看 SYN 进来了？ SYN-ACK 出去了？

# 5. 看 iptables NetworkPolicy 是否挡
$ iptables -L FORWARD -n -v | grep DROP

# 6. cilium / calico 等用户态网络也可能拦
$ hubble observe --verdict DROPPED --pod default/my-pod