容器网络底层：netns / veth / bridge / VXLAN

$ ls -l /proc/*/ns/net 2>/dev/null | awk '{print $NF, $9}' | sort | uniq -c | head
    1 net:[4026531840] /proc/1/ns/net
    5 net:[4026532152] /proc/12345/ns/net
    5 net:[4026532153] /proc/56789/ns/net

# 1. 创建两个 netns
ip netns add ns1
ip netns add ns2

# 2. 看
ip netns list
# ns1
# ns2

# 3. 在 ns1 里跑命令
ip netns exec ns1 ip addr
# 1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN
#     link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
# 只有 loopback，啥都没！而且 loopback 也 DOWN 着

ip netns exec ns1 ip link set lo up
ip netns exec ns1 ping 127.0.0.1
# 通了

# 1. 创建一对 veth
ip link add veth-a type veth peer name veth-b

# 此时它们都在 root netns
ip link show veth-a
# veth-a@veth-b: ...

# 2. 把 veth-a 移到 ns1
ip link set veth-a netns ns1

# 3. 把 veth-b 移到 ns2
ip link set veth-b netns ns2

# 4. 在 ns1 里配置 veth-a
ip netns exec ns1 ip link set veth-a up
ip netns exec ns1 ip addr add 10.1.1.1/24 dev veth-a

# 5. 在 ns2 里配置 veth-b
ip netns exec ns2 ip link set veth-b up
ip netns exec ns2 ip addr add 10.1.1.2/24 dev veth-b

# 6. 互通
ip netns exec ns1 ping 10.1.1.2
# 64 bytes from 10.1.1.2: ...

# 1. 创建 bridge
ip link add br0 type bridge
ip link set br0 up
ip addr add 10.1.1.1/24 dev br0

# 2. 对每个 netns:
#    a) 创建 veth pair
#    b) 一头放进 netns、配 IP
#    c) 另一头连到 bridge

for ns in nsA nsB nsC; do
  ip netns add $ns
  ip netns exec $ns ip link set lo up

  # 创建 veth pair
  ip link add veth-$ns-host type veth peer name veth-$ns-ns

  # netns 端：放进去 + 配 IP
  ip link set veth-$ns-ns netns $ns
  case $ns in
    nsA) IP=10.1.1.10 ;;
    nsB) IP=10.1.1.20 ;;
    nsC) IP=10.1.1.30 ;;
  esac
  ip netns exec $ns ip link set veth-$ns-ns up
  ip netns exec $ns ip addr add $IP/24 dev veth-$ns-ns
  ip netns exec $ns ip route add default via 10.1.1.1

  # host 端：up + 加入 bridge
  ip link set veth-$ns-host up
  ip link set veth-$ns-host master br0
done

# 3. 验证三个 ns 互通
ip netns exec nsA ping -c 1 10.1.1.20
ip netns exec nsB ping -c 1 10.1.1.30
ip netns exec nsC ping -c 1 10.1.1.10
# 都通

$ ip link show master br0
3: veth-nsA-host@if2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 master br0
5: veth-nsB-host@if2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 master br0
7: veth-nsC-host@if2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 master br0

$ bridge link show
3: veth-nsA-host@if2: ... master br0 state forwarding ...

$ bridge fdb show br br0
33:33:00:00:00:01 dev br0 self permanent
ee:ee:01:02:03:04 dev veth-nsA-host master br0
ff:ff:02:03:04:05 dev veth-nsB-host master br0
...

$ ip link
1: lo: <LOOPBACK,UP,LOWER_UP> ...
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 ...     # 物理网卡
3: cni0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 ...     # CNI 创建的 bridge
4: flannel.1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 ...# Flannel overlay 设备
5: vethABCD1234@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> ... master cni0   # pod A 的 host 端
6: vethEFGH5678@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> ... master cni0   # pod B 的 host 端

$ kubectl exec my-pod -- ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> ...
3: eth0@if5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 ...
    link/ether 0a:58:0a:f4:00:05 brd ff:ff:ff:ff:ff:ff
    inet 10.244.0.5/24 brd 10.244.0.255 scope global eth0

# 节点上看所有 veth
$ ip -o link | grep '@if'
5: vethABCD1234@if3: ...
6: vethEFGH5678@if3: ...

# 进 pod netns 看 pod 的 eth0 对端 ifindex
$ kubectl get pod my-pod -o jsonpath='{.metadata.uid}'
abc123-...
$ PID=$(crictl inspect $(crictl ps -q --name my-pod) | jq '.info.pid')
$ nsenter -t $PID -n ip link
3: eth0@if5: ...                         # ← @if5 表示 host 侧 ifindex=5

# 节点上找 ifindex=5 的网卡
$ ip link show | awk -F': ' '/^5:/{print $2}'
vethABCD1234@if3                          # ← 这就是 my-pod 的 veth

$ ip link show flannel.1 | grep mtu
4: flannel.1: ... mtu 1450 ...

nsenter -t <PID> <namespace flags> <command>

# 进 pod 的网络 namespace 跑命令
nsenter -t <PID> -n ip addr
nsenter -t <PID> -n ip route
nsenter -t <PID> -n ss -lntp
nsenter -t <PID> -n tcpdump -i any -nn

# 进 mount namespace 看 pod 文件系统视角
nsenter -t <PID> -m ls /

# 多 namespace 一起进
nsenter -t <PID> -n -m -p bash
#               ^^ ^^ ^^
#               网络  挂载  pid

# 进所有
nsenter -t <PID> -a bash

# 1. 找 pod 容器 PID
PID=$(crictl inspect $(crictl ps -q --name my-pod) | jq '.info.pid')

# 2. 进 pod netns 抓包
nsenter -t $PID -n tcpdump -i any -nn -c 30

# 3. 进 pod netns 测连通
nsenter -t $PID -n curl -v http://10.96.0.10

# 4. 进 pod netns 看 socket 状态
nsenter -t $PID -n ss -tnp

#!/bin/bash
set -e

# 清理上次（容错）
ip netns del nsA 2>/dev/null || true
ip netns del nsB 2>/dev/null || true
ip link del br-demo 2>/dev/null || true

echo "=== 1. 创建 bridge (模拟 cni0) ==="
ip link add br-demo type bridge
ip link set br-demo up
ip addr add 10.99.0.1/24 dev br-demo

echo "=== 2. 创建 podA 的 netns + veth ==="
ip netns add nsA
ip netns exec nsA ip link set lo up

ip link add veth-a type veth peer name veth-a-host
ip link set veth-a netns nsA
ip link set veth-a-host master br-demo
ip link set veth-a-host up

ip netns exec nsA ip link set veth-a up
ip netns exec nsA ip addr add 10.99.0.10/24 dev veth-a
ip netns exec nsA ip route add default via 10.99.0.1

echo "=== 3. 创建 podB ==="
ip netns add nsB
ip netns exec nsB ip link set lo up

ip link add veth-b type veth peer name veth-b-host
ip link set veth-b netns nsB
ip link set veth-b-host master br-demo
ip link set veth-b-host up

ip netns exec nsB ip link set veth-b up
ip netns exec nsB ip addr add 10.99.0.20/24 dev veth-b
ip netns exec nsB ip route add default via 10.99.0.1

echo "=== 4. 验证 ==="
ip netns exec nsA ping -c 2 10.99.0.20
ip netns exec nsB ping -c 2 10.99.0.10

echo "=== 5. 让 podA 能访问外网 ==="
# 启用主机转发
sysctl -w net.ipv4.ip_forward=1
# SNAT 出口流量
iptables -t nat -A POSTROUTING -s 10.99.0.0/24 ! -o br-demo -j MASQUERADE

ip netns exec nsA ping -c 2 8.8.8.8
# 通了

echo "=== 完成 ==="

ip netns del nsA
ip netns del nsB
ip link del br-demo

ip link del veth-a            # 删一头
ip link show veth-b            # 不存在了

$ ip netns list
# 空 / 只有手动 add 的

PID=$(crictl inspect ... | jq '.info.pid')
mkdir -p /var/run/netns
ln -s /proc/$PID/ns/net /var/run/netns/mypod-ns

# 现在能用 ip netns
ip netns exec mypod-ns ip addr
ip netns list                      # 能看到 mypod-ns 了

# 节点上几乎所有网络元素
$ ip link show | grep -E "^[0-9]+:"
1: lo: ...                                                      # 回环
2: eth0: ...                                                    # 物理网卡
3: docker0: ...                                                 # docker（可能存在残留）
4: cni0: ... master br0                                         # K8s pod bridge
5: flannel.1: ...                                               # overlay 设备（如有）
6: vxlan.calico: ...                                            # Calico VXLAN
7: cilium_host@cilium_net: ...                                  # Cilium
8: cilium_net@cilium_host: ...
9: vethABCD@if3: ... master cni0                                # pod A 的 host 端
10: vethEFGH@if3: ... master cni0                                # pod B 的 host 端
...

# 看 K8s pod CIDR 怎么分的
$ kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.podCIDR}{"\n"}{end}'
m1    10.244.0.0/24
m2    10.244.1.0/24
m3    10.244.2.0/24
m4    10.244.3.0/24
m5    10.244.4.0/24

# 看节点上的 CNI 配置
$ cat /etc/cni/net.d/*
{
  "cniVersion": "0.3.1",
  "name": "cbr0",
  "plugins": [
    { "type": "flannel", ... },
    { "type": "portmap", ... }
  ]
}

# 看 pod 网络配置（CNI 状态）
$ ls /var/lib/cni/                       # 一些 CNI 会在这里存状态

# 看 pod 实际用的 netns
$ for pod in $(crictl ps -q); do
    pid=$(crictl inspect $pod | jq -r '.info.pid')
    echo "pod=$pod pid=$pid netns=$(readlink /proc/$pid/ns/net)"
  done