DNS 深入：CoreDNS / resolv.conf / ndots / 解析坑

$ kubectl exec my-pod -- cat /etc/resolv.conf
search default.svc.cluster.local svc.cluster.local cluster.local
nameserver 10.96.0.10
options ndots:5

# 应用大量调用外部 API
curl https://api.example.com    # 1 个点 < 5 → 4 次 NXDOMAIN
curl https://github.com          # 1 个点 < 5 → 4 次 NXDOMAIN
curl https://prom.example.com   # 2 个点 < 5 → 4 次 NXDOMAIN

$ kubectl get cm -n kube-system coredns -o yaml
data:
  Corefile: |
    .:53 {
        errors
        health {
           lameduck 5s
        }
        ready
        kubernetes cluster.local in-addr.arpa ip6.arpa {
           pods insecure
           fallthrough in-addr.arpa ip6.arpa
           ttl 30
        }
        prometheus :9153
        forward . /etc/resolv.conf {
           max_concurrent 1000
        }
        cache 30
        loop
        reload
        loadbalance
    }

# CoreDNS pod 里看的 resolv.conf
$ kubectl exec -n kube-system coredns-xxx -- cat /etc/resolv.conf
nameserver 8.8.8.8
nameserver 1.1.1.1

$ kubectl exec test-pod -- dig SRV _http._tcp.my-headless.default.svc.cluster.local
;; ANSWER SECTION:
_http._tcp.my-headless.default.svc.cluster.local. 30 IN SRV 0 33 8080 10-244-0-5.my-headless.default.svc.cluster.local.
_http._tcp.my-headless.default.svc.cluster.local. 30 IN SRV 0 33 8080 10-244-1-6.my-headless.default.svc.cluster.local.
_http._tcp.my-headless.default.svc.cluster.local. 30 IN SRV 0 33 8080 10-244-2-7.my-headless.default.svc.cluster.local.

$ kubectl get pod -n kube-system -l k8s-app=kube-dns
NAME                       READY   STATUS    AGE
coredns-7b6c8df-abc123     1/1     Running   1d
coredns-7b6c8df-def456     1/1     Running   1d

$ kubectl logs -n kube-system coredns-7b6c8df-abc123 --tail=30

$ kubectl exec test-pod -- nslookup kubernetes.default
Server:    10.96.0.10
Address:   10.96.0.10:53
Name:      kubernetes.default.svc.cluster.local
Address:   10.96.0.1

# 或者用 dig（更详细）
$ kubectl exec test-pod -- dig kubernetes.default.svc.cluster.local @10.96.0.10

$ kubectl exec test-pod -- dig my-svc.my-ns.svc.cluster.local @10.96.0.10
;; QUESTION SECTION:
;my-svc.my-ns.svc.cluster.local. IN A

;; ANSWER SECTION:
my-svc.my-ns.svc.cluster.local. 30 IN A 10.96.1.5

;; status: NOERROR

# 在 pod 里
$ kubectl exec test-pod -- cat /etc/resolv.conf
search default.svc.cluster.local svc.cluster.local cluster.local
nameserver 10.96.0.10
options ndots:5

# 模拟 ndots:5 + search 的拼接
$ kubectl exec test-pod -- nslookup my-svc
Server:    10.96.0.10
Address:   10.96.0.10:53
Name:      my-svc.default.svc.cluster.local
Address:   10.96.1.5

# 在 pod 里测外部域名
$ kubectl exec test-pod -- dig github.com @10.96.0.10
;; ANSWER SECTION:
github.com. 60 IN A 140.82.114.4

# 也可以直接问节点上游
$ kubectl exec test-pod -- dig github.com @8.8.8.8

# 看 CoreDNS pod 自己能不能解析外部
$ kubectl exec -n kube-system coredns-xxx -- dig github.com

# 节点上抓 pod 的 DNS 流量
$ PID=$(crictl inspect $(crictl ps -q --name test-pod) | jq '.info.pid')
$ nsenter -t $PID -n tcpdump -i any -nn 'port 53' -A

# 触发解析
$ kubectl exec test-pod -- dig my-svc

# 看抓到的包

$ kubectl exec my-pod -- time curl https://api.example.com
# real    8m20.xxxs                  ← 8 分钟！

$ kubectl exec my-pod -- time dig api.example.com
# Query time: 8200 ms                ← DNS 慢

$ kubectl exec my-pod -- cat /etc/resolv.conf
options ndots:5
search default.svc.cluster.local ... example.com ... prod.example.com   ← 6 个 search domain

$ kubectl get events -n kube-system | grep coredns
... Killing coredns-xxx (OOMKilled)         ← OOM 了！

$ kubectl top pods -n kube-system | grep coredns
coredns-xxx    100m   170Mi/170Mi          ← 内存满

$ kubectl edit deploy -n kube-system coredns
# resources.limits.memory: 512Mi

$ kubectl exec my-pod -- for i in 1 2 3 4 5; do dig +timeout=2 github.com; done
# 第 1、4 次失败、其它通

# 看 CoreDNS forward 上游
$ kubectl exec -n kube-system coredns-xxx -- dig github.com
# 也间歇性失败

# 节点上看
$ ssh m1 'cat /etc/resolv.conf'
nameserver 192.168.1.1                # ← 老路由器、不稳

$ ssh m1 'dig github.com @192.168.1.1'
# 间歇性失败

$ kubectl exec mysql-0 -- dig mysql-1.mysql.default.svc.cluster.local
;; ANSWER SECTION:
mysql-1.mysql.default.svc.cluster.local. 30 IN A 10.244.1.6   ← 解析对了

$ kubectl exec mysql-0 -- ping 10.244.1.6
# 通

# 但应用配置文件用的是 mysql-1（短名）
$ kubectl exec mysql-0 -- nslookup mysql-1
# Could not resolve            ← 短名解析失败