问题

[root@master busybox]# kubectl get pod -nkube-system -owide
NAME                             READY   STATUS    RESTARTS   AGE     IP               NODE     NOMINATED NODE   READINESS GATES
coredns-5c98db65d4-8zjps         1/1     Running   1          2d23h   10.244.0.13      master   <none>           <none>
coredns-5c98db65d4-d2kth         1/1     Running   1          2d23h   10.244.0.14      master   <none>           <none>

[root@master busybox]# kubectl get pod -owide
NAME                      READY   STATUS    RESTARTS   AGE    IP            NODE     NOMINATED NODE   READINESS GATES
curl-6bf6db5c4f-pjld9     1/1     Running   1          3d     10.244.1.2    node2    <none>           <none>
gateway-99b655cc6-np685   1/1     Running   0          44s    10.244.0.54   master   <none>           <none>
test-post-start1          1/1     Running   0          115s   10.244.1.6    node2    <none>           <none>
test-post-start2          1/1     Running   0          115s   10.244.0.52   master   <none>           <none>
test-post-start3          1/1     Running   0          115s   10.244.0.53   master   <none>           <none>

如上所示，出现部署在master节点上的pod，无法解析gateway.default.svc.cluster.local域名，但是部署在node2，确可以解析，如上curl-6bf6db5c4f-pjld9,test-post-start1通过nslookup都可以解析.

# 报错
/ # nslookup gateway
nslookup: can't resolve '(null)': Name does not resolve

nslookup: can't resolve 'gateway': Try again

/ # nslookup gateway.default.svc.cluster.local
Server:    10.244.0.10
Address 1: 10.244.0.10

nslookup: can't resolve 'gateway.default.svc.cluster.local'

分析

进入master节点pod，直接通过coredns pod ip解析测试

kubectl exec -it test-post-start2 sh

/ # nslookup gateway.default.svc.cluster.local 10.244.0.13
Server:    10.244.0.13
Address 1: 10.244.0.13 10-244-0-13.kube-dns.kube-system.svc.cluster.local

Name:      gateway.default.svc.cluster.local
Address 1: 10.244.106.29 gateway.default.svc.cluster.local

发现直接通过coredns pod ip解析可以成功，证明coredns服务本身没有问题.

查看dns clusterIP.

[root@master ~]# kubectl get svc -nkube-system
NAME            TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                  AGE
kube-dns        ClusterIP   10.244.0.10    <none>        53/UDP,53/TCP,9153/TCP   21m

# 通过clusterIP解析域名失败
nslookup gateway.default.svc.cluster.local 10.244.0.10

通过以上测试证明问题出现在coredns service上.

解决

导出现有kube-dns service配置

kubectl get svc -nkube-system kube-dns -oyaml > kube-dns-svc.yaml

修改kube-dns-svc.yaml.

apiVersion: v1
kind: Service
metadata:
  annotations:
    prometheus.io/port: "9153"
    prometheus.io/scrape: "true"
  labels:
    k8s-app: kube-dns
    kubernetes.io/cluster-service: "true"
    kubernetes.io/name: KubeDNS
  name: kube-dns
  namespace: kube-system
spec:
  ports:
  - name: dns
    port: 53
    protocol: UDP
    targetPort: 53
  - name: dns-tcp
    port: 53
    protocol: TCP
    targetPort: 53
  - name: metrics
    port: 9153
    protocol: TCP
    targetPort: 9153
  selector:
    k8s-app: kube-dns
  sessionAffinity: None
  type: ClusterIP

kubectl apply -f kube-dns-svc.yaml

查看最新的coredns clusterIP,当前为10.244.47.231.

[root@master ~]# kubectl get svc -nkube-system
NAME            TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                  AGE
kube-dns        ClusterIP   10.244.47.231    <none>        53/UDP,53/TCP,9153/TCP   21m

进去之前无法解析的pod中测试，证明新的clusterIP没有问题.

nslookup gateway.default.svc.cluster.local 10.244.47.231

修改kubelet --clusterDNS,这样新创建的pod /etc/resolv.conf中nameserver为新的coredns clusterIP.

# 修改kubelet配置
vim  /var/lib/kubelet/config.yaml

# 找到clusterDNS
clusterDNS:
- 10.244.47.231

# 重启kubelet生效,注意k8s中所有节点都需要修改重启
systemctl restart kubelet.service

最后测试，新的pod中/etc/resolv.conf.解析没有问题.

/ # cat /etc/resolv.conf 
nameserver 10.244.47.231
search default.svc.cluster.local svc.cluster.local cluster.local
options ndots:5