Failed to create pod sandbox: rpc error: code = Unknown desc = [failed to set up sandbox container "63a01c2405a404eb56920f2d7af34f19ed88fef0519e76cb4e493fa3f0c4bba9" network for pod "web-nginx-5f769fdc6-dlnqq":
networkPlugin cni failed to set up pod "web-nginx-5f769fdc6-dlnqq_default" network: rpc error:
code = Unavailable desc = connection error: desc = "transport: Error while dialing dial unix /var/run/antrea/cni.sock: connect: connection refused",
failed to clean up sandbox container "63a01c2405a404eb56920f2d7af34f19ed88fef0519e76cb4e493fa3f0c4bba9" network for pod "web-nginx-5f769fdc6-dlnqq": networkPlugin cni failed to teardown
pod "web-nginx-5f769fdc6-dlnqq_default" network: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing dial unix /var/run/antrea/cni.sock: connect: connection refused"]
root@ubuntu:~# kubectl describe pod web-nginx-5f769fdc6-dlnqq
Name: web-nginx-5f769fdc6-dlnqq
Namespace: default
Priority: 0
Node: cloud/10.10.16.47
Start Time: Fri, 18 Jun 2021 10:03:43 +0800
Labels: app=web-nginx
pod-template-hash=5f769fdc6
Annotations: <none>
Status: Pending
IP:
IPs: <none>
Controlled By: ReplicaSet/web-nginx-5f769fdc6
Containers:
web2-worker:
Container ID:
Image: nginx
Image ID:
Port: 8087/TCP
Host Port: 0/TCP
State: Waiting
Reason: ContainerCreating
Ready: False
Restart Count: 0
Environment: <none>
Mounts:
/var/run/secrets/kubernetes.io/serviceaccount from default-token-ckv9x (ro)
Conditions:
Type Status
Initialized True
Ready False
ContainersReady False
PodScheduled True
Volumes:
default-token-ckv9x:
Type: Secret (a volume populated by a Secret)
SecretName: default-token-ckv9x
Optional: false
QoS Class: BestEffort
Node-Selectors: <none>
Tolerations: node.kubernetes.io/not-ready:NoExecute for 300s
node.kubernetes.io/unreachable:NoExecute for 300s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Scheduled <unknown> default-scheduler Successfully assigned default/web-nginx-5f769fdc6-dlnqq to cloud
Warning FailedCreatePodSandBox 51s kubelet, cloud Failed to create pod sandbox: rpc error: code = Unknown desc = [failed to set up sandbox container "63a01c2405a404eb56920f2d7af34f19ed88fef0519e76cb4e493fa3f0c4bba9" network for pod "web-nginx-5f769fdc6-dlnqq": networkPlugin cni failed to set up pod "web-nginx-5f769fdc6-dlnqq_default" network: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing dial unix /var/run/antrea/cni.sock: connect: connection refused", failed to clean up sandbox container "63a01c2405a404eb56920f2d7af34f19ed88fef0519e76cb4e493fa3f0c4bba9" network for pod "web-nginx-5f769fdc6-dlnqq": networkPlugin cni failed to teardown pod "web-nginx-5f769fdc6-dlnqq_default" network: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing dial unix /var/run/antrea/cni.sock: connect: connection refused"]
Normal SandboxChanged 2s (x5 over 51s) kubelet, cloud Pod sandbox changed, it will be killed and re-created.
root@cloud:~# journalctl -u kubelet -f
-- Logs begin at Tue 2020-10-20 19:26:58 CST. --
Jun 18 10:09:23 cloud kubelet[406675]: E0618 10:09:23.235161 406675 remote_runtime.go:144] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \"web-nginx-5f769fdc6-tt8mf_default\" network: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial unix /var/run/antrea/cni.sock: connect: connection refused\"" podSandboxID="3eca7a32f1c2fb9d72ad58ab7dae887b4c6db5bf7892809a2d3e6897c48655b8"
Jun 18 10:09:23 cloud kubelet[406675]: E0618 10:09:23.235212 406675 kuberuntime_manager.go:958] "Failed to stop sandbox" podSandboxID={Type:docker ID:3eca7a32f1c2fb9d72ad58ab7dae887b4c6db5bf7892809a2d3e6897c48655b8}
Jun 18 10:09:23 cloud kubelet[406675]: E0618 10:09:23.235301 406675 kuberuntime_manager.go:729] "killPodWithSyncResult failed" err="failed to \"KillPodSandbox\" for \"b045617d-721d-477f-8db4-62c4fdc0c358\" with KillPodSandboxError: \"rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \\\"web-nginx-5f769fdc6-tt8mf_default\\\" network: rpc error: code = Unavailable desc = connection error: desc = \\\"transport: Error while dialing dial unix /var/run/antrea/cni.sock: connect: connection refused\\\"\""
Jun 18 10:09:23 cloud kubelet[406675]: E0618 10:09:23.235369 406675 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"KillPodSandbox\" for \"b045617d-721d-477f-8db4-62c4fdc0c358\" with KillPodSandboxError: \"rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \\\"web-nginx-5f769fdc6-tt8mf_default\\\" network: rpc error: code = Unavailable desc = connection error: desc = \\\"transport: Error while dialing dial unix /var/run/antrea/cni.sock: connect: connection refused\\\"\"" pod="default/web-nginx-5f769fdc6-tt8mf" podUID=b045617d-721d-477f-8db4-62c4fdc0c358
Jun 18 10:09:28 cloud kubelet[406675]: I0618 10:09:28.217152 406675 cni.go:333] "CNI failed to retrieve network namespace path" err="cannot find network namespace for the terminated container \"63a01c2405a404eb56920f2d7af34f19ed88fef0519e76cb4e493fa3f0c4bba9\""
Jun 18 10:09:28 cloud kubelet[406675]: E0618 10:09:28.233226 406675 cni.go:380] "Error deleting pod from network" err="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial unix /var/run/antrea/cni.sock: connect: connection refused\"" pod="default/web-nginx-5f769fdc6-dlnqq" podSandboxID={Type:docker ID:63a01c2405a404eb56920f2d7af34f19ed88fef0519e76cb4e493fa3f0c4bba9} podNetnsPath="" networkType="antrea" networkName="antrea"
Jun 18 10:09:28 cloud kubelet[406675]: E0618 10:09:28.233972 406675 remote_runtime.go:144] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \"web-nginx-5f769fdc6-dlnqq_default\" network: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial unix /var/run/antrea/cni.sock: connect: connection refused\"" podSandboxID="63a01c2405a404eb56920f2d7af34f19ed88fef0519e76cb4e493fa3f0c4bba9"
Jun 18 10:09:28 cloud kubelet[406675]: E0618 10:09:28.234021 406675 kuberuntime_manager.go:958] "Failed to stop sandbox" podSandboxID={Type:docker ID:63a01c2405a404eb56920f2d7af34f19ed88fef0519e76cb4e493fa3f0c4bba9}
Jun 18 10:09:28 cloud kubelet[406675]: E0618 10:09:28.234103 406675 kuberuntime_manager.go:729] "killPodWithSyncResult failed" err="failed to \"KillPodSandbox\" for \"b44537e7-292f-481d-b347-f6df12f1e53a\" with KillPodSandboxError: \"rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \\\"web-nginx-5f769fdc6-dlnqq_default\\\" network: rpc error: code = Unavailable desc = connection error: desc = \\\"transport: Error while dialing dial unix /var/run/antrea/cni.sock: connect: connection refused\\\"\""
Jun 18 10:09:28 cloud kubelet[406675]: E0618 10:09:28.234183 406675 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"KillPodSandbox\" for \"b44537e7-292f-481d-b347-f6df12f1e53a\" with KillPodSandboxError: \"rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \\\"web-nginx-5f769fdc6-dlnqq_default\\\" network: rpc error: code = Unavailable desc = connection error: desc = \\\"transport: Error while dialing dial unix /var/run/antrea/cni.sock: connect: connection refused\\\"\"" pod="default/web-nginx-5f769fdc6-dlnqq" podUID=b44537e7-292f-481d-b347-f6df12f1e53a
Jun 18 10:09:35 cloud kubelet[406675]: I0618 10:09:35.217071 406675 cni.go:333] "CNI failed to retrieve network namespace path" err="cannot find network namespace for the terminated container \"3eca7a32f1c2fb9d72ad58ab7dae887b4c6db5bf7892809a2d3e6897c48655b8\""
Jun 18 10:09:35 cloud kubelet[406675]: E0618 10:09:35.232950 406675 cni.go:380] "Error deleting pod from network" err="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial unix /var/run/antrea/cni.sock: connect: connection refused\"" pod="default/web-nginx-5f769fdc6-tt8mf" podSandboxID={Type:docker ID:3eca7a32f1c2fb9d72ad58ab7dae887b4c6db5bf7892809a2d3e6897c48655b8} podNetnsPath="" networkType="antrea" networkName="antrea"
Jun 18 10:09:35 cloud kubelet[406675]: E0618 10:09:35.233568 406675 remote_runtime.go:144] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \"web-nginx-5f769fdc6-tt8mf_default\" network: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial unix /var/run/antrea/cni.sock: connect: connection refused\"" podSandboxID="3eca7a32f1c2fb9d72ad58ab7dae887b4c6db5bf7892809a2d3e6897c48655b8"
Jun 18 10:09:35 cloud kubelet[406675]: E0618 10:09:35.233611 406675 kuberuntime_manager.go:958] "Failed to stop sandbox" podSandboxID={Type:docker ID:3eca7a32f1c2fb9d72ad58ab7dae887b4c6db5bf7892809a2d3e6897c48655b8}
Jun 18 10:09:35 cloud kubelet[406675]: E0618 10:09:35.234523 406675 kuberuntime_manager.go:729] "killPodWithSyncResult failed" err="failed to \"KillPodSandbox\" for \"b045617d-721d-477f-8db4-62c4fdc0c358\" with KillPodSandboxError: \"rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \\\"web-nginx-5f769fdc6-tt8mf_default\\\" network: rpc error: code = Unavailable desc = connection error: desc = \\\"transport: Error while dialing dial unix /var/run/antrea/cni.sock: connect: connection refused\\\"\""
Jun 18 10:09:35 cloud kubelet[406675]: E0618 10:09:35.234612 406675 pod_workers.go:190] "Error syncing pod, skipping" err="failed to \"KillPodSandbox\" for \"b045617d-721d-477f-8db4-62c4fdc0c358\" with KillPodSandboxError: \"rpc error: code = Unknown desc = networkPlugin cni failed to teardown pod \\\"web-nginx-5f769fdc6-tt8mf_default\\\" network: rpc error: code = Unavailable desc = connection error: desc = \\\"transport: Error while dialing dial unix /var/run/antrea/cni.sock: connect: connection refused\\\"\"" pod="default/web-nginx-5f769fdc6-tt8mf" podUID=b045617d-721d-477f-8db4-62c4fdc0c358
^C
root@cloud:~#
root@ubuntu:~# kubectl get pods NAME READY STATUS RESTARTS AGE debian-6c44fc6956-ltsrt 0/1 CrashLoopBackOff 5065 17d mc1 2/2 Running 0 17d my-deployment-68bdbbb5cc-bbszv 0/1 ImagePullBackOff 0 36d my-deployment-68bdbbb5cc-nrst9 0/1 ImagePullBackOff 0 36d my-deployment-68bdbbb5cc-rlgzt 0/1 ImagePullBackOff 0 36d my-nginx-5dc4865748-jqx54 1/1 Running 2 36d my-nginx-5dc4865748-pcrbg 1/1 Running 2 36d nginx 0/1 ImagePullBackOff 0 36d nginx-deployment-6b474476c4-r6z5b 1/1 Running 0 9d nginx-deployment-6b474476c4-w6xh9 1/1 Running 0 9d web-nginx-5f769fdc6-dlnqq 0/1 ContainerCreating 0 7m49s web-nginx-5f769fdc6-tt8mf 0/1 ContainerCreating 0 7m49s root@ubuntu:~# kubectl describe daemonset cloud -n kube-system | grep Image | cut -d "/" -f 2 Error from server (NotFound): daemonsets.apps "cloud" not found root@ubuntu:~# kubectl describe daemonset web-nginx-5f769fdc6-dlnqq -n default | grep Image | cut -d "/" -f 2 Error from server (NotFound): daemonsets.apps "web-nginx-5f769fdc6-dlnqq" not found root@ubuntu:~# root@ubuntu:~# kubectl delete --namespace=default deployment web-nginx deployment.apps "web-nginx" deleted root@ubuntu:~# kubectl get pods NAME READY STATUS RESTARTS AGE debian-6c44fc6956-ltsrt 0/1 CrashLoopBackOff 5067 17d mc1 2/2 Running 0 17d my-deployment-68bdbbb5cc-bbszv 0/1 ImagePullBackOff 0 36d my-deployment-68bdbbb5cc-nrst9 0/1 ImagePullBackOff 0 36d my-deployment-68bdbbb5cc-rlgzt 0/1 ImagePullBackOff 0 36d my-nginx-5dc4865748-jqx54 1/1 Running 2 36d my-nginx-5dc4865748-pcrbg 1/1 Running 2 36d nginx 0/1 ImagePullBackOff 0 36d nginx-deployment-6b474476c4-r6z5b 1/1 Running 0 9d nginx-deployment-6b474476c4-w6xh9 1/1 Running 0 9d web-nginx-5f769fdc6-dlnqq 0/1 Terminating 0 14m web-nginx-5f769fdc6-tt8mf 0/1 Terminating 0 14m root@ubuntu:~#
把
/etc/cni/net.d/10-antrea.conflist 删除
root@cloud:~# ls /var/lib/cni/ cache root@cloud:~# ls /etc/cni/net.d/ 10-antrea.conflist 10-flannel.conflist root@cloud:~# rm -rf /etc/cni/net.d/10-antrea.conflist root@cloud:~#
kubectl apply -f web-anti-affinity.yaml 成功了
root@ubuntu:~# kubectl get pods NAME READY STATUS RESTARTS AGE debian-6c44fc6956-ltsrt 0/1 Completed 5069 18d mc1 2/2 Running 0 17d my-deployment-68bdbbb5cc-bbszv 0/1 ImagePullBackOff 0 36d my-deployment-68bdbbb5cc-nrst9 0/1 ImagePullBackOff 0 36d my-deployment-68bdbbb5cc-rlgzt 0/1 ImagePullBackOff 0 36d my-nginx-5dc4865748-jqx54 1/1 Running 2 36d my-nginx-5dc4865748-pcrbg 1/1 Running 2 36d nginx 0/1 ImagePullBackOff 0 36d nginx-deployment-6b474476c4-r6z5b 1/1 Running 0 9d nginx-deployment-6b474476c4-w6xh9 1/1 Running 0 9d web-nginx-5f769fdc6-779lg 1/1 Running 0 2s web-nginx-5f769fdc6-sstj4 1/1 Running 0 2s root@ubuntu:~#
root@ubuntu:~# kubectl get pods web-nginx-5f769fdc6-779lg -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES web-nginx-5f769fdc6-779lg 1/1 Running 0 102s 10.244.2.4 cloud <none> <none> root@ubuntu:~# kubectl get pods web-nginx-5f769fdc6-sstj4 -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES web-nginx-5f769fdc6-sstj4 1/1 Running 0 111s 10.244.2.5 cloud <none> <none> root@ubuntu:~#