Kubernetes CKA认证运维工程师笔记-Kubernetes故障排查
1. 应用程序故障排查
# 置顶type类型，是Pod还是deployment等，再接名称
kubectl describe TYPE/NAME
# 查看日志，一个Pod有两个容器的话，要用-c指定容器名
kubectl logs TYPE/NAME [-c CONTAINER]
# 进入容器中进行查看
kubectl exec POD [-c CONTAINER] --COMMAND [args...]
[root@k8s-master ~]# kubectl get pods
NAME READY STATUS RESTARTS AGE
client1 1/1 Running 5 2d23h
client2 0/1 ImagePullBackOff 4 2d23h
configmap-demo-pod 0/1 ImagePullBackOff 3 4d10h
my-pod2 1/1 Running 11 4d17h
nfs-client-provisioner-58d675cd5-dx7n4 0/1 ImagePullBackOff 6 4d11h
pod-taint 1/1 Running 9 10d
secret-demo-pod 1/1 Running 4 4d9h
sh 1/1 Running 6 4d10h
test-76846b5956-gftn9 1/1 Running 2 4d10h
test-76846b5956-r7s9k 1/1 Running 2 4d10h
test-76846b5956-trpbn 1/1 Running 2 4d10h
test2-78c4694588-87b9r 1/1 Running 5 4d12h
web-0 1/1 Running 4 4d11h
web-1 0/1 ImagePullBackOff 3 4d11h
web-2 0/1 ImagePullBackOff 3 4d11h
web-96d5df5c8-vc9kf 1/1 Running 3 3d
[root@k8s-master ~]# kubectl get pods
NAME READY STATUS RESTARTS AGE
client1 1/1 Running 5 2d23h
client2 0/1 ImagePullBackOff 4 2d23h
configmap-demo-pod 0/1 ImagePullBackOff 3 4d10h
my-pod2 1/1 Running 11 4d17h
nfs-client-provisioner-58d675cd5-dx7n4 0/1 ImagePullBackOff 6 4d11h
pod-taint 1/1 Running 9 10d
secret-demo-pod 1/1 Running 4 4d9h
sh 1/1 Running 6 4d10h
test-76846b5956-gftn9 1/1 Running 2 4d10h
test-76846b5956-r7s9k 1/1 Running 2 4d10h
test-76846b5956-trpbn 1/1 Running 2 4d10h
test2-78c4694588-87b9r 1/1 Running 5 4d12h
web-0 1/1 Running 4 4d11h
web-1 0/1 ImagePullBackOff 3 4d11h
web-2 0/1 ImagePullBackOff 3 4d11h
web-96d5df5c8-vc9kf 1/1 Running 3 3d
[root@k8s-master ~]# kubectl describe pod web-96d5df5c8-vc9kf
Name: web-96d5df5c8-vc9kf
Namespace: default
Priority: 0
Node: k8s-node2/10.0.0.63
Start Time: Wed, 22 Dec 2021 22:11:51 +0800
Labels: app=web
pod-template-hash=96d5df5c8
Annotations: cni.projectcalico.org/podIP: 10.244.169.158/32
cni.projectcalico.org/podIPs: 10.244.169.158/32
Status: Running
IP: 10.244.169.158
IPs:
IP: 10.244.169.158
Controlled By: ReplicaSet/web-96d5df5c8
Containers:
nginx:
Container ID: docker://f3243ba267e377896e3c5de8a2909d9dd12ed3b2a3fbd80b0094711e5a3f8c81
Image: nginx
Image ID: docker-pullable://nginx@sha256:366e9f1ddebdb844044c2fafd13b75271a9f620819370f8971220c2b330a9254
Port: <none>
Host Port: <none>
State: Running
Started: Sat, 25 Dec 2021 22:09:45 +0800
Last State: Terminated
Reason: Error
Exit Code: 255
Started: Fri, 24 Dec 2021 15:08:58 +0800
Finished: Sat, 25 Dec 2021 22:02:39 +0800
Ready: True
Restart Count: 3
Environment: <none>
Mounts:
/var/run/secrets/kubernetes.io/serviceaccount from default-token-8grtj (ro)
Conditions:
Type Status
Initialized True
Ready True
ContainersReady True
PodScheduled True
Volumes:
default-token-8grtj:
Type: Secret (a volume populated by a Secret)
SecretName: default-token-8grtj
Optional: false
QoS Class: BestEffort
Node-Selectors: <none>
Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal SandboxChanged 10m kubelet Pod sandbox changed, it will be killed and re-created.
Normal Pulling 9m38s kubelet Pulling image "nginx"
Normal SandboxChanged 4m5s kubelet Pod sandbox changed, it will be killed and re-created.
Normal Pulling 3m53s kubelet Pulling image "nginx"
Normal Pulled 3m37s kubelet Successfully pulled image "nginx" in 16.296403014s
Normal Created 3m36s kubelet Created container nginx
Normal Started 3m36s kubelet Started container nginx
[root@k8s-master ~]# kubectl logs web-96d5df5c8-vc9kf
/docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration
/docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/
/docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
10-listen-on-ipv6-by-default.sh: info: Getting the checksum of /etc/nginx/conf.d/default.conf
10-listen-on-ipv6-by-default.sh: info: Enabled listen on IPv6 in /etc/nginx/conf.d/default.conf
/docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh
/docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-processes.sh
/docker-entrypoint.sh: Configuration complete; ready for start up
2021/12/25 14:09:46 [notice] 1#1: using the "epoll" event method
2021/12/25 14:09:46 [notice] 1#1: nginx/1.21.4
2021/12/25 14:09:46 [notice] 1#1: built by gcc 10.2.1 20210110 (Debian 10.2.1-6)
2021/12/25 14:09:46 [notice] 1#1: OS: Linux 3.10.0-1160.45.1.el7.x86_64
2021/12/25 14:09:46 [notice] 1#1: getrlimit(RLIMIT_NOFILE): 1048576:1048576
2021/12/25 14:09:46 [notice] 1#1: start worker processes
2021/12/25 14:09:46 [notice] 1#1: start worker process 31
2021/12/25 14:09:46 [notice] 1#1: start worker process 32
[root@k8s-master ~]# kubectl exec -it web-96d5df5c8-vc9kf bash
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
root@web-96d5df5c8-vc9kf:/# exit
exit
[root@k8s-master ~]# kubectl exec -it web-96d5df5c8-vc9kf -- bash
root@web-96d5df5c8-vc9kf:/#
2. 管理节点与工作节点异常排查
管理节点组件：
- kube-apiserver
- kube-controller-manager
- kube-scheduler
工作节点组件：
- kubelet
- kube-proxy
Kubernetes集群架构图
需要先区分部署方式：
1、kubeadm
除kubelet外，其他组件均采用静态Pod启动
[root@k8s-master ~]# ls /etc/kubernetes/manifests/
etcd.yaml kube-controller-manager.yaml
kube-apiserver.yaml kube-scheduler.yaml
[root@k8s-master ~]# cat /var/lib/kubelet/config.yaml
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
anonymous:
enabled: false
webhook:
cacheTTL: 0s
enabled: true
x509:
clientCAFile: /etc/kubernetes/pki/ca.crt
authorization:
mode: Webhook
webhook:
cacheAuthorizedTTL: 0s
cacheUnauthorizedTTL: 0s
cgroupDriver: cgroupfs
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
cpuManagerReconcilePeriod: 0s
evictionPressureTransitionPeriod: 0s
fileCheckFrequency: 0s
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 0s
imageMinimumGCAge: 0s
kind: KubeletConfiguration
logging: {}
nodeStatusReportFrequency: 0s
nodeStatusUpdateFrequency: 0s
rotateCertificates: true
runtimeRequestTimeout: 0s
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 0s
syncFrequency: 0s
volumeStatsAggPeriod: 0s
[root@k8s-master ~]# kubectl get pods -n kube-system
NAME READY STATUS RESTARTS AGE
calico-kube-controllers-97769f7c7-c4cg5 1/1 Running 3 30h
calico-node-4pwdc 1/1 Running 16 33d
calico-node-9r6zd 1/1 Running 16 33d
calico-node-vqzdj 1/1 Running 17 33d
client1 1/1 Running 5 2d23h
coredns-6d56c8448f-gcgrh 1/1 Running 16 33d
coredns-6d56c8448f-mdl7c 1/1 Running 2 30h
etcd-k8s-master 1/1 Running 3 30h
filebeat-5pwh7 1/1 Running 11 10d
filebeat-pt848 1/1 Running 11 10d
kube-apiserver-k8s-master 1/1 Running 3 30h
kube-controller-manager-k8s-master 1/1 Running 3 30h
kube-proxy-87lbj 1/1 Running 3 30h
kube-proxy-mcdnv 1/1 Running 2 30h
kube-proxy-mchc9 1/1 Running 2 30h
kube-scheduler-k8s-master 1/1 Running 3 30h
metrics-server-84f9866fdf-rz676 1/1 Running 15 4d15h
2、二进制
所有组件均采用systemd管理
常见问题：
- 网络不通
- ping 节点
- telnet 节点端口
- 启动失败，一般配置文件或者依赖服务
- 平台不兼容
# kubeadm部署的，直接查看kube-apiserver-k8s-master的Pod日志
[root@k8s-master ~]# kubectl logs kube-apiserver-k8s-master -n kube-system
Flag --insecure-port has been deprecated, This flag will be removed in a future version.
I1225 14:12:18.558307 1 server.go:625] external host was not specified, using 10.0.0.61
I1225 14:12:18.558513 1 server.go:163] Version: v1.19.3
I1225 14:12:19.123232 1 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
I1225 14:12:19.123294 1 plugins.go:161] Loaded 10 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
I1225 14:12:19.124128 1 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
I1225 14:12:19.124167 1 plugins.go:161] Loaded 10 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
I1225 14:12:19.126549 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.126601 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.139669 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.139693 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.146921 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.146944 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.147383 1 client.go:360] parsed scheme: "passthrough"
I1225 14:12:19.147669 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:12:19.147718 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:12:19.194115 1 master.go:271] Using reconciler: lease
I1225 14:12:19.194533 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.194550 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.221352 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.221377 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.230469 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.230511 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.240139 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.240181 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.255518 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.255555 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.265105 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.265191 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.275038 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.275076 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.285281 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.285336 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.302076 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.302102 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.314415 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.314679 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.327616 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.327671 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.338580 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.338901 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.354401 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.354487 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.363624 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.363651 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.376090 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.376133 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.386480 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.386534 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.394978 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.395030 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.404842 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.404888 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.559645 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.559692 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.576723 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.576767 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.588265 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.588284 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.596125 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.596145 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.608161 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.608212 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.619144 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.619196 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.626852 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.626895 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.644521 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.644550 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.658031 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.658090 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.669971 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.670265 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.692800 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.692836 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.708784 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.708826 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.734898 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.735032 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.755957 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.755982 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.772847 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.772872 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.788862 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.788886 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.803723 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.803754 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.818516 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.818551 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.826818 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.826857 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.837298 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.837339 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.844194 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.844217 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.857209 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.857597 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.867066 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.867181 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.877262 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.877302 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.889062 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.889099 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.896457 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.902303 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.910393 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.910423 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.927814 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.927861 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.940076 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.940098 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.952012 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.952115 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.961099 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.961123 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.975537 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.975585 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.988067 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.988145 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:19.995939 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.995965 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:20.018436 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.018502 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:20.109379 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.109398 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:20.121750 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.121777 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:20.138751 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.138786 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:20.148112 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.151713 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:20.161554 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.161578 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:20.175335 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.175359 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:20.193425 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.194080 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:20.262691 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.262740 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:20.277204 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.277249 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:20.299607 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.299713 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:20.315284 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.315481 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:20.328823 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.328848 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:20.345828 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.345871 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:20.361304 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.361328 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
W1225 14:12:20.640827 1 genericapiserver.go:412] Skipping API batch/v2alpha1 because it has no resources.
W1225 14:12:20.659984 1 genericapiserver.go:412] Skipping API discovery.k8s.io/v1alpha1 because it has no resources.
W1225 14:12:20.685600 1 genericapiserver.go:412] Skipping API node.k8s.io/v1alpha1 because it has no resources.
W1225 14:12:20.717635 1 genericapiserver.go:412] Skipping API rbac.authorization.k8s.io/v1alpha1 because it has no resources.
W1225 14:12:20.722620 1 genericapiserver.go:412] Skipping API scheduling.k8s.io/v1alpha1 because it has no resources.
W1225 14:12:20.746581 1 genericapiserver.go:412] Skipping API storage.k8s.io/v1alpha1 because it has no resources.
W1225 14:12:20.774071 1 genericapiserver.go:412] Skipping API apps/v1beta2 because it has no resources.
W1225 14:12:20.774104 1 genericapiserver.go:412] Skipping API apps/v1beta1 because it has no resources.
I1225 14:12:20.794493 1 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
I1225 14:12:20.794524 1 plugins.go:161] Loaded 10 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
I1225 14:12:20.801886 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.801939 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:20.810029 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.810055 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:23.548796 1 dynamic_cafile_content.go:167] Starting request-header::/etc/kubernetes/pki/front-proxy-ca.crt
I1225 14:12:23.548865 1 dynamic_cafile_content.go:167] Starting client-ca-bundle::/etc/kubernetes/pki/ca.crt
I1225 14:12:23.549360 1 dynamic_serving_content.go:130] Starting serving-cert::/etc/kubernetes/pki/apiserver.crt::/etc/kubernetes/pki/apiserver.key
I1225 14:12:23.549780 1 secure_serving.go:197] Serving securely on [::]:6443
I1225 14:12:23.549835 1 dynamic_serving_content.go:130] Starting aggregator-proxy-cert::/etc/kubernetes/pki/front-proxy-client.crt::/etc/kubernetes/pki/front-proxy-client.key
I1225 14:12:23.549858 1 tlsconfig.go:240] Starting DynamicServingCertificateController
I1225 14:12:23.552336 1 cluster_authentication_trust_controller.go:440] Starting cluster_authentication_trust_controller controller
I1225 14:12:23.552372 1 shared_informer.go:240] Waiting for caches to sync for cluster_authentication_trust_controller
I1225 14:12:23.553014 1 apiservice_controller.go:97] Starting APIServiceRegistrationController
I1225 14:12:23.553087 1 cache.go:32] Waiting for caches to sync for APIServiceRegistrationController controller
I1225 14:12:23.553110 1 controller.go:83] Starting OpenAPI AggregationController
I1225 14:12:23.553250 1 dynamic_cafile_content.go:167] Starting client-ca-bundle::/etc/kubernetes/pki/ca.crt
I1225 14:12:23.553295 1 dynamic_cafile_content.go:167] Starting request-header::/etc/kubernetes/pki/front-proxy-ca.crt
I1225 14:12:23.561604 1 available_controller.go:404] Starting AvailableConditionController
I1225 14:12:23.561627 1 cache.go:32] Waiting for caches to sync for AvailableConditionController controller
I1225 14:12:23.561671 1 autoregister_controller.go:141] Starting autoregister controller
I1225 14:12:23.561678 1 cache.go:32] Waiting for caches to sync for autoregister controller
I1225 14:12:23.561791 1 customresource_discovery_controller.go:209] Starting DiscoveryController
E1225 14:12:23.666220 1 controller.go:152] Unable to remove old endpoints from kubernetes service: StorageError: key not found, Code: 1, Key: /registry/masterleases/10.0.0.61, ResourceVersion: 0, AdditionalErrorMsg:
I1225 14:12:23.954656 1 shared_informer.go:247] Caches are synced for cluster_authentication_trust_controller
I1225 14:12:23.995142 1 crdregistration_controller.go:111] Starting crd-autoregister controller
I1225 14:12:23.995162 1 shared_informer.go:240] Waiting for caches to sync for crd-autoregister
I1225 14:12:23.995170 1 shared_informer.go:247] Caches are synced for crd-autoregister
I1225 14:12:23.995261 1 controller.go:86] Starting OpenAPI controller
I1225 14:12:24.019280 1 naming_controller.go:291] Starting NamingConditionController
I1225 14:12:24.019448 1 establishing_controller.go:76] Starting EstablishingController
I1225 14:12:24.019750 1 nonstructuralschema_controller.go:186] Starting NonStructuralSchemaConditionController
I1225 14:12:24.021435 1 apiapproval_controller.go:186] Starting KubernetesAPIApprovalPolicyConformantConditionController
I1225 14:12:24.021505 1 crd_finalizer.go:266] Starting CRDFinalizer
I1225 14:12:24.084663 1 cache.go:39] Caches are synced for AvailableConditionController controller
I1225 14:12:24.089038 1 cache.go:39] Caches are synced for autoregister controller
I1225 14:12:24.155442 1 cache.go:39] Caches are synced for APIServiceRegistrationController controller
I1225 14:12:24.299909 1 trace.go:205] Trace[1198217794]: "Get" url:/api/v1/namespaces/ingress-nginx/secrets/nginx-ingress-serviceaccount-token-vh69r,user-agent:kube-apiserver/v1.19.3 (linux/amd64) kubernetes/1e11e4a,client:::1 (25-Dec-2021 14:12:23.797) (total time: 502ms):
Trace[1198217794]: ---"About to write a response" 502ms (14:12:00.299)
Trace[1198217794]: [502.112729ms] [502.112729ms] END
I1225 14:12:24.318639 1 controller.go:606] quota admission added evaluator for: leases.coordination.k8s.io
I1225 14:12:24.356155 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:24.356216 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:24.389194 1 trace.go:205] Trace[414373803]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/k8s-node2,user-agent:kubelet/v1.19.0 (linux/amd64) kubernetes/e199641,client:10.0.0.63 (25-Dec-2021 14:12:23.849) (total time: 539ms):
Trace[414373803]: ---"About to write a response" 539ms (14:12:00.389)
Trace[414373803]: [539.865826ms] [539.865826ms] END
I1225 14:12:24.389582 1 trace.go:205] Trace[346194256]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/k8s-node1,user-agent:kubelet/v1.19.3 (linux/amd64) kubernetes/1e11e4a,client:10.0.0.62 (25-Dec-2021 14:12:23.761) (total time: 627ms):
Trace[346194256]: ---"About to write a response" 627ms (14:12:00.389)
Trace[346194256]: [627.763742ms] [627.763742ms] END
I1225 14:12:24.393405 1 trace.go:205] Trace[538299640]: "Get" url:/api/v1/namespaces/kube-system/pods/kube-apiserver-k8s-master,user-agent:kubelet/v1.19.3 (linux/amd64) kubernetes/1e11e4a,client:10.0.0.61 (25-Dec-2021 14:12:23.845) (total time: 547ms):
Trace[538299640]: ---"About to write a response" 510ms (14:12:00.356)
Trace[538299640]: [547.414287ms] [547.414287ms] END
I1225 14:12:24.512082 1 trace.go:205] Trace[82502510]: "Create" url:/api/v1/namespaces/default/events,user-agent:kubelet/v1.19.3 (linux/amd64) kubernetes/1e11e4a,client:10.0.0.61 (25-Dec-2021 14:12:23.846) (total time: 665ms):
Trace[82502510]: ---"Object stored in database" 665ms (14:12:00.511)
Trace[82502510]: [665.364934ms] [665.364934ms] END
I1225 14:12:24.516643 1 trace.go:205] Trace[1819760371]: "GuaranteedUpdate etcd3" type:*core.Event (25-Dec-2021 14:12:23.818) (total time: 698ms):
Trace[1819760371]: ---"Transaction prepared" 459ms (14:12:00.277)
Trace[1819760371]: ---"Transaction committed" 238ms (14:12:00.516)
Trace[1819760371]: [698.586941ms] [698.586941ms] END
I1225 14:12:24.523401 1 trace.go:205] Trace[1243567460]: "Patch" url:/api/v1/namespaces/default/events/configmap-demo-pod.16c404be8eee341b,user-agent:kubelet/v1.19.3 (linux/amd64) kubernetes/1e11e4a,client:10.0.0.62 (25-Dec-2021 14:12:23.795) (total time: 721ms):
Trace[1243567460]: ---"Object stored in database" 696ms (14:12:00.516)
Trace[1243567460]: [721.328661ms] [721.328661ms] END
I1225 14:12:24.629195 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:24.635269 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:24.851524 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:24.851565 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:24.925875 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:24.925902 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:24.941178 1 storage_scheduling.go:143] all system priority classes are created successfully or already exist.
I1225 14:12:25.002392 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:25.002477 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:25.093867 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:25.093912 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
E1225 14:12:25.245098 1 customresource_handler.go:668] error building openapi models for hostendpoints.crd.projectcalico.org: ERROR $root.definitions.org.projectcalico.crd.v1.HostEndpoint.properties.spec.properties.ports.items.<array>.properties.protocol has invalid property: anyOf
I1225 14:12:25.245607 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:25.245627 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:25.277321 1 controller.go:132] OpenAPI AggregationController: action for item : Nothing (removed from the queue).
E1225 14:12:25.283228 1 customresource_handler.go:668] error building openapi models for felixconfigurations.crd.projectcalico.org: ERROR $root.definitions.org.projectcalico.crd.v1.FelixConfiguration.properties.spec.properties.kubeNodePortRanges.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.FelixConfiguration.properties.spec.properties.natPortRange has invalid property: anyOf
I1225 14:12:25.284239 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:25.284261 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
E1225 14:12:25.329108 1 customresource_handler.go:668] error building openapi models for globalnetworkpolicies.crd.projectcalico.org: ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.egress.items.<array>.properties.destination.properties.notPorts.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.egress.items.<array>.properties.destination.properties.ports.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.egress.items.<array>.properties.notProtocol has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.egress.items.<array>.properties.protocol has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.egress.items.<array>.properties.source.properties.notPorts.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.egress.items.<array>.properties.source.properties.ports.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.destination.properties.notPorts.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.destination.properties.ports.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.notProtocol has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.protocol has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.source.properties.notPorts.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.source.properties.ports.items.<array> has invalid property: anyOf
I1225 14:12:25.330596 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:25.330710 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:25.357189 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:25.357217 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:25.392966 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:25.392992 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
E1225 14:12:25.438707 1 customresource_handler.go:668] error building openapi models for networkpolicies.crd.projectcalico.org: ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.egress.items.<array>.properties.destination.properties.notPorts.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.egress.items.<array>.properties.destination.properties.ports.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.egress.items.<array>.properties.notProtocol has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.egress.items.<array>.properties.protocol has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.egress.items.<array>.properties.source.properties.notPorts.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.egress.items.<array>.properties.source.properties.ports.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.destination.properties.notPorts.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.destination.properties.ports.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.notProtocol has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.protocol has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.source.properties.notPorts.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.source.properties.ports.items.<array> has invalid property: anyOf
I1225 14:12:25.439540 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:25.439593 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:25.448117 1 trace.go:205] Trace[1794566532]: "Get" url:/apis/rbac.authorization.k8s.io/v1/clusterroles/system:aggregate-to-edit,user-agent:kube-apiserver/v1.19.3 (linux/amd64) kubernetes/1e11e4a,client:::1 (25-Dec-2021 14:12:24.878) (total time: 569ms):
Trace[1794566532]: ---"About to write a response" 569ms (14:12:00.447)
Trace[1794566532]: [569.28003ms] [569.28003ms] END
I1225 14:12:25.654884 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:25.654910 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:26.441116 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:26.441157 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:12:26.579163 1 controller.go:132] OpenAPI AggregationController: action for item k8s_internal_local_delegation_chain_0000000000: Nothing (removed from the queue).
E1225 14:12:29.106613 1 available_controller.go:437] v1beta1.metrics.k8s.io failed with: failing or missing response from https://10.103.249.20:443/apis/metrics.k8s.io/v1beta1: Get "https://10.103.249.20:443/apis/metrics.k8s.io/v1beta1": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)
E1225 14:12:34.125834 1 available_controller.go:437] v1beta1.metrics.k8s.io failed with: failing or missing response from https://10.103.249.20:443/apis/metrics.k8s.io/v1beta1: Get "https://10.103.249.20:443/apis/metrics.k8s.io/v1beta1": context deadline exceeded
E1225 14:12:39.126932 1 available_controller.go:437] v1beta1.metrics.k8s.io failed with: failing or missing response from https://10.103.249.20:443/apis/metrics.k8s.io/v1beta1: Get "https://10.103.249.20:443/apis/metrics.k8s.io/v1beta1": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)
I1225 14:12:41.508946 1 controller.go:606] quota admission added evaluator for: endpoints
I1225 14:12:54.169433 1 client.go:360] parsed scheme: "passthrough"
I1225 14:12:54.169468 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:12:54.169476 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:12:56.191213 1 controller.go:606] quota admission added evaluator for: endpointslices.discovery.k8s.io
I1225 14:12:56.631645 1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:56.631729 1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379 <nil> 0 <nil>}]
I1225 14:13:36.699461 1 client.go:360] parsed scheme: "passthrough"
I1225 14:13:36.699504 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:13:36.699512 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:14:12.928690 1 client.go:360] parsed scheme: "passthrough"
I1225 14:14:12.928831 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:14:12.928859 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:14:51.609220 1 client.go:360] parsed scheme: "passthrough"
I1225 14:14:51.609377 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:14:51.609409 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:15:30.414981 1 client.go:360] parsed scheme: "passthrough"
I1225 14:15:30.415048 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:15:30.415057 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:16:13.416069 1 client.go:360] parsed scheme: "passthrough"
I1225 14:16:13.416140 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:16:13.416158 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:16:53.202182 1 client.go:360] parsed scheme: "passthrough"
I1225 14:16:53.202277 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:16:53.202288 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:17:27.709485 1 client.go:360] parsed scheme: "passthrough"
I1225 14:17:27.709530 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:17:27.709542 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:18:10.159300 1 client.go:360] parsed scheme: "passthrough"
I1225 14:18:10.159338 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:18:10.159345 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:18:40.716569 1 client.go:360] parsed scheme: "passthrough"
I1225 14:18:40.716701 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:18:40.716722 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:19:24.247113 1 client.go:360] parsed scheme: "passthrough"
I1225 14:19:24.247185 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:19:24.247219 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:20:08.987275 1 client.go:360] parsed scheme: "passthrough"
I1225 14:20:08.987543 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:20:08.987583 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:20:44.858512 1 client.go:360] parsed scheme: "passthrough"
I1225 14:20:44.858557 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:20:44.858569 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:21:21.613762 1 client.go:360] parsed scheme: "passthrough"
I1225 14:21:21.613892 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:21:21.614077 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:21:52.143822 1 client.go:360] parsed scheme: "passthrough"
I1225 14:21:52.143911 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:21:52.143929 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:22:27.359651 1 client.go:360] parsed scheme: "passthrough"
I1225 14:22:27.359762 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:22:27.359787 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:23:11.063713 1 client.go:360] parsed scheme: "passthrough"
I1225 14:23:11.063746 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:23:11.063754 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:23:42.744602 1 client.go:360] parsed scheme: "passthrough"
I1225 14:23:42.744670 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:23:42.744688 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:24:15.053047 1 client.go:360] parsed scheme: "passthrough"
I1225 14:24:15.053141 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:24:15.053167 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:24:57.281040 1 client.go:360] parsed scheme: "passthrough"
I1225 14:24:57.286666 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:24:57.286712 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:25:38.863844 1 client.go:360] parsed scheme: "passthrough"
I1225 14:25:38.863903 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:25:38.863912 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:26:18.572451 1 client.go:360] parsed scheme: "passthrough"
I1225 14:26:18.572482 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:26:18.572489 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:26:53.678319 1 client.go:360] parsed scheme: "passthrough"
I1225 14:26:53.678531 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:26:53.678573 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:27:36.433874 1 client.go:360] parsed scheme: "passthrough"
I1225 14:27:36.434093 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:27:36.434125 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:28:18.084057 1 client.go:360] parsed scheme: "passthrough"
I1225 14:28:18.084239 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:28:18.084255 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:28:50.563060 1 client.go:360] parsed scheme: "passthrough"
I1225 14:28:50.563113 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:28:50.563124 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:29:21.855603 1 client.go:360] parsed scheme: "passthrough"
I1225 14:29:21.855751 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:29:21.856461 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:29:52.347034 1 client.go:360] parsed scheme: "passthrough"
I1225 14:29:52.347112 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I1225 14:29:52.347130 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
# 二进制部署的，用journalctl -u kube-apiserver
[root@k8s-master ~]# journalctl -u kube-apiserver
-- No entries --
[root@k8s-master ~]# journalctl -u kubelet
-- Logs begin at Sat 2021-12-25 22:12:00 CST, end at Sat 2021-12-25 22:40:02 CST. --
Dec 25 22:12:07 k8s-master systemd[1]: Started kubelet: The Kubernetes Node Agent.
Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.729022 1419 server.go:411] Version: v1.19.3
Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.729772 1419 server.go:831] Client rotation is on, will bootstrap in background
Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.739425 1419 certificate_store.go:130] Loading cert/key pair from "/var/lib/kubelet/pki/kubelet-client-current.pem".
Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.745546 1419 dynamic_cafile_content.go:167] Starting client-ca-bundle::/etc/kubernetes/pki/ca.crt
Dec 25 22:12:10 k8s-master kubelet[1419]: W1225 22:12:10.157376 1419 nvidia.go:61] NVIDIA GPU metrics will not be available: no NVIDIA devices found
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.208464 1419 server.go:640] --cgroups-per-qos enabled, but --cgroup-root was not specified. defaulting to /
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.209254 1419 container_manager_linux.go:276] container manager verified user specified cgroup-root exists: []
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.209278 1419 container_manager_linux.go:281] Creating Container Manager object based on Node Config: {RuntimeCgroupsName:
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.212786 1419 topology_manager.go:126] [topologymanager] Creating topology manager with none policy
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.212810 1419 container_manager_linux.go:311] [topologymanager] Initializing Topology Manager with none policy
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.212815 1419 container_manager_linux.go:316] Creating device plugin manager: true
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.222255 1419 client.go:77] Connecting to docker on unix:///var/run/docker.sock
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.222336 1419 client.go:94] Start docker client with request timeout=2m0s
Dec 25 22:12:10 k8s-master kubelet[1419]: W1225 22:12:10.261344 1419 docker_service.go:565] Hairpin mode set to "promiscuous-bridge" but kubenet is not enabled, falling back to
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.261377 1419 docker_service.go:241] Hairpin mode set to "hairpin-veth"
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.560569 1419 docker_service.go:256] Docker cri networking managed by cni
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.591354 1419 docker_service.go:261] Docker Info: &{ID:LZWZ:7SPV:BJT7:3OAX:HPZJ:2U5R:3D3E:SXVB:A5PX:PJX3:3IHG:OEDN Contain
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.591426 1419 docker_service.go:274] Setting cgroupDriver to cgroupfs
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.634518 1419 remote_runtime.go:59] parsed scheme: ""
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.634540 1419 remote_runtime.go:59] scheme "" not registered, fallback to default scheme
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635527 1419 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{/var/run/dockershim.sock <nil> 0 <nil>}] <ni
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635572 1419 clientconn.go:948] ClientConn switching balancer to "pick_first"
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635629 1419 remote_image.go:50] parsed scheme: ""
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635637 1419 remote_image.go:50] scheme "" not registered, fallback to default scheme
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635651 1419 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{/var/run/dockershim.sock <nil> 0 <nil>}] <ni
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635657 1419 clientconn.go:948] ClientConn switching balancer to "pick_first"
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635731 1419 kubelet.go:261] Adding pod path: /etc/kubernetes/manifests
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635762 1419 kubelet.go:273] Watching apiserver
Dec 25 22:12:10 k8s-master kubelet[1419]: E1225 22:12:10.678016 1419 reflector.go:127] k8s.io/kubernetes/pkg/kubelet/config/apiserver.go:46: Failed to watch *v1.Pod: failed to l
Dec 25 22:12:10 k8s-master kubelet[1419]: E1225 22:12:10.678147 1419 reflector.go:127] k8s.io/kubernetes/pkg/kubelet/kubelet.go:438: Failed to watch *v1.Node: failed to list *v1
Dec 25 22:12:10 k8s-master kubelet[1419]: E1225 22:12:10.678221 1419 reflector.go:127] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Service: failed to list *v1
Dec 25 22:12:10 k8s-master kubelet[1419]: E1225 22:12:10.977478 1419 aws_credentials.go:77] while getting AWS credentials NoCredentialProviders: no valid providers in chain. Dep
Dec 25 22:12:10 k8s-master kubelet[1419]: For verbose messaging see aws.Config.CredentialsChainVerboseErrors
Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.040815 1419 kuberuntime_manager.go:214] Container runtime docker initialized, version: 20.10.11, apiVersion: 1.41.0
Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.042898 1419 server.go:1147] Started kubelet
Dec 25 22:12:11 k8s-master kubelet[1419]: E1225 22:12:11.044515 1419 kubelet.go:1218] Image garbage collection failed once. Stats initialization may not have completed yet: fail
Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.053448 1419 fs_resource_analyzer.go:64] Starting FS ResourceAnalyzer
Dec 25 22:12:11 k8s-master kubelet[1419]: E1225 22:12:11.056752 1419 event.go:273] Unable to write event: 'Post "https://10.0.0.61:6443/api/v1/namespaces/default/events": dial t
Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.062250 1419 volume_manager.go:265] Starting Kubelet Volume Manager
Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.062322 1419 server.go:152] Starting to listen on 0.0.0.0:10250
Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.071183 1419 server.go:424] Adding debug handlers to kubelet server.
Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.079514 1419 desired_state_of_world_populator.go:139] Desired state populator starts to run
Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.088034 1419 topology_manager.go:219] [topologymanager] RemoveContainer - Container ID: db716200328937af6f50e1cd3c23d1391
Dec 25 22:12:11 k8s-master kubelet[1419]: E1225 22:12:11.102900 1419 controller.go:136] failed to ensure node lease exists, will retry in 200ms, error: Get "https://10.0.0.61:64
Dec 25 22:12:11 k8s-master kubelet[1419]: E1225 22:12:11.103298 1419 reflector.go:127] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.CSIDriver: failed to list *
Dec 25 22:12:11 k8s-master kubelet[1419]: E1225 22:12:11.179554 1419 kubelet.go:2183] node "k8s-master" not found
Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.186533 1419 client.go:87] parsed scheme: "unix"
[root@k8s-master ~]# journalctl -u kubelet > a.txt
[root@k8s-master ~]# more a.txt
-- Logs begin at Sat 2021-12-25 22:12:00 CST, end at Sat 2021-12-25 22:40:02 CST. --
Dec 25 22:12:07 k8s-master systemd[1]: Started kubelet: The Kubernetes Node Agent.
Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.729022 1419 server.go:411] Version: v1.19.3
Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.729772 1419 server.go:831] Client rotation is on, will bootstrap in background
Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.739425 1419 certificate_store.go:130] Loading cert/key pair from "/var/lib/kubelet/pki/kubelet-client-current.pem".
Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.745546 1419 dynamic_cafile_content.go:167] Starting client-ca-bundle::/etc/kubernetes/pki/ca.crt
Dec 25 22:12:10 k8s-master kubelet[1419]: W1225 22:12:10.157376 1419 nvidia.go:61] NVIDIA GPU metrics will not be available: no NVIDIA devices found
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.208464 1419 server.go:640] --cgroups-per-qos enabled, but --cgroup-root was not specified. defaulting to /
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.209254 1419 container_manager_linux.go:276] container manager verified user specified cgroup-root exists: []
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.209278 1419 container_manager_linux.go:281] Creating Container Manager object based on Node Config: {RuntimeCgroupsName:
SystemCgroupsName: KubeletCgroupsName: ContainerRuntime:docker CgroupsPerQOS:true CgroupRoot:/ CgroupDriver:cgroupfs KubeletRootDir:/var/lib/kubelet ProtectKernelDefaults:false No
deAllocatableConfig:{KubeReservedCgroupName: SystemReservedCgroupName: ReservedSystemCPUs: EnforceNodeAllocatable:map[pods:{}] KubeReserved:map[] SystemReserved:map[] HardEvictionT
hresholds:[{Signal:nodefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.1} GracePeriod:0s MinReclaim:<nil>} {Signal:nodefs.inodesFree Operator:LessThan Value:{Quan
tity:<nil> Percentage:0.05} GracePeriod:0s MinReclaim:<nil>} {Signal:imagefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.15} GracePeriod:0s MinReclaim:<nil>} {Si
gnal:memory.available Operator:LessThan Value:{Quantity:100Mi Percentage:0} GracePeriod:0s MinReclaim:<nil>}]} QOSReserved:map[] ExperimentalCPUManagerPolicy:none ExperimentalCPUMa
nagerReconcilePeriod:10s ExperimentalPodPidsLimit:-1 EnforceCPULimits:true CPUCFSQuotaPeriod:100ms ExperimentalTopologyManagerPolicy:none}
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.212786 1419 topology_manager.go:126] [topologymanager] Creating topology manager with none policy
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.212810 1419 container_manager_linux.go:311] [topologymanager] Initializing Topology Manager with none policy
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.212815 1419 container_manager_linux.go:316] Creating device plugin manager: true
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.222255 1419 client.go:77] Connecting to docker on unix:///var/run/docker.sock
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.222336 1419 client.go:94] Start docker client with request timeout=2m0s
Dec 25 22:12:10 k8s-master kubelet[1419]: W1225 22:12:10.261344 1419 docker_service.go:565] Hairpin mode set to "promiscuous-bridge" but kubenet is not enabled, falling back to
"hairpin-veth"
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.261377 1419 docker_service.go:241] Hairpin mode set to "hairpin-veth"
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.560569 1419 docker_service.go:256] Docker cri networking managed by cni
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.591354 1419 docker_service.go:261] Docker Info: &{ID:LZWZ:7SPV:BJT7:3OAX:HPZJ:2U5R:3D3E:SXVB:A5PX:PJX3:3IHG:OEDN Contain
ers:27 ContainersRunning:0 ContainersPaused:0 ContainersStopped:27 Images:16 Driver:overlay2 DriverStatus:[[Backing Filesystem xfs] [Supports d_type true] [Native Overlay Diff true
] [userxattr false]] SystemStatus:[] Plugins:{Volume:[local] Network:[bridge host ipvlan macvlan null overlay] Authorization:[] Log:[awslogs fluentd gcplogs gelf journald json-file
local logentries splunk syslog]} MemoryLimit:true SwapLimit:true KernelMemory:true KernelMemoryTCP:true CPUCfsPeriod:true CPUCfsQuota:true CPUShares:true CPUSet:true PidsLimit:tru
e IPv4Forwarding:true BridgeNfIptables:true BridgeNfIP6tables:true Debug:false NFd:25 OomKillDisable:true NGoroutines:34 SystemTime:2021-12-25T22:12:10.561900318+08:00 LoggingDrive
r:json-file CgroupDriver:cgroupfs NEventsListener:0 KernelVersion:3.10.0-1160.45.1.el7.x86_64 OperatingSystem:CentOS Linux 7 (Core) OSType:linux Architecture:x86_64 IndexServerAddr
ess:https://index.docker.io/v1/ RegistryConfig:0xc00067cc40 NCPU:2 MemTotal:1907732480 GenericResources:[] DockerRootDir:/var/lib/docker HTTPProxy: HTTPSProxy: NoProxy: Name:k8s-ma
ster Labels:[] ExperimentalBuild:false ServerVersion:20.10.11 ClusterStore: ClusterAdvertise: Runtimes:map[io.containerd.runc.v2:{Path:runc Args:[]} io.containerd.runtime.v1.linux:
{Path:runc Args:[]} runc:{Path:runc Args:[]}] DefaultRuntime:runc Swarm:{NodeID: NodeAddr: LocalNodeState:inactive ControlAvailable:false Error: RemoteManagers:[] Nodes:0 Managers:
0 Cluster:<nil> Warnings:[]} LiveRestoreEnabled:false Isolation: InitBinary:docker-init ContainerdCommit:{ID:7b11cfaabd73bb80907dd23182b9347b4245eb5d Expected:7b11cfaabd73bb80907dd
23182b9347b4245eb5d} RuncCommit:{ID:v1.0.2-0-g52b36a2 Expected:v1.0.2-0-g52b36a2} InitCommit:{ID:de40ad0 Expected:de40ad0} SecurityOptions:[name=seccomp,profile=default] ProductLic
ense: Warnings:[]}
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.591426 1419 docker_service.go:274] Setting cgroupDriver to cgroupfs
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.634518 1419 remote_runtime.go:59] parsed scheme: ""
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.634540 1419 remote_runtime.go:59] scheme "" not registered, fallback to default scheme
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635527 1419 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{/var/run/dockershim.sock <nil> 0 <nil>}] <ni
l> <nil>}
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635572 1419 clientconn.go:948] ClientConn switching balancer to "pick_first"
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635629 1419 remote_image.go:50] parsed scheme: ""
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635637 1419 remote_image.go:50] scheme "" not registered, fallback to default scheme
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635651 1419 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{/var/run/dockershim.sock <nil> 0 <nil>}] <ni
l> <nil>}
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635657 1419 clientconn.go:948] ClientConn switching balancer to "pick_first"
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635731 1419 kubelet.go:261] Adding pod path: /etc/kubernetes/manifests
[root@k8s-master ~]# cat /usr/lib/systemd/system/kubelet.service
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=https://kubernetes.io/docs/
Wants=network-online.target
After=network-online.target
[Service]
ExecStart=/usr/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10
[Install]
WantedBy=multi-user.target
[root@k8s-master ~]# /usr/bin/kubelet
I1225 22:44:25.980513 28854 server.go:411] Version: v1.19.3
W1225 22:44:25.980849 28854 server.go:553] standalone mode, no API client
W1225 22:44:25.980983 28854 container_manager_linux.go:951] CPUAccounting not enabled for pid: 28854
W1225 22:44:25.980991 28854 container_manager_linux.go:954] MemoryAccounting not enabled for pid: 28854
W1225 22:44:26.082976 28854 nvidia.go:61] NVIDIA GPU metrics will not be available: no NVIDIA devices found
W1225 22:44:26.127990 28854 server.go:468] No api server defined - no events will be sent to API server.
I1225 22:44:26.128031 28854 server.go:640] --cgroups-per-qos enabled, but --cgroup-root was not specified. defaulting to /
I1225 22:44:26.128393 28854 container_manager_linux.go:276] container manager verified user specified cgroup-root exists: []
I1225 22:44:26.128413 28854 container_manager_linux.go:281] Creating Container Manager object based on Node Config: {RuntimeCgroupsName: SystemCgroupsName: KubeletCgroupsName: ContainerRuntime:docker CgroupsPerQOS:true CgroupRoot:/ CgroupDriver:cgroupfs KubeletRootDir:/var/lib/kubelet ProtectKernelDefaults:false NodeAllocatableConfig:{KubeReservedCgroupName: SystemReservedCgroupName: ReservedSystemCPUs: EnforceNodeAllocatable:map[pods:{}] KubeReserved:map[] SystemReserved:map[] HardEvictionThresholds:[{Signal:nodefs.inodesFree Operator:LessThan Value:{Quantity:<nil> Percentage:0.05} GracePeriod:0s MinReclaim:<nil>} {Signal:imagefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.15} GracePeriod:0s MinReclaim:<nil>} {Signal:memory.available Operator:LessThan Value:{Quantity:100Mi Percentage:0} GracePeriod:0s MinReclaim:<nil>} {Signal:nodefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.1} GracePeriod:0s MinReclaim:<nil>}]} QOSReserved:map[] ExperimentalCPUManagerPolicy:none ExperimentalCPUManagerReconcilePeriod:10s ExperimentalPodPidsLimit:-1 EnforceCPULimits:true CPUCFSQuotaPeriod:100ms ExperimentalTopologyManagerPolicy:none}
I1225 22:44:26.128818 28854 topology_manager.go:126] [topologymanager] Creating topology manager with none policy
I1225 22:44:26.128827 28854 container_manager_linux.go:311] [topologymanager] Initializing Topology Manager with none policy
I1225 22:44:26.128833 28854 container_manager_linux.go:316] Creating device plugin manager: true
I1225 22:44:26.129807 28854 client.go:77] Connecting to docker on unix:///var/run/docker.sock
I1225 22:44:26.129830 28854 client.go:94] Start docker client with request timeout=2m0s
W1225 22:44:26.183754 28854 docker_service.go:565] Hairpin mode set to "promiscuous-bridge" but kubenet is not enabled, falling back to "hairpin-veth"
I1225 22:44:26.183783 28854 docker_service.go:241] Hairpin mode set to "hairpin-veth"
I1225 22:44:26.247433 28854 docker_service.go:256] Docker cri networking managed by kubernetes.io/no-op
......
3. Service访问异常排查
Service工作流程图
Service一般是访问不通，有以下可能性：
- Service是否关联Pod？
# 查看标签是否正确，已关联到Pod；是否已创建Pod
[root@k8s-master ~]# kubectl get ep
NAME ENDPOINTS AGE
fuseim.pri-ifs <none> 4d12h
kubernetes 10.0.0.61:6443 33d
my-dep <none> 30d
my-service 10.244.36.119:80,10.244.36.122:80,10.244.36.98:80 24d
nginx 10.244.36.119:80,10.244.36.122:80,10.244.36.98:80 4d11h
[root@k8s-master ~]# kubectl get pods
NAME READY STATUS RESTARTS AGE
client1 1/1 Running 5 3d
client2 1/1 Running 5 3d
configmap-demo-pod 1/1 Running 4 4d10h
my-pod2 1/1 Running 11 4d17h
nfs-client-provisioner-58d675cd5-dx7n4 1/1 Running 7 4d12h
pod-taint 1/1 Running 9 10d
secret-demo-pod 1/1 Running 4 4d9h
sh 1/1 Running 6 4d11h
test-76846b5956-gftn9 1/1 Running 2 4d11h
test-76846b5956-r7s9k 1/1 Running 2 4d11h
test-76846b5956-trpbn 1/1 Running 2 4d11h
test2-78c4694588-87b9r 1/1 Running 5 4d12h
web-0 1/1 Running 4 4d11h
web-1 1/1 Running 4 4d11h
web-2 1/1 Running 4 4d11h
web-96d5df5c8-vc9kf 1/1 Running 3 3d
- Service指定target-port端口是否正常？
# 查看service中的target-port是否是指定的端口
[root@k8s-master ~]# kubectl edit svc nginx
Edit cancelled, no changes made.
# Please edit the object below. Lines beginning with a '#' will be ignored,
# and an empty file will abort the edit. If an error occurs while saving this file will be
# reopened with the relevant failures.
#
apiVersion: v1
kind: Service
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"labels":{"app":"nginx"},"name":"nginx","namespace":"default"},"spec":{"clusterIP":"None","ports":[{"name":"web","port":80}],"selector":{"app":"nginx"}}}
creationTimestamp: "2021-12-21T02:56:44Z"
labels:
app: nginx
name: nginx
namespace: default
resourceVersion: "2334070"
selfLink: /api/v1/namespaces/default/services/nginx
uid: 5f07839a-04e4-4214-bbbe-d69357de10d4
spec:
clusterIP: None
ports:
- name: web
port: 80
protocol: TCP
targetPort: 80
selector:
app: nginx
sessionAffinity: None
type: ClusterIP
status:
loadBalancer: {}
- Pod正常工作吗？
# 查看Pod的IP地址，然后用curl命令去看是否返回正常的内容
[root@k8s-master ~]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
client1 1/1 Running 5 3d 10.244.36.99 k8s-node1 <none> <none>
client2 1/1 Running 5 3d 10.244.36.92 k8s-node1 <none> <none>
configmap-demo-pod 1/1 Running 4 4d10h 10.244.36.101 k8s-node1 <none> <none>
my-pod2 1/1 Running 11 4d17h 10.244.169.130 k8s-node2 <none> <none>
nfs-client-provisioner-58d675cd5-dx7n4 1/1 Running 7 4d12h 10.244.36.116 k8s-node1 <none> <none>
pod-taint 1/1 Running 9 10d 10.244.169.132 k8s-node2 <none> <none>
secret-demo-pod 1/1 Running 4 4d9h 10.244.36.118 k8s-node1 <none> <none>
sh 1/1 Running 6 4d11h 10.244.36.114 k8s-node1 <none> <none>
test-76846b5956-gftn9 1/1 Running 2 4d11h 10.244.36.111 k8s-node1 <none> <none>
test-76846b5956-r7s9k 1/1 Running 2 4d11h 10.244.36.100 k8s-node1 <none> <none>
test-76846b5956-trpbn 1/1 Running 2 4d11h 10.244.169.185 k8s-node2 <none> <none>
test2-78c4694588-87b9r 1/1 Running 5 4d13h 10.244.36.123 k8s-node1 <none> <none>
web-0 1/1 Running 4 4d12h 10.244.36.122 k8s-node1 <none> <none>
web-1 1/1 Running 4 4d12h 10.244.36.119 k8s-node1 <none> <none>
web-2 1/1 Running 4 4d12h 10.244.36.98 k8s-node1 <none> <none>
web-96d5df5c8-vc9kf 1/1 Running 3 3d 10.244.169.158 k8s-node2 <none> <none>
[root@k8s-master ~]# curl 10.244.169.158
- Service是否通过DNS工作？
# 查看coredns组件是否正常工作
[root@k8s-master ~]# kubectl get pods -n kube-system
NAME READY STATUS RESTARTS AGE
calico-kube-controllers-97769f7c7-c4cg5 1/1 Running 3 31h
calico-node-4pwdc 1/1 Running 16 33d
calico-node-9r6zd 1/1 Running 16 33d
calico-node-vqzdj 1/1 Running 17 33d
client1 1/1 Running 5 3d
coredns-6d56c8448f-gcgrh 1/1 Running 16 33d
coredns-6d56c8448f-mdl7c 1/1 Running 2 31h
etcd-k8s-master 1/1 Running 3 31h
filebeat-5pwh7 1/1 Running 11 10d
filebeat-pt848 1/1 Running 11 10d
kube-apiserver-k8s-master 1/1 Running 3 31h
kube-controller-manager-k8s-master 1/1 Running 3 31h
kube-proxy-87lbj 1/1 Running 3 31h
kube-proxy-mcdnv 1/1 Running 2 31h
kube-proxy-mchc9 1/1 Running 2 31h
kube-scheduler-k8s-master 1/1 Running 3 31h
metrics-server-84f9866fdf-rz676 1/1 Running 15 4d16h
- kube-proxy正常工作吗？
# kubeadm部署的查看kube-proxy看是否工作正常
[root@k8s-master ~]# kubectl get pods -n kube-system
NAME READY STATUS RESTARTS AGE
calico-kube-controllers-97769f7c7-c4cg5 1/1 Running 3 31h
calico-node-4pwdc 1/1 Running 16 33d
calico-node-9r6zd 1/1 Running 16 33d
calico-node-vqzdj 1/1 Running 17 33d
client1 1/1 Running 5 3d
coredns-6d56c8448f-gcgrh 1/1 Running 16 33d
coredns-6d56c8448f-mdl7c 1/1 Running 2 31h
etcd-k8s-master 1/1 Running 3 31h
filebeat-5pwh7 1/1 Running 11 10d
filebeat-pt848 1/1 Running 11 10d
kube-apiserver-k8s-master 1/1 Running 3 31h
kube-controller-manager-k8s-master 1/1 Running 3 31h
kube-proxy-87lbj 1/1 Running 3 31h
kube-proxy-mcdnv 1/1 Running 2 31h
kube-proxy-mchc9 1/1 Running 2 31h
kube-scheduler-k8s-master 1/1 Running 3 31h
metrics-server-84f9866fdf-rz676 1/1 Running 15 4d16h
# 二进制查看systemd
- kube-proxy是否正常写iptables规则？
# 使用iptables-save |grep service名称来查看，对应规则是否被创建
[root@k8s-master ~]# iptables -L -n
Chain INPUT (policy ACCEPT)
target prot opt source destination
cali-INPUT all -- 0.0.0.0/0 0.0.0.0/0 /* cali:Cz_u1IQiXIMmKD4c */
KUBE-FIREWALL all -- 0.0.0.0/0 0.0.0.0/0
Chain FORWARD (policy DROP)
target prot opt source destination
cali-FORWARD all -- 0.0.0.0/0 0.0.0.0/0 /* cali:wUHhoiAYhphO9Mso */
KUBE-FORWARD all -- 0.0.0.0/0 0.0.0.0/0 /* kubernetes forwarding rules */
DOCKER-USER all -- 0.0.0.0/0 0.0.0.0/0
DOCKER-ISOLATION-STAGE-1 all -- 0.0.0.0/0 0.0.0.0/0
ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 ctstate RELATED,ESTABLISHED
DOCKER all -- 0.0.0.0/0 0.0.0.0/0
ACCEPT all -- 0.0.0.0/0 0.0.0.0/0
ACCEPT all -- 0.0.0.0/0 0.0.0.0/0
Chain OUTPUT (policy ACCEPT)
target prot opt source destination
cali-OUTPUT all -- 0.0.0.0/0 0.0.0.0/0 /* cali:tVnHkvAo15HuiPy0 */
KUBE-FIREWALL all -- 0.0.0.0/0 0.0.0.0/0
Chain DOCKER (1 references)
target prot opt source destination
Chain DOCKER-ISOLATION-STAGE-1 (1 references)
target prot opt source destination
DOCKER-ISOLATION-STAGE-2 all -- 0.0.0.0/0 0.0.0.0/0
RETURN all -- 0.0.0.0/0 0.0.0.0/0
Chain DOCKER-ISOLATION-STAGE-2 (1 references)
target prot opt source destination
DROP all -- 0.0.0.0/0 0.0.0.0/0
RETURN all -- 0.0.0.0/0 0.0.0.0/0
Chain DOCKER-USER (1 references)
target prot opt source destination
RETURN all -- 0.0.0.0/0 0.0.0.0/0
Chain KUBE-FIREWALL (2 references)
target prot opt source destination
DROP all -- 0.0.0.0/0 0.0.0.0/0 /* kubernetes firewall for dropping marked packets */ mark match 0x8000/0x8000
DROP all -- !127.0.0.0/8 127.0.0.0/8 /* block incoming localnet connections */ ! ctstate RELATED,ESTABLISHED,DNAT
Chain KUBE-FORWARD (1 references)
target prot opt source destination
ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 /* kubernetes forwarding rules */ mark match 0x4000/0x4000
ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 /* kubernetes forwarding conntrack pod source rule */ ctstate RELATED,ESTABLISHED
ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 /* kubernetes forwarding conntrack pod destination rule */ ctstate RELATED,ESTABLISHED
Chain KUBE-KUBELET-CANARY (0 references)
target prot opt source destination
Chain cali-FORWARD (1 references)
target prot opt source destination
MARK all -- 0.0.0.0/0 0.0.0.0/0 /* cali:vjrMJCRpqwy5oRoX */ MARK and 0xfff1ffff
cali-from-hep-forward all -- 0.0.0.0/0 0.0.0.0/0 /* cali:A_sPAO0mcxbT9mOV */ mark match 0x0/0x10000
cali-from-wl-dispatch all -- 0.0.0.0/0 0.0.0.0/0 /* cali:8ZoYfO5HKXWbB3pk */
cali-to-wl-dispatch all -- 0.0.0.0/0 0.0.0.0/0 /* cali:jdEuaPBe14V2hutn */
cali-to-hep-forward all -- 0.0.0.0/0 0.0.0.0/0 /* cali:12bc6HljsMKsmfr- */
ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 /* cali:MH9kMp5aNICL-Olv */ /* Policy explicitly accepted packet. */ mark match 0x10000/0x10000
Chain cali-INPUT (1 references)
target prot opt source destination
ACCEPT 4 -- 0.0.0.0/0 0.0.0.0/0 /* cali:PajejrV4aFdkZojI */ /* Allow IPIP packets from Calico hosts */ match-set cali40all-hosts-net src ADDRTYPE match dst-type LOCAL
DROP 4 -- 0.0.0.0/0 0.0.0.0/0 /* cali:_wjq-Yrma8Ly1Svo */ /* Drop IPIP packets from non-Calico hosts */
MARK all -- 0.0.0.0/0 0.0.0.0/0 /* cali:ss8lEMQsXi-s6qYT */ MARK and 0xfffff
cali-forward-check all -- 0.0.0.0/0 0.0.0.0/0 /* cali:PgIW-V0nEjwPhF_8 */
RETURN all -- 0.0.0.0/0 0.0.0.0/0 /* cali:QMJlDwlS0OjHyfMN */ mark match ! 0x0/0xfff00000
cali-wl-to-host all -- 0.0.0.0/0 0.0.0.0/0 [goto] /* cali:nDRe73txrna-aZjG */
ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 /* cali:iX2AYvqGXaVqwkro */ mark match 0x10000/0x10000
MARK all -- 0.0.0.0/0 0.0.0.0/0 /* cali:bhpnxD5IRtBP8KW0 */ MARK and 0xfff0ffff
cali-from-host-endpoint all -- 0.0.0.0/0 0.0.0.0/0 /* cali:H5_bccAbHV0sooVy */
ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 /* cali:inBL01YlfurT0dbI */ /* Host endpoint policy accepted packet. */ mark match 0x10000/0x10000
Chain cali-OUTPUT (1 references)
target prot opt source destination
ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 /* cali:Mq1_rAdXXH3YkrzW */ mark match 0x10000/0x10000
cali-forward-endpoint-mark all -- 0.0.0.0/0 0.0.0.0/0 [goto] /* cali:5Z67OUUpTOM7Xa1a */ mark match ! 0x0/0xfff00000
RETURN all -- 0.0.0.0/0 0.0.0.0/0 /* cali:M2Wf0OehNdig8MHR */
ACCEPT 4 -- 0.0.0.0/0 0.0.0.0/0 /* cali:AJBkLho_0Qd8LNr3 */ /* Allow IPIP packets to other Calico hosts */ match-set cali40all-hosts-net dst ADDRTYPE match src-type LOCAL
MARK all -- 0.0.0.0/0 0.0.0.0/0 /* cali:iz2RWXlXJDUfsLpe */ MARK and 0xfff0ffff
cali-to-host-endpoint all -- 0.0.0.0/0 0.0.0.0/0 /* cali:hXojbnLundZDgZyw */
ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 /* cali:wankpMDC2Cy1KfBv */ /* Host endpoint policy accepted packet. */ mark match 0x10000/0x10000
Chain cali-forward-check (1 references)
target prot opt source destination
RETURN all -- 0.0.0.0/0 0.0.0.0/0 /* cali:Pbldlb4FaULvpdD8 */ ctstate RELATED,ESTABLISHED
cali-set-endpoint-mark tcp -- 0.0.0.0/0 0.0.0.0/0 [goto] /* cali:ZD-6UxuUtGW-xtzg */ /* To kubernetes NodePort service */ multiport dports 30000:32767 match-set cali40this-host dst
cali-set-endpoint-mark udp -- 0.0.0.0/0 0.0.0.0/0 [goto] /* cali:CbPfUajQ2bFVnDq4 */ /* To kubernetes NodePort service */ multiport dports 30000:32767 match-set cali40this-host dst
cali-set-endpoint-mark all -- 0.0.0.0/0 0.0.0.0/0 /* cali:jmhU0ODogX-Zfe5g */ /* To kubernetes service */ ! match-set cali40this-host dst
Chain cali-forward-endpoint-mark (1 references)
target prot opt source destination
cali-from-endpoint-mark all -- 0.0.0.0/0 0.0.0.0/0 /* cali:O0SmFDrnm7KggWqW */ mark match ! 0x100000/0xfff00000
cali-to-wl-dispatch all -- 0.0.0.0/0 0.0.0.0/0 /* cali:aFl0WFKRxDqj8oA6 */
cali-to-hep-forward all -- 0.0.0.0/0 0.0.0.0/0 /* cali:AZKVrO3i_8cLai5f */
MARK all -- 0.0.0.0/0 0.0.0.0/0 /* cali:96HaP1sFtb-NYoYA */ MARK and 0xfffff
ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 /* cali:VxO6hyNWz62YEtul */ /* Policy explicitly accepted packet. */ mark match 0x10000/0x10000
Chain cali-from-endpoint-mark (1 references)
target prot opt source destination
DROP all -- 0.0.0.0/0 0.0.0.0/0 /* cali:9dpftzl-pNycbr37 */ /* Unknown interface */
Chain cali-from-hep-forward (1 references)
target prot opt source destination
Chain cali-from-host-endpoint (1 references)
target prot opt source destination
Chain cali-from-wl-dispatch (2 references)
target prot opt source destination
DROP all -- 0.0.0.0/0 0.0.0.0/0 /* cali:zTj6P0TIgYvgz-md */ /* Unknown interface */
Chain cali-set-endpoint-mark (3 references)
target prot opt source destination
DROP all -- 0.0.0.0/0 0.0.0.0/0 /* cali:MN61lcxFj1yWuYBo */ /* Unknown endpoint */
MARK all -- 0.0.0.0/0 0.0.0.0/0 /* cali:nKOjq8N2yzfmS3jk */ /* Non-Cali endpoint mark */ MARK xset 0x100000/0xfff00000
Chain cali-to-hep-forward (2 references)
target prot opt source destination
Chain cali-to-host-endpoint (1 references)
target prot opt source destination
Chain cali-to-wl-dispatch (2 references)
target prot opt source destination
DROP all -- 0.0.0.0/0 0.0.0.0/0 /* cali:7KNphB1nNHw80nIO */ /* Unknown interface */
Chain cali-wl-to-host (1 references)
target prot opt source destination
cali-from-wl-dispatch all -- 0.0.0.0/0 0.0.0.0/0 /* cali:Ee9Sbo10IpVujdIY */
ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 /* cali:nSZbcOoG1xPONxb8 */ /* Configured DefaultEndpointToHostAction */
[root@k8s-master ~]#
[root@k8s-master ~]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 33d
my-dep NodePort 10.111.199.51 <none> 80:31734/TCP 30d
my-service NodePort 10.100.228.0 <none> 80:32433/TCP 24d
nginx ClusterIP None <none> 80/TCP 4d12h
[root@k8s-master ~]# iptables-save |grep nginx
[root@k8s-master ~]# iptables-save |grep my-dep
[root@k8s-master ~]# iptables-save |grep my-service
[root@k8s-master ~]# iptables-save |grep kubernetes
-A FORWARD -m comment --comment "kubernetes forwarding rules" -j KUBE-FORWARD
-A KUBE-FIREWALL -m comment --comment "kubernetes firewall for dropping marked packets" -m mark --mark 0x8000/0x8000 -j DROP
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A cali-forward-check -p tcp -m comment --comment "cali:ZD-6UxuUtGW-xtzg" -m comment --comment "To kubernetes NodePort service" -m multiport --dports 30000:32767 -m set --match-set cali40this-host dst -g cali-set-endpoint-mark
-A cali-forward-check -p udp -m comment --comment "cali:CbPfUajQ2bFVnDq4" -m comment --comment "To kubernetes NodePort service" -m multiport --dports 30000:32767 -m set --match-set cali40this-host dst -g cali-set-endpoint-mark
-A cali-forward-check -m comment --comment "cali:jmhU0ODogX-Zfe5g" -m comment --comment "To kubernetes service" -m set ! --match-set cali40this-host dst -j cali-set-endpoint-mark
-A PREROUTING -m comment --comment "kubernetes service portals" -j KUBE-SERVICES
-A OUTPUT -m comment --comment "kubernetes service portals" -j KUBE-SERVICES
-A POSTROUTING -m comment --comment "kubernetes postrouting rules" -j KUBE-POSTROUTING
-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE
- cni网络插件是否正常工作？
扁平化、转发、实现不同节点间的通信
还是去查看IP，然后去curl那个IP地址，看是否返回正常值
小结：
- 关闭kube-apiserver，执行不了kubectl命令
- 关闭kube-controller-manager，创建deployment，但是deployment不会去创建具体的pod
- 关闭kube-scheduler，Pod无法调度
- 关闭kubelet，Node显示notready
- 关闭kube-proxy，网络不通