Kubernetes 用kubeadm安装kubernetes_v1.18.x
安装参考
测试yaml 语法是否正确
kubectl apply -f nginx-ingress-service.yaml --dry-run
######################################################################################################################
kubeadm安装kubernetes_v1.18.x (2020-05-13)
安装后的软件版本为
Kubernetes v1.18.x
calico 3.13.1
nginx-ingress 1.5.5
Docker 19.03.8
#选择2台服务器：
10.100.100.227 demo-master-a-1 master 123456789
10.100.100.229 demo-worker-a-1 slave 123456789
#时间同步
/usr/sbin/ntpdate 192.168.200.2
#查看防火墙的规则
firewall-cmd --list-all
systemctl stop firewalld.service
systemctl stop iptables
iptables -F
iptables -X
iptables-save
#关闭系统selinux
sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
#在slave角色worker节点设置主机名
hostnamectl set-hostname demo-worker-a-1
#添加解析
cat /etc/hosts
10.100.100.227 demo-master-a-1
10.100.100.229 demo-worker-a-1
#设置master主机名
hostnamectl set-hostname demo-master-a-1
#关闭系统selinux
#添加解析
cat /etc/hosts
127.0.0.1 demo-master-a-1
10.100.100.227 demo-master-a-1
10.100.100.229 demo-worker-a-1
# 在 master 节点和 worker 节点都要执行
#安装基础安装包
export REGISTRY_MIRROR=https://registry.cn-hangzhou.aliyuncs.com
curl -sSL https://kuboard.cn/install-script/v1.18.x/install_kubelet.sh | sh -s 1.18.2
# 以下操作，在 master节点下执行
# 替换 10.100.100.227 为 master 节点实际 IP（请使用内网 IP）
# export 命令只在当前 shell 会话中有效，开启新的 shell 窗口后，如果要继续安装过程，请重新执行此处的 export 命令
# 替换 apiserver.demo 为 您想要的 dnsName
# Kubernetes 容器组所在的网段，该网段安装完成后，由 kubernetes 创建，事先并不存在于您的物理网络中
# 如果添加hosts 可以不用执行
# 在 master 节点和 worker 节点都要执行
export MASTER_IP=10.100.100.227
export APISERVER_NAME=apiserver.demo
export POD_SUBNET=10.209.0.1/24
echo "${MASTER_IP} ${APISERVER_NAME}" >> /etc/hosts
curl -sSL https://kuboard.cn/install-script/v1.18.x/init_master.sh | sh -s 1.18.2
# 只在 master 节点执行
# 执行如下命令，等待 3-10 分钟，直到所有的容器组处于 Running 状态
# 查看 master 节点初始化结果
watch kubectl get pod -n kube-system -o wide
kubectl get nodes -o wide
#主节点执行的这个命令，获取的token,
kubeadm token create --print-join-command
[root@demo-master-a-1 ~]# kubeadm token create --print-join-command
W0513 20:34:06.138972 9139 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]
kubeadm join apiserver.demo:6443 --token 6zdbqb.klxlzuq1nftjcrp2 --discovery-token-ca-cert-hash sha256:d1ae83ed3a6ab7fe976c49c0d8081ff0b102a816042f6d43b19b3eb7122585b0
[root@demo-master-a-1 ~]#
# 只在 worker 节点执行
# 替换 x.x.x.x 为 master 节点的内网 IP
export MASTER_IP=x.x.x.x
# 替换 apiserver.demo 为初始化 master 节点时所使用的 APISERVER_NAME
export APISERVER_NAME=apiserver.demo
echo "${MASTER_IP} ${APISERVER_NAME}" >> /etc/hosts
# 替换为 master 节点上 kubeadm token create 命令的输出
kubeadm join apiserver.demo:6443 --token k5boem.iub5bthvw1756mf6 --discovery-token-ca-cert-hash sha256:6d5259b468b8f8e056e8ce3e76c1e31b5c5dc4ef744edac3f9c9d598397a6f8c
在worker节点执行master 主节点输出的信息
# kubeadm token create 命令的输出
# 以下操作，在 master节点下执行
# 替换 10.100.100.227 为 master 节点实际 IP（请使用内网 IP）
# export 命令只在当前 shell 会话中有效，开启新的 shell 窗口后，如果要继续安装过程，请重新执行此处的 export 命令
# 替换 apiserver.demo 为 您想要的 dnsName
# Kubernetes 容器组所在的网段，该网段安装完成后，由 kubernetes 创建，事先并不存在于您的物理网络中
#如果添加hosts 可以不用执行
# 在 master 节点和 worker 节点都要执行
export MASTER_IP=10.100.100.227
export APISERVER_NAME=apiserver.demo
export POD_SUBNET=10.209.0.1/24
echo "${MASTER_IP} ${APISERVER_NAME}" >> /etc/hosts
kubeadm join apiserver.demo:6443 --token 5iif0a.wgbde5vcu1x2v7j7 --discovery-token-ca-cert-hash sha256:e79fe1202e8f5df226085db4b183674c8409e76ed5442e987934a3e2fc2b2374
[root@demo-master-a-1 kubernetes]# scp admin.conf root@10.100.100.229:/etc/kubernetes/
[root@demo-worker-a-1 kubernetes]# cat /etc/profile
export KUBECONFIG=/etc/kubernetes/admin.conf
source /etc/profile
在master 安装 Kubernetes Dashboard
[root@demo-master-a-1 ~]# kubectl apply -f https://kuboard.cn/install-script/k8s-dashboard/v2.0.0-beta5.yaml
namespace/kubernetes-dashboard created
serviceaccount/kubernetes-dashboard created
service/kubernetes-dashboard created
secret/kubernetes-dashboard-certs created
secret/kubernetes-dashboard-csrf created
secret/kubernetes-dashboard-key-holder created
configmap/kubernetes-dashboard-settings created
role.rbac.authorization.k8s.io/kubernetes-dashboard created
clusterrole.rbac.authorization.k8s.io/kubernetes-dashboard created
rolebinding.rbac.authorization.k8s.io/kubernetes-dashboard created
clusterrolebinding.rbac.authorization.k8s.io/kubernetes-dashboard created
deployment.apps/kubernetes-dashboard created
service/dashboard-metrics-scraper created
deployment.apps/dashboard-metrics-scraper created
root@demo-master-a-1 ~]# kubectl apply -f https://kuboard.cn/install-script/k8s-dashboard/auth.yaml
serviceaccount/admin-user created
clusterrolebinding.rbac.authorization.k8s.io/admin-user created
[root@demo-master-a-1 ~]# kubectl -n kubernetes-dashboard describe secret $(kubectl -n kubernetes-dashboard get secret | grep admin-user | awk '{print $1}')
Name: admin-user-token-6m7kp
Namespace: kubernetes-dashboard
Labels: <none>
Annotations: kubernetes.io/service-account.name: admin-user
kubernetes.io/service-account.uid: 21cb53d8-c0be-41c8-abb2-a2dabf142156
Type: kubernetes.io/service-account-token
Data
====
ca.crt: 1025 bytes
namespace: 20 bytes
token: eyJhbGciOiJSUzI1NiIsImtpZCI6InZhc1I1NHRibkFQXzRxMTVvcWNOTWdaUkFYd2tsLWRuYWV5YUtNbEhuVDQifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi11c2VyLXRva2VuLTZtN2twIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImFkbWluLXVzZXIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiIyMWNiNTNkOC1jMGJlLTQxYzgtYWJiMi1hMmRhYmYxNDIxNTYiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZXJuZXRlcy1kYXNoYm9hcmQ6YWRtaW4tdXNlciJ9.TdVK8AYjzfpObvTSq3dj3xQ91C6Dk_rXeuXZ5qekrvY61fpj37EZf28jHWW44SHBx3JgGXVUHTiyH8nGyp9ho7E2bDaX4RfE3GvdjfZdNdQmlMyts7iUPpr2zwdSg8vvMMGmaW_mDaCn0dcHFEmxnH1_lMyD7rGtS8qxSRgxsRPu-h51Z0oWRd1D-UGjHTdQjfO3np4DQ8OON-dCmPGYBP-etJ6fxYgpv3QmmJ7mgRH52tIsi45hfG9nv1WYAklUL8iC3kv0Q6ApZsNuqMp6cQ1jD0RFxCcIsGiry3QzaKLPjiE_LjuPB0hubbTEC7hBAU8Rdw2Aav6VnRew49rZNQ
[root@demo-master-a-1 ~]#
将127.0.0.1 改成用本机IP访问：
# 推荐使用下面这种方式
$ wget https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta5/aio/deploy/recommended.yaml
$ vi recommended.yaml
# 修改Service为NodePort类型
#添加这个type: NodePort
cat recommended.yaml
apiVersion: v1
kind: Namespace
metadata:
name: kubernetes-dashboard
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
---
kind: Service
apiVersion: v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
spec:
ports:
- port: 443
targetPort: 8443
type: NodePort
selector:
k8s-app: kubernetes-dashboard
---
apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-certs
namespace: kubernetes-dashboard
type: Opaque
---
apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-csrf
namespace: kubernetes-dashboard
type: Opaque
data:
csrf: ""
---
apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-key-holder
namespace: kubernetes-dashboard
type: Opaque
---
kind: ConfigMap
apiVersion: v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-settings
namespace: kubernetes-dashboard
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
rules:
# Allow Dashboard to get, update and delete Dashboard exclusive secrets.
- apiGroups: [""]
resources: ["secrets"]
resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]
verbs: ["get", "update", "delete"]
# Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
- apiGroups: [""]
resources: ["configmaps"]
resourceNames: ["kubernetes-dashboard-settings"]
verbs: ["get", "update"]
# Allow Dashboard to get metrics.
- apiGroups: [""]
resources: ["services"]
resourceNames: ["heapster", "dashboard-metrics-scraper"]
verbs: ["proxy"]
- apiGroups: [""]
resources: ["services/proxy"]
resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]
verbs: ["get"]
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
rules:
# Allow Metrics Scraper to get metrics from the Metrics server
- apiGroups: ["metrics.k8s.io"]
resources: ["pods", "nodes"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: kubernetes-dashboard
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard
namespace: kubernetes-dashboard
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kubernetes-dashboard
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: kubernetes-dashboard
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard
namespace: kubernetes-dashboard
---
kind: Deployment
apiVersion: apps/v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: kubernetes-dashboard
template:
metadata:
labels:
k8s-app: kubernetes-dashboard
spec:
containers:
- name: kubernetes-dashboard
image: kubernetesui/dashboard:v2.0.0-beta5
imagePullPolicy: Always
ports:
- containerPort: 8443
protocol: TCP
args:
- --auto-generate-certificates
- --namespace=kubernetes-dashboard
# Uncomment the following line to manually specify Kubernetes API server Host
# If not specified, Dashboard will attempt to auto discover the API server and connect
# to it. Uncomment only if the default does not work.
# - --apiserver-host=http://my-address:port
volumeMounts:
- name: kubernetes-dashboard-certs
mountPath: /certs
# Create on-disk volume to store exec logs
- mountPath: /tmp
name: tmp-volume
livenessProbe:
httpGet:
scheme: HTTPS
path: /
port: 8443
initialDelaySeconds: 30
timeoutSeconds: 30
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsUser: 1001
runAsGroup: 2001
volumes:
- name: kubernetes-dashboard-certs
secret:
secretName: kubernetes-dashboard-certs
- name: tmp-volume
emptyDir: {}
serviceAccountName: kubernetes-dashboard
nodeSelector:
"beta.kubernetes.io/os": linux
# Comment the following tolerations if Dashboard must not be deployed on master
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
---
kind: Service
apiVersion: v1
metadata:
labels:
k8s-app: dashboard-metrics-scraper
name: dashboard-metrics-scraper
namespace: kubernetes-dashboard
spec:
ports:
- port: 8000
targetPort: 8000
selector:
k8s-app: dashboard-metrics-scraper
---
kind: Deployment
apiVersion: apps/v1
metadata:
labels:
k8s-app: dashboard-metrics-scraper
name: dashboard-metrics-scraper
namespace: kubernetes-dashboard
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: dashboard-metrics-scraper
template:
metadata:
labels:
k8s-app: dashboard-metrics-scraper
annotations:
seccomp.security.alpha.kubernetes.io/pod: 'runtime/default'
spec:
containers:
- name: dashboard-metrics-scraper
image: kubernetesui/metrics-scraper:v1.0.1
ports:
- containerPort: 8000
protocol: TCP
livenessProbe:
httpGet:
scheme: HTTP
path: /
port: 8000
initialDelaySeconds: 30
timeoutSeconds: 30
volumeMounts:
- mountPath: /tmp
name: tmp-volume
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsUser: 1001
runAsGroup: 2001
serviceAccountName: kubernetes-dashboard
nodeSelector:
"beta.kubernetes.io/os": linux
# Comment the following tolerations if Dashboard must not be deployed on master
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
volumes:
- name: tmp-volume
emptyDir: {}
[root@demo-master-a-1 home]#
[root@demo-master-a-1 ~]# kubectl apply -f recommended.yaml --dry-run
W0513 21:27:21.844061 84298 helpers.go:535] --dry-run is deprecated and can be replaced with --dry-run=client.
namespace/kubernetes-dashboard configured (dry run)
serviceaccount/kubernetes-dashboard configured (dry run)
service/kubernetes-dashboard configured (dry run)
secret/kubernetes-dashboard-certs configured (dry run)
secret/kubernetes-dashboard-csrf configured (dry run)
secret/kubernetes-dashboard-key-holder configured (dry run)
configmap/kubernetes-dashboard-settings configured (dry run)
role.rbac.authorization.k8s.io/kubernetes-dashboard configured (dry run)
clusterrole.rbac.authorization.k8s.io/kubernetes-dashboard configured (dry run)
rolebinding.rbac.authorization.k8s.io/kubernetes-dashboard configured (dry run)
clusterrolebinding.rbac.authorization.k8s.io/kubernetes-dashboard configured (dry run)
deployment.apps/kubernetes-dashboard configured (dry run)
service/dashboard-metrics-scraper configured (dry run)
deployment.apps/dashboard-metrics-scraper configured (dry run)
[root@demo-master-a-1 ~]# kubectl apply -f recommended.yaml
namespace/kubernetes-dashboard unchanged
serviceaccount/kubernetes-dashboard unchanged
service/kubernetes-dashboard configured
secret/kubernetes-dashboard-certs unchanged
secret/kubernetes-dashboard-csrf configured
Warning: kubectl apply should be used on resource created by either kubectl create --save-config or kubectl apply
secret/kubernetes-dashboard-key-holder configured
configmap/kubernetes-dashboard-settings unchanged
role.rbac.authorization.k8s.io/kubernetes-dashboard unchanged
clusterrole.rbac.authorization.k8s.io/kubernetes-dashboard unchanged
rolebinding.rbac.authorization.k8s.io/kubernetes-dashboard unchanged
clusterrolebinding.rbac.authorization.k8s.io/kubernetes-dashboard unchanged
deployment.apps/kubernetes-dashboard unchanged
service/dashboard-metrics-scraper unchanged
deployment.apps/dashboard-metrics-scraper unchanged
[root@demo-master-a-1 ~]#
查看dashboard页面
[root@demo-master-a-1 ~]# kubectl get pods -n kubernetes-dashboard -l k8s-app=kubernetes-dashboard
NAME READY STATUS RESTARTS AGE
kubernetes-dashboard-7d9ddf9f8f-qg2sj 1/1 Running 0 29m
查看dashboard登陆端口
[root@demo-master-a-1 ~]# kubectl get svc -n kubernetes-dashboard
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
dashboard-metrics-scraper ClusterIP 10.96.210.52 <none> 8000/TCP 29m
kubernetes-dashboard NodePort 10.96.229.35 <none> 443:31831/TCP 29m
[root@demo-master-a-1 ~]#
登陆验证
https://10.100.100.227:31831
[root@demo-master-a-1 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
demo-master-a-1 Ready master 100m v1.18.2
demo-worker-a-1 Ready <none> 93m v1.18.2
https://github.com/cnych/kubernetes-learning/blob/master/docs/index.md
视频
https://www.qikqiak.com/k8s-book/docs/67.Upgrade%E9%9B%86%E7%BE%A4.html
安装
https://www.qikqiak.com/k8strain/k8s-basic/install/
https://github.com/cnych/kubernetes-learning/blob/master/docs/40.ingress.md
####################################################################################################################################################################################################################
日常命令总结
查看更详细的信息
kubectl get pods -n kube-system -o wide
查看运行的docker
kubectl get pods -o wide
查看组件信息
kubectl get pods -n kube-system
NAME READY STATUS RESTARTS AGE
coredns-667f964f9b-fbjcq 1/1 Running 0 3d21h
coredns-667f964f9b-jgxs4 1/1 Running 0 3d21h
etcd-demo-master-a-1 1/1 Running 0 3d21h
kube-apiserver-demo-master-a-1 1/1 Running 0 3d21h
kube-controller-manager-demo-master-a-1 1/1 Running 2 3d21h
kube-flannel-ds-amd64-8kckc 1/1 Running 0 3d21h
kube-flannel-ds-amd64-x7l59 1/1 Running 0 3d21h
kube-proxy-7bhjl 1/1 Running 0 3d21h
kube-proxy-8n6q8 1/1 Running 0 3d21h
kube-scheduler-demo-master-a-1 1/1 Running 1 3d21h
查询信息
[root@demo-master-a-1 ~]# kubectl get svc -n kube-system
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kube-dns ClusterIP 10.96.0.10 <none> 53/UDP,53/TCP,9153/TCP 3d21h
创建负载服务及映射端口
[root@demo-master-a-1 ~]# kubectl create service clusterip ngx-dep --tcp=4321:8765
service/ngx-dep created
查看服务
kubectl get services
查看pods的数量
kubectl get pods
扩容的方法
[root@demo-master-a-1 ~]# kubectl scale --replicas=3 deployment nginx-deploy
deployment.apps/nginx-deploy scaled
查询pods信息
[root@demo-master-a-1 ~]# kubectl get pods
NAME READY STATUS RESTARTS AGE
nginx-deploy-54f57cf6bf-27hkg 1/1 Running 0 8s
nginx-deploy-54f57cf6bf-4rsb9 1/1 Running 0 3d19h
nginx-deploy-54f57cf6bf-9mnxg 1/1 Running 0 3d19h
[root@demo-master-a-1 ~]#
缩容的方法
[root@demo-master-a-1 ~]# kubectl scale --replicas=2 deployment nginx-deploy
deployment.apps/nginx-deploy scaled
查询pods
[root@demo-master-a-1 ~]# kubectl get pods
NAME READY STATUS RESTARTS AGE
nginx-deploy-54f57cf6bf-27hkg 0/1 Terminating 0 5m29s
nginx-deploy-54f57cf6bf-4rsb9 1/1 Running 0 3d19h
nginx-deploy-54f57cf6bf-9mnxg 1/1 Running 0 3d19h
查询pods
[root@demo-master-a-1 ~]# kubectl get pods
NAME READY STATUS RESTARTS AGE
nginx-deploy-54f57cf6bf-4rsb9 1/1 Running 0 3d19h
nginx-deploy-54f57cf6bf-9mnxg 1/1 Running 0 3d19h
[root@demo-master-a-1 ~]#
通过这个方法映射docker 容器的端口， 7654是容器的端口，--target-port 指宿主主机的端口
kubectl expose deployment nginx-deploy --port=7654 --target-port=80 --type=NodePort
kubectl get all
NAME READY STATUS RESTARTS AGE
pod/nginx-deploy-54f57cf6bf-4rsb9 1/1 Running 0 6d11h
pod/nginx-deploy-54f57cf6bf-5j6v6 1/1 Running 0 2d15h
pod/nginx-deploy-54f57cf6bf-n5p59 1/1 Running 0 2d15h
pod/nginx-deploy-54f57cf6bf-t96r4 1/1 Running 0 2d15h
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 6d13h
service/nginx-deploy NodePort 10.105.127.56 <none> 7654:30631/TCP 3m11s
service/nginx-deploy-54f57cf6bf-4rsb9 ClusterIP 10.98.4.171 <none> 4322/TCP 2d16h
service/ngx-dep ClusterIP 10.104.10.249 <none> 4321/TCP 2d16h
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/nginx-deploy 4/4 4 4 6d11h
NAME DESIRED CURRENT READY AGE
replicaset.apps/nginx-deploy-54f57cf6bf 4 4 4 6d11h
[root@demo-master-a-1 ~]#
[root@demo-master-a-1 ~]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 6d15h
nginx-deploy NodePort 10.105.127.56 <none> 7654:30631/TCP 71m
nginx-deploy-54f57cf6bf-4rsb9 ClusterIP 10.98.4.171 <none> 4322/TCP 2d17h
ngx-dep ClusterIP 10.104.10.249 <none> 4321/TCP 2d17h