1.
基础概念
statefulset,可以翻译成有状态的设定.
和deployment的对比
deployment部署创建的pod是无状态的,重新调度pod,pod名字hostname,启动pod顺序删除pod顺序都是随机的.deployment使用的是共享存储,所有pod共用一个存储卷.
statefulset部署创建的pod是有状态的,重新调度pod,pod名字hostname保持固定不变,启动pod顺序删除pod顺序都可以根据定义的顺序有序执行操作,有序的动态更新,statefulset使用的存储,不是共用一个存储卷,一个pod对应一个存储卷(pv).pod重新创建调度挂载的存储卷保持不变.
2.
statefulset实现用到的技术概念
headless service
headless service不需要配置clusterIP,对应的每一个pod都会有相应的dns域名.因此可以为每个pod生成可解析的dns记录.
statefulset
statefulset 用来部署管理pod资源
volumeClaimTemplates
实现每个pod对应一个pv存储卷
3.
测试环境准备
kubernets集群必须部署dns,能正常解析.
kubernetes集群必须配置storageclass,能够实现动态创建pv.
我这里用的是nfs存储环境.
4.
headless service和service解析对比
service
[root@k8s-master1 ~]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
httpd-svc NodePort 10.254.33.250 <none> 80:8768/TCP 92d
kubernetes ClusterIP 10.254.0.1 <none> 443/TCP 92d注意下面解析用到dns的ip
[root@k8s-master1 ~]# kubectl get svc -n kube-system
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kube-dns ClusterIP 10.254.0.2 <none> 53/UDP,53/TCP 92d[root@k8s-master1 ~]# kubectl describe svc httpd-svc
Name: httpd-svc
Namespace: default
Labels: <none>
Annotations: kubectl.kubernetes.io/last-applied-configuration={"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"name":"httpd-svc","namespace":"default"},"spec":{"ports":[{"port":80}],"selector":{"a...
Selector: app=httpd-app
Type: NodePort
IP: 10.254.33.250
Port: <unset> 80/TCP
TargetPort: 80/TCP
NodePort: <unset> 8768/TCP
Endpoints: 172.30.37.7:80
Session Affinity: None
External Traffic Policy: Cluster
Events: <none>
[root@k8s-master1 ~]#随便进个namespace是default的pod用nslookup解析
[root@k8s-master1 ~]# kubectl exec -it nfs-client-provisioner-65bf6bd464-qdzcj nslookup httpd-svc.default.svc.cluster.local 10.254.0.2
Server: 10.254.0.2
Address 1: 10.254.0.2 kube-dns.kube-system.svc.cluster.local
Name: httpd-svc.default.svc.cluster.local
Address 1: 10.254.33.250 httpd-svc.default.svc.cluster.local
[root@k8s-master1 ~]#headless service
[root@k8s-master1 test]# cat headless-svc-test.yaml
apiVersion: v1
kind: Service
metadata:
name: headless-svc
labels:
app: headless-svc
spec:
ports:
- port: 80
name: myweb
selector:
app: headless-pod
clusterIP: None
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: headless-test
spec:
replicas: 3
selector:
matchLabels:
app: headless-pod
template:
metadata:
labels:
app: headless-pod
spec:
containers:
- image: httpd
name: myhttpd
ports:
- containerPort: 80
[root@k8s-master1 test]#[root@k8s-master1 test]# kubectl describe svc headless-svc
Name: headless-svc
Namespace: default
Labels: app=headless-svc
Annotations: kubectl.kubernetes.io/last-applied-configuration={"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"labels":{"app":"headless-svc"},"name":"headless-svc","namespace":"default"},"spec":{"...
Selector: app=headless-pod
Type: ClusterIP
IP: None
Port: myweb 80/TCP
TargetPort: 80/TCP
Endpoints: 172.30.65.4:80,172.30.65.5:80,172.30.81.6:80
Session Affinity: None
Events: <none>
[root@k8s-master1 test]#随便进个namespace是default的pod用nslookup解析
[root@k8s-master1 test]# kubectl exec -it nfs-client-provisioner-65bf6bd464-qdzcj nslookup headless-svc.default.svc.cluster.local 10.254.0.2
Server: 10.254.0.2
Address 1: 10.254.0.2 kube-dns.kube-system.svc.cluster.local
Name: headless-svc.default.svc.cluster.local
Address 1: 172.30.65.5 172-30-65-5.headless-svc.default.svc.cluster.local
Address 2: 172.30.65.4 172-30-65-4.headless-svc.default.svc.cluster.local
Address 3: 172.30.81.6 172-30-81-6.headless-svc.default.svc.cluster.local
[root@k8s-master1 test]#对比可知:
配置headless service 和service的区别只是加一个clusterIP: None
dns解析的不同
service 解析返回svc的ip和dns域名
headless service 解析返回pod的ip和dns域名
headless service解析能够返回pod的hostname.
statefulset需要固定pod name,hostname,必须要能解析pod的hostname.因此使用headless service.
5.
statefulset配置测试
[root@k8s-master1 test]# cat statefulset-test.yaml
apiVersion: v1
kind: Service
metadata:
name: headless-svc
labels:
app: headless-svc
spec:
ports:
- port: 80
name: myweb
selector:
app: headless-pod
clusterIP: None
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: statefulset-test
spec:
serviceName: headless-svc
replicas: 3
selector:
matchLabels:
app: headless-pod
template:
metadata:
labels:
app: headless-pod
spec:
containers:
- image: httpd
name: myhttpd
ports:
- containerPort: 80
name: httpd
[root@k8s-master1 test]#[root@k8s-master1 test]# kubectl describe svc headless-svc
Name: headless-svc
Namespace: default
Labels: app=headless-svc
Annotations: kubectl.kubernetes.io/last-applied-configuration={"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"labels":{"app":"headless-svc"},"name":"headless-svc","namespace":"default"},"spec":{"...
Selector: app=headless-pod
Type: ClusterIP
IP: None
Port: myweb 80/TCP
TargetPort: 80/TCP
Endpoints: 172.30.65.4:80,172.30.65.5:80,172.30.81.6:80
Session Affinity: None
Events: <none>[root@k8s-master1 test]# kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE
statefulset-test-0 1/1 Running 0 4m 172.30.65.4 k8s-master3
statefulset-test-1 1/1 Running 0 4m 172.30.81.6 k8s-master2
statefulset-test-2 1/1 Running 0 4m 172.30.65.5 k8s-master3hostname
[root@k8s-master1 test]# kubectl exec -it statefulset-test-0 hostname
statefulset-test-0
[root@k8s-master1 test]# kubectl exec -it statefulset-test-1 hostname
statefulset-test-1
[root@k8s-master1 test]# kubectl exec -it statefulset-test-2 hostname
statefulset-test-2解析
[root@k8s-master1 test]# kubectl exec -it nfs-client-provisioner-65bf6bd464-qdzcj nslookup headless-svc.default.svc.cluster.local 10.254.0.2
Server: 10.254.0.2
Address 1: 10.254.0.2 kube-dns.kube-system.svc.cluster.local
Name: headless-svc.default.svc.cluster.local
Address 1: 172.30.65.5 statefulset-test-2.headless-svc.default.svc.cluster.local
Address 2: 172.30.65.4 statefulset-test-0.headless-svc.default.svc.cluster.local
Address 3: 172.30.81.6 statefulset-test-1.headless-svc.default.svc.cluster.local
[root@k8s-master1 test]#statefulset部署和上面的head services使用deployment部署对比
pod名字是按照顺序生成,从0开始每次+1命名pod
创建pod,也是按照顺序一个一个创建,没有像deployment部署那样批量一次性全部创建.
测试
删除pod
[root@k8s-master1 test]# kubectl delete pod statefulset-test-0
pod "statefulset-test-0" deleted[root@k8s-master1 test]# kubectl get pod
NAME READY STATUS RESTARTS AGE
statefulset-test-0 0/1 Terminating 0 19m
statefulset-test-1 1/1 Running 0 19m
statefulset-test-2 1/1 Running 0 19m重新自动创建pod
[root@k8s-master1 test]# kubectl get pod
NAME READY STATUS RESTARTS AGE
statefulset-test-0 0/1 ContainerCreating 0 0s
statefulset-test-1 1/1 Running 0 19m
statefulset-test-2 1/1 Running 0 19m[root@k8s-master1 test]# kubectl get pod
NAME READY STATUS RESTARTS AGE
statefulset-test-0 1/1 Running 0 4s
statefulset-test-1 1/1 Running 0 20m
statefulset-test-2 1/1 Running 0 19m[root@k8s-master1 test]# kubectl exec -it statefulset-test-0 hostname
statefulset-test-0
[root@k8s-master1 test]#可见重新创建pod,pod的名字和hostname没有改变.
把replicas改成1
[root@k8s-master1 test]# kubectl apply -f statefulset-test.yaml
service "headless-svc" unchanged
statefulset.apps "statefulset-test" configured
[root@k8s-master1 test]# kubectl get pod
NAME READY STATUS RESTARTS AGE
statefulset-test-0 1/1 Running 0 4m
statefulset-test-1 1/1 Running 0 24m
statefulset-test-2 0/1 Terminating 0 24m
[root@k8s-master1 test]# kubectl get pod
NAME READY STATUS RESTARTS AGE
statefulset-test-0 1/1 Running 0 4m
statefulset-test-1 0/1 Terminating 0 24m
[root@k8s-master1 test]# kubectl get pod
NAME READY STATUS RESTARTS AGE
statefulset-test-0 1/1 Running 0 5mpod按照顺序一个一个删除
把replicas改回3
[root@k8s-master1 test]# kubectl apply -f statefulset-test.yaml
service "headless-svc" unchanged
statefulset.apps "statefulset-test" configured
[root@k8s-master1 test]# kubectl get pod
NAME READY STATUS RESTARTS AGE
statefulset-test-0 1/1 Running 0 6m
statefulset-test-1 0/1 ContainerCreating 0 4s
[root@k8s-master1 test]# kubectl get pod
NAME READY STATUS RESTARTS AGE
statefulset-test-0 1/1 Running 0 6m
statefulset-test-1 1/1 Running 0 8s
statefulset-test-2 0/1 ContainerCreating 0 1s
[root@k8s-master1 test]# kubectl get pod
NAME READY STATUS RESTARTS AGE
statefulset-test-0 1/1 Running 0 6m
statefulset-test-1 1/1 Running 0 14s
statefulset-test-2 1/1 Running 0 7s
[root@k8s-master1 test]#
[root@k8s-master1 test]# kubectl exec -it statefulset-test-2 hostname
statefulset-test-2
[root@k8s-master1 test]#pod按照顺序顺序一个一个创建,并且pod名字和hostname没有改变
6.
statefulset结合volumeClaimTemplates测试
[root@k8s-master1 test]# cat statefulset-test.yaml
apiVersion: v1
kind: Service
metadata:
name: headless-svc
labels:
app: headless-svc
spec:
ports:
- port: 80
name: myweb
selector:
app: headless-pod
clusterIP: None
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: statefulset-test
spec:
serviceName: headless-svc
replicas: 3
selector:
matchLabels:
app: headless-pod
template:
metadata:
labels:
app: headless-pod
spec:
containers:
- image: httpd
name: myhttpd
ports:
- containerPort: 80
name: httpd
volumeMounts:
- mountPath: /mnt
name: test
volumeClaimTemplates:
- metadata:
name: test
annotations:
volume.beta.kubernetes.io/storage-class: managed-nfs-storage
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 100Mi注意格式,多个空格少个空格都会失败.
annotations:
volume.beta.kubernetes.io/storage-class: managed-nfs-storage这是指定storageclass
[root@k8s-master1 test]# kubectl get pv,pvc
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
persistentvolume/pvc-a1c6928d-aebc-11e9-8580-000c291d7023 100Mi RWO Delete Bound default/test-statefulset-test-0 managed-nfs-storage 1m
persistentvolume/pvc-a94813df-aebc-11e9-8580-000c291d7023 100Mi RWO Delete Bound default/test-statefulset-test-1 managed-nfs-storage 1m
persistentvolume/pvc-b234f5a7-aebc-11e9-8580-000c291d7023 100Mi RWO Delete Bound default/test-statefulset-test-2 managed-nfs-storage 49s
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
persistentvolumeclaim/test-statefulset-test-0 Bound pvc-a1c6928d-aebc-11e9-8580-000c291d7023 100Mi RWO managed-nfs-storage 1m
persistentvolumeclaim/test-statefulset-test-1 Bound pvc-a94813df-aebc-11e9-8580-000c291d7023 100Mi RWO managed-nfs-storage 1m
persistentvolumeclaim/test-statefulset-test-2 Bound pvc-b234f5a7-aebc-11e9-8580-000c291d7023 100Mi RWO managed-nfs-storage 1m
[root@k8s-master1 test]#检索pod对应的pv卷,截取volumes部分对比
[root@k8s-master1 test]# kubectl describe pod statefulset-test-0
Name: statefulset-test-0
Namespace: default
Volumes:
test:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: test-statefulset-test-0
ReadOnly: false
[root@k8s-master1 test]# kubectl describe pod statefulset-test-1
Name: statefulset-test-1
Namespace: default
Volumes:
test:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: test-statefulset-test-1
ReadOnly: false
[root@k8s-master1 test]# kubectl describe pod statefulset-test-2
Name: statefulset-test-2
Namespace: default
Volumes:
test:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: test-statefulset-test-2
ReadOnly: false可以看到一个pod对应使用了一个pv卷.
检索nfs server存取目录,可以看到新建的pvc目录
[root@k8s-master3 k8s]# ls
default-test-statefulset-test-0-pvc-a1c6928d-aebc-11e9-8580-000c291d7023
default-test-statefulset-test-1-pvc-a94813df-aebc-11e9-8580-000c291d7023
default-test-statefulset-test-2-pvc-b234f5a7-aebc-11e9-8580-000c291d7023
[root@k8s-master3 k8s]#测试
删除pod,看看能重新对应原来的pvc卷吗？
[root@k8s-master1 test]# kubectl get pod
NAME READY STATUS RESTARTS AGE
statefulset-test-0 1/1 Running 0 12m
statefulset-test-1 1/1 Running 0 11m
statefulset-test-2 1/1 Running 0 11m在statefulset-test-1和statefulset-test-2 新建个文件存储在pvc
[root@k8s-master1 test]# kubectl exec -it statefulset-test-1 touch /mnt/t1
[root@k8s-master1 test]# kubectl exec -it statefulset-test-2 touch /mnt/t2检索pvc卷
[root@k8s-master3 k8s]# cd default-test-statefulset-test-1-pvc-a94813df-aebc-11e9-8580-000c291d7023/
[root@k8s-master3 default-test-statefulset-test-1-pvc-a94813df-aebc-11e9-8580-000c291d7023]# ls
t1
[root@k8s-master3 default-test-statefulset-test-1-pvc-a94813df-aebc-11e9-8580-000c291d7023]# cd ..
[root@k8s-master3 k8s]# cd default-test-statefulset-test-2-pvc-b234f5a7-aebc-11e9-8580-000c291d7023/
[root@k8s-master3 default-test-statefulset-test-2-pvc-b234f5a7-aebc-11e9-8580-000c291d7023]# ls
t2删除这两个pod
[root@k8s-master1 test]# kubectl delete pod statefulset-test-1 statefulset-test-2
pod "statefulset-test-1" deleted
pod "statefulset-test-2" deleted删除后,pod自动创建创建
再检索重新创建的pod
[root@k8s-master1 test]# kubectl describe pod statefulset-test-1
Name: statefulset-test-1
Namespace: default
Volumes:
test:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: test-statefulset-test-1
ReadOnly: false
[root@k8s-master1 test]# kubectl describe pod statefulset-test-2
Name: statefulset-test-2
Namespace: default
Volumes:
test:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: test-statefulset-test-2
ReadOnly: false可以看到pod命令hostname和存储卷全部都和删除前保持一样.
[root@k8s-master1 test]# kubectl exec -it statefulset-test-1 ls /mnt
t1
[root@k8s-master1 test]# kubectl exec -it statefulset-test-2 ls /mnt
t2
[root@k8s-master1 test]#前面创建的保存在存储卷的文件也保持不变.
7.
留下的一个疑问:
deployment部署,svc通过kube-proxy结合ipvs和iptables实现访问后端pod.
statefulset部署,svc是如何实现访问后端pod呢?规则是什么?