8.2 k8s 基于StatefulSet运行mysql 一主多从 ,数据通过pv/pvc结合NFS服务器持久化

1.准备mysql和xtrabackup镜像

下载mysql官方镜像并上传到本地harbor

docker pull mysql:5.7
docker tag m ysql:5.7 192.168.1.110/base/mysql:5.7
docker push 192.168.1.110/base/mysql:5.7

下载xtrabackup镜像并上传到本地harbor

docker pull registry.cn-hangzhou.aliyuncs.com/hxpdocker/xtrabackup:1.0
docker tag registry.cn-hangzhou.aliyuncs.com/hxpdocker/xtrabackup:1.0 192.168.1.110/base/xtrabackup:1.0
docker push 192.168.1.110/base/xtrabackup:1.0

2.为mysql数据目录准备PV

nfs创建目录

# mkdir /data/k8s-data/mysql/mysql-datadir-1 -p
# mkdir /data/k8s-data/mysql/mysql-datadir-2 -p
# mkdir /data/k8s-data/mysql/mysql-datadir-3 -p
# mkdir /data/k8s-data/mysql/mysql-datadir-4 -p
# mkdir /data/k8s-data/mysql/mysql-datadir-5 -p

编写pv的yaml

root@k8-deploy:~/k8s-yaml/mysql# cat pv/mysql-datadir-pv.yaml 
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-1
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.2.10
    path: /data/k8s-data/mysql/mysql-datadir-1
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-2
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.2.10
    path: /data/k8s-data/mysql/mysql-datadir-2
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-3
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.2.10
    path: /data/k8s-data/mysql/mysql-datadir-3
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-4
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.2.10
    path: /data/k8s-data/mysql/mysql-datadir-4
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-5
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.2.10
    path: /data/k8s-data/mysql/mysql-datadir-5

创建PV

root@k8-deploy:~/k8s-yaml/mysql# kubectl apply -f mysql-datadir-pv.yaml 
persistentvolume/mysql-datadir-1 created
persistentvolume/mysql-datadir-2 created
persistentvolume/mysql-datadir-3 created
persistentvolume/mysql-datadir-4 created
persistentvolume/mysql-datadir-5 created

root@k8-deploy:~/k8s-yaml/mysql# kubectl get pv
NAME                     CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM                           STORAGECLASS   REASON   AGE
mysql-datadir-1          10Gi       RWO            Retain           Available                                                           3s
mysql-datadir-2          10Gi       RWO            Retain           Available                                                           3s
mysql-datadir-3          10Gi       RWO            Retain           Available                                                           3s
mysql-datadir-4          10Gi       RWO            Retain           Available                                                           3s
mysql-datadir-5          10Gi       RWO            Retain           Available                                                           3s

3.部署StatefulSet的mysql集群

3.1 编写yaml文件

configmap.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: mysql
  labels:
    app: mysql
data:
  master.cnf: |
    # Apply this config only on the master.
    [mysqld]
    log-bin    
  slave.cnf: |
    # Apply this config only on slaves.
    [mysqld]
    super-read-only    

mysql-services.yaml

 Headless service for stable DNS entries of StatefulSet members.
apiVersion: v1
kind: Service
metadata:
  name: mysql
  labels:
    app: mysql
spec:
  ports:
  - name: mysql
    port: 3306
  clusterIP: None
  selector:
    app: mysql
---
# Client service for connecting to any MySQL instance for reads.
# For writes, you must instead connect to the master: mysql-0.mysql.
apiVersion: v1
kind: Service
metadata:
  name: mysql-read
  labels:
    app: mysql
spec:
  ports:
  - name: mysql
    port: 3306
  selector:
    app: mysql

statefulset.yaml

# cat mysql-statefulset.yaml 
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: mysql
spec:
  selector:
    matchLabels:
      app: mysql
  serviceName: mysql
  replicas: 3
  template:
    metadata:
      labels:
        app: mysql
    spec:
      initContainers:
      - name: init-mysql
        image: 192.168.1.110/base/mysql:5.7
        command:
        - bash
        - "-c"
        - |
          set -ex
          # Generate mysql server-id from pod ordinal index.
          [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
          ordinal=${BASH_REMATCH[1]}
          echo [mysqld] > /mnt/conf.d/server-id.cnf
          # Add an offset to avoid reserved server-id=0 value.
          echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf
          # Copy appropriate conf.d files from config-map to emptyDir.
          if [[ $ordinal -eq 0 ]]; then
            cp /mnt/config-map/master.cnf /mnt/conf.d/
          else
            cp /mnt/config-map/slave.cnf /mnt/conf.d/
          fi          
        volumeMounts:
        - name: conf
          mountPath: /mnt/conf.d
        - name: config-map
          mountPath: /mnt/config-map
      - name: clone-mysql
        image: 192.168.1.110/base/xtrabackup:1.0
        command:
        - bash
        - "-c"
        - |
          set -ex
          # Skip the clone if data already exists.
          [[ -d /var/lib/mysql/mysql ]] && exit 0
          # Skip the clone on master (ordinal index 0).
          [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
          ordinal=${BASH_REMATCH[1]}
          [[ $ordinal -eq 0 ]] && exit 0
          # Clone data from previous peer.
          ncat --recv-only mysql-$(($ordinal-1)).mysql 3307 | xbstream -x -C /var/lib/mysql
          # Prepare the backup.
          xtrabackup --prepare --target-dir=/var/lib/mysql          
        volumeMounts:
        - name: data
          mountPath: /var/lib/mysql
          subPath: mysql
        - name: conf
          mountPath: /etc/mysql/conf.d
      containers:
      - name: mysql
        image: 192.168.1.110/base/mysql:5.7
        env:
        - name: MYSQL_ALLOW_EMPTY_PASSWORD
          value: "1"
        ports:
        - name: mysql
          containerPort: 3306
        volumeMounts:
        - name: data
          mountPath: /var/lib/mysql
          subPath: mysql
        - name: conf
          mountPath: /etc/mysql/conf.d
        resources:
          requests:
            cpu: 500m
            memory: 1Gi
        livenessProbe:
          exec:
            command: ["mysqladmin", "ping"]
          initialDelaySeconds: 30
          periodSeconds: 10
          timeoutSeconds: 5
        readinessProbe:
          exec:
            # Check we can execute queries over TCP (skip-networking is off).
            command: ["mysql", "-h", "127.0.0.1", "-e", "SELECT 1"]
          initialDelaySeconds: 5
          periodSeconds: 2
          timeoutSeconds: 1
      - name: xtrabackup
        image: 192.168.1.110/base/xtrabackup:1.0
        ports:
        - name: xtrabackup
          containerPort: 3307
        command:
        - bash
        - "-c"
        - |
          set -ex
          cd /var/lib/mysql

          # Determine binlog position of cloned data, if any.
          if [[ -f xtrabackup_slave_info && "x$(<xtrabackup_slave_info)" != "x" ]]; then
            # XtraBackup already generated a partial "CHANGE MASTER TO" query
            # because we're cloning from an existing slave. (Need to remove the tailing semicolon!)
            cat xtrabackup_slave_info | sed -E 's/;$//g' > change_master_to.sql.in
            # Ignore xtrabackup_binlog_info in this case (it's useless).
            rm -f xtrabackup_slave_info xtrabackup_binlog_info
          elif [[ -f xtrabackup_binlog_info ]]; then
            # We're cloning directly from master. Parse binlog position.
            [[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1
            rm -f xtrabackup_binlog_info xtrabackup_slave_info
            echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\
                  MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in
          fi

          # Check if we need to complete a clone by starting replication.
          if [[ -f change_master_to.sql.in ]]; then
            echo "Waiting for mysqld to be ready (accepting connections)"
            until mysql -h 127.0.0.1 -e "SELECT 1"; do sleep 1; done

            echo "Initializing replication from clone position"
            mysql -h 127.0.0.1 \
                  -e "$(<change_master_to.sql.in), \
                          MASTER_HOST='mysql-0.mysql', \
                          MASTER_USER='root', \
                          MASTER_PASSWORD='', \
                          MASTER_CONNECT_RETRY=10; \
                        START SLAVE;" || exit 1
            # In case of container restart, attempt this at-most-once.
            mv change_master_to.sql.in change_master_to.sql.orig
          fi

          # Start a server to send backups when requested by peers.
          exec ncat --listen --keep-open --send-only --max-conns=1 3307 -c \
            "xtrabackup --backup --slave-info --stream=xbstream --host=127.0.0.1 --user=root"          
        volumeMounts:
        - name: data
          mountPath: /var/lib/mysql
          subPath: mysql
        - name: conf
          mountPath: /etc/mysql/conf.d
        resources:
          requests:
            cpu: 100m
            memory: 100Mi
      volumes:
      - name: conf
        emptyDir: {}
      - name: config-map
        configMap:
          name: mysql
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes: ["ReadWriteOnce"]
      resources:
        requests:
          storage: 10Gi

3.2 创建集群

kubectl apply -f mysql-configmap.yaml
kubectl apply -f mysql-services.yaml
kubectl apply -f mysql-statefulset.yaml

3.3 测试集群

查看pod是否创建成功

root@k8-deploy:~/k8s-yaml/mysql# kubectl get pod
NAME      READY   STATUS    RESTARTS   AGE
mysql-0   2/2     Running   0          3m13s
mysql-1   2/2     Running   1          2m44s
mysql-2   2/2     Running   1          99s

进入pod查看挂载pv

root@k8-deploy:~/k8s-yaml/mysql# kubectl exec mysql-0 -it -- bash
root@mysql-0:/# df -h
Filesystem                                               Size  Used Avail Use% Mounted on
...
192.168.2.10:/data/k8s-data/mysql/mysql-datadir-4/mysql   98G   20G   74G  21% /var/lib/mysql
...

进入mysql-0 pod(mysql主)查看mysql状态,并创建测试数据库

root@mysql-0:/# mysql
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 148
Server version: 5.7.36-log MySQL Community Server (GPL)

Copyright (c) 2000, 2021, Oracle and/or its affiliates.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql> show master status\G;
*************************** 1. row ***************************
             File: mysql-0-bin.000003
         Position: 154
     Binlog_Do_DB: 
 Binlog_Ignore_DB: 
Executed_Gtid_Set: 
1 row in set (0.00 sec)

ERROR: 
No query specified

mysql> shwo databases;
ERROR 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'shwo databases' at line 1
mysql> show  databases;  
+------------------------+
| Database               |
+------------------------+
| information_schema     |
| mysql                  |
| performance_schema     |
| sys                    |
| xtrabackup_backupfiles |
+------------------------+
5 rows in set (0.04 sec)

mysql> create database yan_test;
Query OK, 1 row affected (0.02 sec)

mysql> show  databases;
+------------------------+
| Database               |
+------------------------+
| information_schema     |
| mysql                  |
| performance_schema     |
| sys                    |
| xtrabackup_backupfiles |
| yan_test               |
+------------------------+
6 rows in set (0.01 sec)

mysql> exit

依次查看其他从库的主从同步状态

root@mysql-1:/# mysql
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 140
Server version: 5.7.36 MySQL Community Server (GPL)

Copyright (c) 2000, 2021, Oracle and/or its affiliates.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql> show slave status\G;
*************************** 1. row ***************************
               Slave_IO_State: Waiting for master to send event
                  Master_Host: mysql-0.mysql
                  Master_User: root
                  Master_Port: 3306
                Connect_Retry: 10
              Master_Log_File: mysql-0-bin.000003
          Read_Master_Log_Pos: 325
               Relay_Log_File: mysql-1-relay-bin.000002
                Relay_Log_Pos: 493
        Relay_Master_Log_File: mysql-0-bin.000003
             Slave_IO_Running: Yes
            Slave_SQL_Running: Yes
...

mysql> show databases;
+------------------------+
| Database               |
+------------------------+
| information_schema     |
| mysql                  |
| performance_schema     |
| sys                    |
| xtrabackup_backupfiles |
| yan_test               |
+------------------------+
6 rows in set (0.02 sec)

查看mysql的数据是否写入nfs

root@k8-deploy:~/k8s-yaml/mysql# tree -L 2 /data/k8s-data/mysql/
/data/k8s-data/mysql/
├── mysql-datadir-1
│   └── mysql
├── mysql-datadir-2
│   └── mysql
├── mysql-datadir-3
├── mysql-datadir-4
│   └── mysql
└── mysql-datadir-5

4.扩缩容测试

4.1 将集群副本增加到4个

root@k8-deploy:~/k8s-yaml/mysql# kubectl scale -n default statefulset mysql --replicas=4
statefulset.apps/mysql scaled

root@k8-deploy:~/k8s-yaml/mysql# kubectl get pod
NAME      READY   STATUS    RESTARTS   AGE
mysql-0   2/2     Running   0          13m
mysql-1   2/2     Running   1          12m
mysql-2   2/2     Running   1          11m
mysql-3   1/2     Running   1          24s

4.2 查看增加的mysql-3的主从状态

root@mysql-3:/# mysql
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 13
Server version: 5.7.36 MySQL Community Server (GPL)

Copyright (c) 2000, 2021, Oracle and/or its affiliates.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql> show slave status\G;
*************************** 1. row ***************************
               Slave_IO_State: Waiting for master to send event
                  Master_Host: mysql-0.mysql
                  Master_User: root
                  Master_Port: 3306
                Connect_Retry: 10
              Master_Log_File: mysql-0-bin.000003
          Read_Master_Log_Pos: 325
               Relay_Log_File: mysql-3-relay-bin.000002
                Relay_Log_Pos: 322
        Relay_Master_Log_File: mysql-0-bin.000003
             Slave_IO_Running: Yes
            Slave_SQL_Running: Yes
...

mysql> show databases;
+------------------------+
| Database               |
+------------------------+
| information_schema     |
| mysql                  |
| performance_schema     |
| sys                    |
| xtrabackup_backupfiles |
| yan_test               |
+------------------------+
6 rows in set (0.02 sec)

4.3 查看nfs mysql数据目录

root@k8-deploy:~/k8s-yaml/mysql# tree -L 2 /data/k8s-data/mysql/
/data/k8s-data/mysql/
├── mysql-datadir-1
│   └── mysql
├── mysql-datadir-2
│   └── mysql
├── mysql-datadir-3
│   └── mysql
├── mysql-datadir-4
│   └── mysql
└── mysql-datadir-5

9 directories, 0 files

4.4 将副本数缩容至2个

root@k8-deploy:~/k8s-yaml/mysql# kubectl scale -n default statefulset mysql --replicas=2
statefulset.apps/mysql scaled

4.4 查看pod缩容的过程(-w 参数)

root@k8-deploy:~/k8s-yaml/mysql# kubectl get pod -w
NAME      READY   STATUS        RESTARTS   AGE
mysql-0   2/2     Running       0          47m
mysql-1   2/2     Running       1          46m
mysql-2   2/2     Running       1          45m
mysql-3   2/2     Terminating   1          34m
mysql-3   0/2     Terminating   1          34m
mysql-3   0/2     Terminating   1          34m
mysql-3   0/2     Terminating   1          34m
mysql-2   2/2     Terminating   1          46m

root@k8-deploy:~/k8s-yaml/mysql# kubectl get pod
NAME      READY   STATUS    RESTARTS   AGE
mysql-0   2/2     Running   0          48m
mysql-1   2/2     Running   1          47m

4.4 再次查看mysql-1的主从状态是否正常。

# kubectl exec mysql-1 -it -- bash 
mysql> show slave status\G;
*************************** 1. row ***************************
               Slave_IO_State: Waiting for master to send event
                  Master_Host: mysql-0.mysql
                  Master_User: root
                  Master_Port: 3306
                Connect_Retry: 10
              Master_Log_File: mysql-0-bin.000003
          Read_Master_Log_Pos: 325
               Relay_Log_File: mysql-1-relay-bin.000002
                Relay_Log_Pos: 493
        Relay_Master_Log_File: mysql-0-bin.000003
             Slave_IO_Running: Yes
            Slave_SQL_Running: Yes

4.4 缩容后再次查看nfs上mysql的数据目录(k8s并不会自动删除清理,需要手动操作)

root@k8-deploy:~/k8s-yaml/mysql# tree -L 2 /data/k8s-data/mysql/
/data/k8s-data/mysql/
├── mysql-datadir-1
│   └── mysql
├── mysql-datadir-2
│   └── mysql
├── mysql-datadir-3
│   └── mysql
├── mysql-datadir-4
│   └── mysql
└── mysql-datadir-5

9 directories, 0 files
上一篇:实现无密钥编排 转载:https://help.aliyun.com/document_detail/86562.html


下一篇:ARM uart-2