docker搭建redis集群高可用分片

2021-08-05 16:24:16

docker搭建redis集群高可用分片

创建redis集群的网卡

docker network create redis --subnet 192.169.0.0/16

运行输出查看网卡

[root@VM-0-3-centos pdx_haokai]# docker network create redis --subnet 192.169.0.0/16
d9437fa2e1bbc7101badf40ba17a9c8f551e0bc40bfc85ea79875fe641015f78
[root@VM-0-3-centos pdx_haokai]# docker network ls
NETWORK ID     NAME      DRIVER    SCOPE
ed8612b1d525   bridge    bridge    local
efbd70fbac53   host      host      local
a8a455ea111c   mynet     bridge    local
8b375c8baa85   none      null      local
d9437fa2e1bb   redis     bridge    local
[root@VM-0-3-centos pdx_haokai]#

编写shell脚本，创建redis集群卷本机目录

# 通过脚本创建6个redis配置
for port in $(seq 1 6); \
do \
mkdir -p /home/pdx_haokai/mydata/redis/node-${port}/conf
touch /home/pdx_haokai/mydata/redis/node-${port}/conf/redis.conf
cat << EQF >/home/pdx_haokai/mydata/redis/node-${port}/conf/redis.conf
port 6379
bind 0.0.0.0
cluster-enabled yes
cluster-config-file nodes-conf
cluster-announce-ip 192.169.0.1${port}
cluster-announce-port 6379
cluster-announce-bus-port 16379
appendonly yes
EQF
done

创建完后，目录结构是这样的

[root@VM-0-3-centos pdx_haokai]# for port in $(seq 1 6); \
> do \
> mkdir -p /home/pdx_haokai/mydata/redis/node-${port}/conf
> touch /home/pdx_haokai/mydata/redis/node-${port}/conf/redis.conf
> cat << EQF >/home/pdx_haokai/mydata/redis/node-${port}/conf/redis.conf
> port 6379
> bind 0.0.0.0
> cluster-enbaled yes
> cluster-config-file nodes-conf
> cluster-announce-ip 192.169.0.1${port}
> cluster-announce-port 6379
> cluster-announce-bus-port 16379
> appendonly yes
> EQF
> done
[root@VM-0-3-centos pdx_haokai]# cd mydata/
[root@VM-0-3-centos mydata]# ll
total 4
drwxr-xr-x 8 root root 4096 Jan 23 14:58 redis
[root@VM-0-3-centos mydata]# cd redis/
[root@VM-0-3-centos redis]# ll
total 24
drwxr-xr-x 3 root root 4096 Jan 23 14:58 node-1
drwxr-xr-x 3 root root 4096 Jan 23 14:58 node-2
drwxr-xr-x 3 root root 4096 Jan 23 14:58 node-3
drwxr-xr-x 3 root root 4096 Jan 23 14:58 node-4
drwxr-xr-x 3 root root 4096 Jan 23 14:58 node-5
drwxr-xr-x 3 root root 4096 Jan 23 14:58 node-6
[root@VM-0-3-centos redis]# cd node-1
[root@VM-0-3-centos node-1]# ll
total 4
drwxr-xr-x 2 root root 4096 Jan 23 14:58 conf
[root@VM-0-3-centos node-1]#

docker run启动6个容器(后面有脚本，可以在创建目录时同时启动容器)

参数解析：
-p开启两个端口映射，一个外网交互，一个集群间通信

docker run -p 6371:6379 -p 16371:16379 --name redis-1 \
-v /home/pdx_haokai/mydata/redis/node-1/data:/data \
-v /home/pdx_haokai/mydata/redis/node-1/conf/redis.conf:/etc/redis/redis.conf \
-d --net redis --ip 192.169.0.11 redis:5.0.9-alpine3.11 redis-server /etc/redis/redis.conf

docker run -p 6372:6379 -p 16372:16379 --name redis-2 \
-v /home/pdx_haokai/mydata/redis/node-2/data:/data \
-v /home/pdx_haokai/mydata/redis/node-2/conf/redis.conf:/etc/redis/redis.conf \
-d --net redis --ip 192.169.0.12 redis:5.0.9-alpine3.11 redis-server /etc/redis/redis.conf

docker run -p 6373:6379 -p 16373:16379 --name redis-3 \
-v /home/pdx_haokai/mydata/redis/node-3/data:/data \
-v /home/pdx_haokai/mydata/redis/node-3/conf/redis.conf:/etc/redis/redis.conf \
-d --net redis --ip 192.169.0.13 redis:5.0.9-alpine3.11 redis-server /etc/redis/redis.conf

docker run -p 6374:6379 -p 16374:16379 --name redis-4 \
-v /home/pdx_haokai/mydata/redis/node-4/data:/data \
-v /home/pdx_haokai/mydata/redis/node-4/conf/redis.conf:/etc/redis/redis.conf \
-d --net redis --ip 192.169.0.14 redis:5.0.9-alpine3.11 redis-server /etc/redis/redis.conf

docker run -p 6375:6379 -p 16375:16379 --name redis-5 \
-v /home/pdx_haokai/mydata/redis/node-5/data:/data \
-v /home/pdx_haokai/mydata/redis/node-5/conf/redis.conf:/etc/redis/redis.conf \
-d --net redis --ip 192.169.0.15 redis:5.0.9-alpine3.11 redis-server /etc/redis/redis.conf

docker run -p 6376:6379 -p 16376:16379 --name redis-6 \
-v /home/pdx_haokai/mydata/redis/node-6/data:/data \
-v /home/pdx_haokai/mydata/redis/node-6/conf/redis.conf:/etc/redis/redis.conf \
-d --net redis --ip 192.169.0.16 redis:5.0.9-alpine3.11 redis-server /etc/redis/redis.conf

运行输出：

[root@VM-0-3-centos pdx_haokai]# docker run -p 6371:6379 -p 16371:16379 --name redis-1 \
> -v /home/pdx_haokai/mydata/redis/node-1/data:/data \
> -v /home/pdx_haokai/mydata/redis/node-1/conf/redis.conf:/etc/redis/redis.conf \
> -d --net redis --ip 192.169.0.11 redis:5.0.9-alpine3.11 redis-server /etc/redis/redis.conf
e4b93f0e6623495a61f5a16fc0caa96ab39cc9d390d9ab6b4f1ff10be5b78b96
[root@VM-0-3-centos pdx_haokai]# docker run -p 6372:6379 -p 16372:16379 --name redis-2 \
> -v /home/pdx_haokai/mydata/redis/node-2/data:/data \
> -v /home/pdx_haokai/mydata/redis/node-2/conf/redis.conf:/etc/redis/redis.conf \
> -d --net redis --ip 192.169.0.12 redis:5.0.9-alpine3.11 redis-server /etc/redis/redis.conf
54f77d59dc984e175b8181bb90a53e586620ad0ec5720ad3de41bc4e22426e33
[root@VM-0-3-centos pdx_haokai]# docker run -p 6373:6379 -p 16373:16379 --name redis-3 \
> -v /home/pdx_haokai/mydata/redis/node-3/data:/data \
> -v /home/pdx_haokai/mydata/redis/node-3/conf/redis.conf:/etc/redis/redis.conf \
> -d --net redis --ip 192.169.0.13 redis:5.0.9-alpine3.11 redis-server /etc/redis/redis.conf
d69a7619d388b78052431c8ba840969a21375b22e6863f4e8b15d64ea44a0d7a
[root@VM-0-3-centos pdx_haokai]# docker run -p 6374:6379 -p 16374:16379 --name redis-4 \
> -v /home/pdx_haokai/mydata/redis/node-4/data:/data \
> -v /home/pdx_haokai/mydata/redis/node-4/conf/redis.conf:/etc/redis/redis.conf \
> -d --net redis --ip 192.169.0.14 redis:5.0.9-alpine3.11 redis-server /etc/redis/redis.conf
2ecbb5c4f6d861394db8c9d49612266a5ec8491ce1b9980675c1b0665a06728d
[root@VM-0-3-centos pdx_haokai]# docker run -p 6375:6379 -p 16375:16379 --name redis-5 \
> -v /home/pdx_haokai/mydata/redis/node-5/data:/data \
> -v /home/pdx_haokai/mydata/redis/node-5/conf/redis.conf:/etc/redis/redis.conf \
> -d --net redis --ip 192.169.0.15 redis:5.0.9-alpine3.11 redis-server /etc/redis/redis.conf
280a424940c6c64d3e4a7d0dd7bdba9cd41ea7fadad770056f5bedb69345bc90
[root@VM-0-3-centos pdx_haokai]# docker run -p 6376:6379 -p 16376:16379 --name redis-6 \
> -v /home/pdx_haokai/mydata/redis/node-6/data:/data \
> -v /home/pdx_haokai/mydata/redis/node-6/conf/redis.conf:/etc/redis/redis.conf \
> -d --net redis --ip 192.169.0.16 redis:5.0.9-alpine3.11 redis-server /etc/redis/redis.conf
80fc041e1be3f13e9569edb4d726582cfa5573171c1008c6fd7e03a3cc10f225
[root@VM-0-3-centos pdx_haokai]#

容器秒退错误信息查看

如果启动容器秒退，代表出错了，可以使用docker start -i 容器id可以看到报错内容，然后解决

[root@VM-0-3-centos pdx_haokai]# docker start -i e4b93f0e6623

*** FATAL CONFIG FILE ERROR ***
Reading the configuration file, at line 3
>>> 'cluster-enbaled yes'
Bad directive or wrong number of arguments
[root@VM-0-3-centos pdx_haokai]#

进入其中一个容器配置redis分片集群

redis-cli：代表redis单机模式
redis-cli -c：代表redis集群模式
cluster-replicas：表示分片，replicas 1 表示我们希望为集群中的每个主节点创建一个从节点，不设置分片的话，所有节点都是master。

这里进入使用的是 docker exec -it 容器id /bin/sh，因为redis容器里没有/bin/bash。

/data # redis-cli --cluster create 192.169.0.11:6379 192.169.0.12:6379 192.169.0.13:6379 192.169.0.14:6379 192.169.0.15:6379 192.169.0.16:6379 --cluster-replicas 1
>>> Performing hash slots allocation on 6 nodes...
Master[0] -> Slots 0 - 5460
Master[1] -> Slots 5461 - 10922
Master[2] -> Slots 10923 - 16383
Adding replica 192.169.0.15:6379 to 192.169.0.11:6379
Adding replica 192.169.0.16:6379 to 192.169.0.12:6379
Adding replica 192.169.0.14:6379 to 192.169.0.13:6379
M: f7388d712ae5b4a8a98ba387b10d5b1d92d124d4 192.169.0.11:6379
   slots:[0-5460] (5461 slots) master
M: 77592714f18f0c3f2d69a6990f4693cf3eeab888 192.169.0.12:6379
   slots:[5461-10922] (5462 slots) master
M: 3b045ff15b78a11509b52b29a65c5734be24244f 192.169.0.13:6379
   slots:[10923-16383] (5461 slots) master
S: dc513257ce71957a51850299edb024e0cc83103d 192.169.0.14:6379
   replicates 3b045ff15b78a11509b52b29a65c5734be24244f
S: 23fb54d508fb13e1a669a6ee77f0e14e0bcc67ca 192.169.0.15:6379
   replicates f7388d712ae5b4a8a98ba387b10d5b1d92d124d4
S: 8336d457035058752e4f703f3ead3dc355305966 192.169.0.16:6379
   replicates 77592714f18f0c3f2d69a6990f4693cf3eeab888
Can I set the above configuration? (type 'yes' to accept): yes
>>> Nodes configuration updated
>>> Assign a different config epoch to each node
>>> Sending CLUSTER MEET messages to join the cluster
Waiting for the cluster to join
....
>>> Performing Cluster Check (using node 192.169.0.11:6379)
M: f7388d712ae5b4a8a98ba387b10d5b1d92d124d4 192.169.0.11:6379
   slots:[0-5460] (5461 slots) master
   1 additional replica(s)
S: dc513257ce71957a51850299edb024e0cc83103d 192.169.0.14:6379
   slots: (0 slots) slave
   replicates 3b045ff15b78a11509b52b29a65c5734be24244f
M: 3b045ff15b78a11509b52b29a65c5734be24244f 192.169.0.13:6379
   slots:[10923-16383] (5461 slots) master
   1 additional replica(s)
S: 8336d457035058752e4f703f3ead3dc355305966 192.169.0.16:6379
   slots: (0 slots) slave
   replicates 77592714f18f0c3f2d69a6990f4693cf3eeab888
M: 77592714f18f0c3f2d69a6990f4693cf3eeab888 192.169.0.12:6379
   slots:[5461-10922] (5462 slots) master
   1 additional replica(s)
S: 23fb54d508fb13e1a669a6ee77f0e14e0bcc67ca 192.169.0.15:6379
   slots: (0 slots) slave
   replicates f7388d712ae5b4a8a98ba387b10d5b1d92d124d4
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.
/data #

redis-cli -c在容器内查看集群主从关系

docker搭建redis集群高可用分片

/data # redis-cli -c
127.0.0.1:6379> cluster info
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:6
cluster_my_epoch:1
cluster_stats_messages_ping_sent:583
cluster_stats_messages_pong_sent:596
cluster_stats_messages_sent:1179
cluster_stats_messages_ping_received:591
cluster_stats_messages_pong_received:583
cluster_stats_messages_meet_received:5
cluster_stats_messages_received:1179
127.0.0.1:6379> cluster nodes
dc513257ce71957a51850299edb024e0cc83103d 192.169.0.14:6379@16379 slave 3b045ff15b78a11509b52b29a65c5734be24244f 0 1642930532000 4 connected
3b045ff15b78a11509b52b29a65c5734be24244f 192.169.0.13:6379@16379 master - 0 1642930533000 3 connected 10923-16383
8336d457035058752e4f703f3ead3dc355305966 192.169.0.16:6379@16379 slave 77592714f18f0c3f2d69a6990f4693cf3eeab888 0 1642930533807 6 connected
f7388d712ae5b4a8a98ba387b10d5b1d92d124d4 192.169.0.11:6379@16379 myself,master - 0 1642930534000 1 connected 0-5460
77592714f18f0c3f2d69a6990f4693cf3eeab888 192.169.0.12:6379@16379 master - 0 1642930532805 2 connected 5461-10922
23fb54d508fb13e1a669a6ee77f0e14e0bcc67ca 192.169.0.15:6379@16379 slave f7388d712ae5b4a8a98ba387b10d5b1d92d124d4 0 1642930534811 5 connected
127.0.0.1:6379>

高可用测试

redis主从高可用，保证主节点挂掉，从节点会自动通过选举成为主节点，同时各节点数据主从复制。

插入值

可以看到这里是哪个节点去执行的操作

127.0.0.1:6379> set a b
-> Redirected to slot [15495] located at 192.169.0.13:6379
OK
192.169.0.13:6379>

删除操作节点

把192.169.0.13这台容器停掉

[root@VM-0-3-centos pdx_haokai]# docker ps
CONTAINER ID   IMAGE                    COMMAND                  CREATED       STATUS       PORTS                                                                                      NAMES
3bbef75d54ab   redis:5.0.9-alpine3.11   "docker-entrypoint.s…"   2 hours ago   Up 2 hours   0.0.0.0:6376->6379/tcp, :::6376->6379/tcp, 0.0.0.0:16376->16379/tcp, :::16376->16379/tcp   redis-6
93964b0af24c   redis:5.0.9-alpine3.11   "docker-entrypoint.s…"   2 hours ago   Up 2 hours   0.0.0.0:6375->6379/tcp, :::6375->6379/tcp, 0.0.0.0:16375->16379/tcp, :::16375->16379/tcp   redis-5
428cf378b4e8   redis:5.0.9-alpine3.11   "docker-entrypoint.s…"   2 hours ago   Up 2 hours   0.0.0.0:6374->6379/tcp, :::6374->6379/tcp, 0.0.0.0:16374->16379/tcp, :::16374->16379/tcp   redis-4
9eb3ddaf7b7e   redis:5.0.9-alpine3.11   "docker-entrypoint.s…"   2 hours ago   Up 2 hours   0.0.0.0:6373->6379/tcp, :::6373->6379/tcp, 0.0.0.0:16373->16379/tcp, :::16373->16379/tcp   redis-3
e434d767a593   redis:5.0.9-alpine3.11   "docker-entrypoint.s…"   2 hours ago   Up 2 hours   0.0.0.0:6372->6379/tcp, :::6372->6379/tcp, 0.0.0.0:16372->16379/tcp, :::16372->16379/tcp   redis-2
a4acd5a31089   redis:5.0.9-alpine3.11   "docker-entrypoint.s…"   2 hours ago   Up 2 hours   0.0.0.0:6371->6379/tcp, :::6371->6379/tcp, 0.0.0.0:16371->16379/tcp, :::16371->16379/tcp   redis-1
[root@VM-0-3-centos pdx_haokai]# docker stop 9eb3ddaf7b7e
9eb3ddaf7b7e
[root@VM-0-3-centos pdx_haokai]#

查看集群节点状态，并获取值

原节点主从关系:
docker搭建redis集群高可用分片
14由从节点被选举为主节点：
docker搭建redis集群高可用分片

127.0.0.1:6379> cluster nodes
dc513257ce71957a51850299edb024e0cc83103d 192.169.0.14:6379@16379 master - 0 1642931001000 7 connected 10923-16383
3b045ff15b78a11509b52b29a65c5734be24244f 192.169.0.13:6379@16379 master,fail - 1642930899733 1642930897000 3 connected
8336d457035058752e4f703f3ead3dc355305966 192.169.0.16:6379@16379 slave 77592714f18f0c3f2d69a6990f4693cf3eeab888 0 1642931001870 6 connected
f7388d712ae5b4a8a98ba387b10d5b1d92d124d4 192.169.0.11:6379@16379 myself,master - 0 1642930998000 1 connected 0-5460
77592714f18f0c3f2d69a6990f4693cf3eeab888 192.169.0.12:6379@16379 master - 0 1642930999000 2 connected 5461-10922
23fb54d508fb13e1a669a6ee77f0e14e0bcc67ca 192.169.0.15:6379@16379 slave f7388d712ae5b4a8a98ba387b10d5b1d92d124d4 0 1642931000867 5 connected
127.0.0.1:6379> get a
-> Redirected to slot [15495] located at 192.169.0.14:6379
"b"
192.169.0.14:6379>

查看本机挂载目录

[pdx_haokai@VM-0-3-centos redis]$ cd node-1
[pdx_haokai@VM-0-3-centos node-1]$ ll
total 8
drwxr-xr-x 2 root    root 4096 Jan 23 15:45 conf
drwxr-xr-x 2 polkitd root 4096 Jan 23 17:25 data
[pdx_haokai@VM-0-3-centos node-1]$ cd data/
[pdx_haokai@VM-0-3-centos data]$ ll
total 8
-rw-r--r-- 1 polkitd pdx_haokai   0 Jan 23 15:46 appendonly.aof
-rw-r--r-- 1 polkitd pdx_haokai 175 Jan 23 17:25 dump.rdb
-rw-r--r-- 1 polkitd pdx_haokai 778 Jan 23 17:41 nodes-conf
[pdx_haokai@VM-0-3-centos data]$

shell脚本执行创建数据卷目录及运行6容器

参数解析：
-p开启两个端口映射，一个外网交互，一个集群间通信

# 通过脚本创建6个redis配置并运行容器
for port in $(seq 1 6); \
do \
mkdir -p /home/pdx_haokai/mydata/redis/node-${port}/conf
touch /home/pdx_haokai/mydata/redis/node-${port}/conf/redis.conf
cat << EQF >/home/pdx_haokai/mydata/redis/node-${port}/conf/redis.conf
port 6379
bind 0.0.0.0
cluster-enabled yes
cluster-config-file nodes-conf
cluster-announce-ip 192.169.0.1${port}
cluster-announce-port 6379
cluster-announce-bus-port 16379
appendonly yes
EQF
docker run -p 637${port}:6379 -p 1637${port}:16379 --name redis-${port} \
-v /home/pdx_haokai/mydata/redis/node-${port}/data:/data \
-v /home/pdx_haokai/mydata/redis/node-${port}/conf/redis.conf:/etc/redis/redis.conf \
-d --net redis --ip 192.169.0.1${port} redis:5.0.9-alpine3.11 redis-server /etc/redis/redis.conf
done

运行输出：

[root@VM-0-3-centos pdx_haokai]# for port in $(seq 1 6); \
> do \
> mkdir -p /home/pdx_haokai/mydata/redis/node-${port}/conf
> touch /home/pdx_haokai/mydata/redis/node-${port}/conf/redis.conf
> cat << EQF >/home/pdx_haokai/mydata/redis/node-${port}/conf/redis.conf
> port 6379
> bind 0.0.0.0
> cluster-enabled yes
> cluster-config-file nodes-conf
> cluster-announce-ip 192.169.0.1${port}
> cluster-announce-port 6379
> cluster-announce-bus-port 16379
> appendonly yes
> EQF
> docker run -p 637${port}:6379 -p 1637${port}:16379 --name redis-${port} \
> -v /home/pdx_haokai/mydata/redis/node-${port}/data:/data \
> -v /home/pdx_haokai/mydata/redis/node-${port}/conf/redis.conf:/etc/redis/redis.conf \
> -d --net redis --ip 192.169.0.1${port} redis:5.0.9-alpine3.11 redis-server /etc/redis/redis.conf
> done
a4acd5a310898b6deafe4b9cfe76253d49de68e41e6b9322d296e20008262622
e434d767a593b993c485560084f39d000c67f9795d5896db140d57d86a99b833
9eb3ddaf7b7ebcde99370b2ebbbadfe1d945b943fd4c8f23c44432a6b3c04dbc
428cf378b4e88e4c9825e17cfb836213d3c34ae9f31ec64eb2f0472d891d4323
93964b0af24cc7c22f4d7f7afafc29591142312f160c2ef51146ae62c3173caf
3bbef75d54ab9eb929274c7628076d1305414954bf8e669a16d7ceedbc6a364e

[root@VM-0-3-centos pdx_haokai]# docker ps
CONTAINER ID   IMAGE                    COMMAND                  CREATED          STATUS          PORTS                                                                                      NAMES
3bbef75d54ab   redis:5.0.9-alpine3.11   "docker-entrypoint.s…"   11 seconds ago   Up 9 seconds    0.0.0.0:6376->6379/tcp, :::6376->6379/tcp, 0.0.0.0:16376->16379/tcp, :::16376->16379/tcp   redis-6
93964b0af24c   redis:5.0.9-alpine3.11   "docker-entrypoint.s…"   11 seconds ago   Up 10 seconds   0.0.0.0:6375->6379/tcp, :::6375->6379/tcp, 0.0.0.0:16375->16379/tcp, :::16375->16379/tcp   redis-5
428cf378b4e8   redis:5.0.9-alpine3.11   "docker-entrypoint.s…"   12 seconds ago   Up 10 seconds   0.0.0.0:6374->6379/tcp, :::6374->6379/tcp, 0.0.0.0:16374->16379/tcp, :::16374->16379/tcp   redis-4
9eb3ddaf7b7e   redis:5.0.9-alpine3.11   "docker-entrypoint.s…"   12 seconds ago   Up 11 seconds   0.0.0.0:6373->6379/tcp, :::6373->6379/tcp, 0.0.0.0:16373->16379/tcp, :::16373->16379/tcp   redis-3
e434d767a593   redis:5.0.9-alpine3.11   "docker-entrypoint.s…"   13 seconds ago   Up 11 seconds   0.0.0.0:6372->6379/tcp, :::6372->6379/tcp, 0.0.0.0:16372->16379/tcp, :::16372->16379/tcp   redis-2
a4acd5a31089   redis:5.0.9-alpine3.11   "docker-entrypoint.s…"   13 seconds ago   Up 12 seconds   0.0.0.0:6371->6379/tcp, :::6371->6379/tcp, 0.0.0.0:16371->16379/tcp, :::16371->16379/tcp   redis-1
[root@VM-0-3-centos pdx_haokai]#
上一篇：docker 在部署 Redis 容器使用 redis.conf 配置启动失败


下一篇：docker安装nginx，以及常用操作