实现 master/master 的 Keepalived 双主架构
master/slave的单主架构,同一时间只有一个Keepalived对外提供服务,此主机繁忙,而另一台主机却
很空闲,利用率低下,可以使用master/master的双主架构,解决此问题。
master/master 的双主架构:
即将两个或以上VIP分别运行在不同的keepalived服务器,以实现服务器并行提供web访问的目的,提高
服务器资源利用率
#ha1主机配置
[root@ka1-centos8 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@qq.com
}
notification_email_from keepalived@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id ka1.longxuan.vip
vrrp_mcast_group4 224.8.8.188
}
vrrp_instance N520 {
state MASTER #在另一个主机上为BACKUP
interface eth0
virtual_router_id 66 #每个vrrp_instance唯一
priority 100 #在另一个主机上为80
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
172.31.0.188 dev eth0 label eth0:1 #指定vrrp_instance各自的VIP
}
}
vrrp_instance N521 { #添加 VI_2 实例
state BACKUP #在另一个主机上为MASTER
interface eth0
virtual_router_id 88 #每个vrrp_instance唯一
priority 80 #在另一个主机上为100
advert_int 1
authentication {
auth_type PASS
auth_pass 12345678
}
virtual_ipaddress {
172.31.0.200 dev eth0 label eth0:2 #指定vrrp_instance各自的VIP
}
}
#ka2主机配置,和ka1配置只需五行不同
[root@ka2-centos8 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@qq.com
}
notification_email_from keepalived@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id ka2.longxuan.vip #修改此行
vrrp_mcast_group4 224.8.8.100
}
vrrp_instance N520 {
state BACKUP #此修改行为BACKUP
interface eth0
virtual_router_id 66
priority 80 #此修改行为80
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
172.31.0.188 dev eth0 label eth0:1
}
}
vrrp_instance N521 {
state MASTER #修改此行为MASTER
interface eth0
virtual_router_id 88
priority 100 #修改此行为100
advert_int 1
authentication {
auth_type PASS
auth_pass 12345678
}
virtual_ipaddress {
172.31.0.200 dev eth0 label eth0:2
}
}
实战案例:利用子配置文件实现master/master的Keepalived双主架构
范例:ka1配置
[root@centos8 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
acassen
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id LVS_DEVEL
}
include /etc/keepalived/conf.d/*.conf
# 创建子目录
[root@centos8 ~]# mkdir /etc/keepalived/conf.d/ -p
# 项目1-master
[root@centos8 ~]# cat /etc/keepalived/conf.d/n520.conf
vrrp_instance n520 {
state MASTER
interface eth0
virtual_router_id 66
priority 100
advert_int 1
#nopreempt
#preempt_delay 10
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
172.31.0.188 dev eth0 label eth0:1
}
unicast_src_ip 172.31.0.28
unicast_peer{
172.31.0.48
}
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"
}
# 项目2-backup
[root@centos8 ~]# vim /etc/keepalived/conf.d/n521.conf
vrrp_instance n521 {
state BACKUP
interface eth0
virtual_router_id 88
priority 80
advert_int 1
authentication {
auth_type PASS
auth_pass 12345678
}
virtual_ipaddress {
172.31.0.200 dev eth0 label eth0:2
}
unicast_src_ip 172.31.0.28
unicast_peer{
172.31.0.48
}
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"
}
# 重启
[root@centos8 ~]# systemctl restart keepalived
检查ip
[root@centos8 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 00:0c:29:ac:f5:a4 brd ff:ff:ff:ff:ff:ff
inet 172.31.0.28/16 brd 172.31.255.255 scope global noprefixroute eth0
valid_lft forever preferred_lft forever
inet 172.31.0.188/32 scope global eth0:1
valid_lft forever preferred_lft forever
inet 172.31.0.200/32 scope global eth0:2
范例:ka2配置
[root@centos8 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
acassen
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id LVS_DEVEL
}
include /etc/keepalived/conf.d/*.conf
# 创建子目录
[root@centos8 ~]# mkdir /etc/keepalived/conf.d/ -p
# 项目1-backup
[root@centos8 ~]# cat /etc/keepalived/conf.d/n520.conf
vrrp_instance n520 {
state BACKUP
interface eth0
virtual_router_id 66
priority 80
advert_int 1
#nopreempt
#preempt_delay 10 #如果设置延迟抢占模式,需要都是BACKUP,不然会造成脑裂
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
172.31.0.188 dev eth0 label eth0:1
}
unicast_src_ip 172.31.0.48
unicast_peer{
172.31.0.28
}
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"
}
# 项目2-master
[root@centos8 ~]# cat //etc/keepalived/conf.d/n521.conf
vrrp_instance n521 {
state MASTER
interface eth0
virtual_router_id 88
priority 80
advert_int 1
authentication {
auth_type PASS
auth_pass 12345678
}
virtual_ipaddress {
172.31.0.200 dev eth0 label eth0:2
}
unicast_src_ip 172.31.0.48
unicast_peer{
172.31.0.28
}
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"
}
# 重启keepalived
[root@centos8 ~]# systemctl restart keepalived
# 查看ip
[root@centos8 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 00:0c:29:16:9a:81 brd ff:ff:ff:ff:ff:ff
inet 172.31.0.48/16 brd 172.31.255.255 scope global noprefixroute eth0
valid_lft forever preferred_lft forever
inet 172.31.0.200/32 scope global eth0:2
valid_lft forever preferred_lft forever
查看ip
#ka1
[root@centos8 ~]# hostname -I
172.31.0.28 172.31.0.188
# ka2
[root@centos8 ~]# hostname -I
172.31.0.48 172.31.0.200
ka1主机故障,测试VIP漂移至ka2主机
[root@centos8 ~]# systemctl stop keepalived
[root@centos8 ~]# hostname -I
172.31.0.28
[root@centos8 ~]# hostname -I
172.31.0.48 172.31.0.188 172.31.0.200
恢复ka1主机
[root@centos8 ~]# hostname -I
172.31.0.28 172.31.0.188
[root@centos8 ~]# hostname -I
172.31.0.48 172.31.0.200
模拟脑裂现象
# ka1
[root@centos8 ~]# iptables -A INPUT -s 172.31.0.48 -j DROP
[root@centos8 ~]# iptables -vnL
Chain INPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
12 480 DROP all -- * * 172.31.0.48 0.0.0.0/0
...
[root@centos8 ~]# hostname -I
172.31.0.28 172.31.0.188 172.31.0.200
[root@centos8 ~]# hostname -I
172.31.0.48 172.31.0.200
# ka2
[root@centos8 ~]# iptables -A INPUT -s 172.31.0.28 -j REJECT
[root@centos8 ~]# iptables -vnL
Chain INPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
10 400 REJECT all -- * * 172.31.0.28 0.0.0.0/0 reject-with icmp-port-unreachable
...
[root@centos8 ~]# hostname -I
172.31.0.48 172.31.0.200 172.31.0.188
[root@centos8 ~]# hostname -I
172.31.0.28 172.31.0.188 172.31.0.200
总结脑裂现象:当ka1添加错误的iptables规则后查询ip就会看到有三个ip地址;ka2查询还是原来的两个ip地址,同时如果ka2也添加了错误的iptables规则后,查询ip就会看到是三个ip地址,首先backup收不到master的vrrp通信心跳检测,backup就会把虚拟vip抢过来,但实际master还存在的,只是拒绝了backup请求响应从而造成脑裂现象
实现IPVS的高可用性
虚拟服务器配置结构
virtual_server IP port {
...
real_server {
...
}
real_server {
...
}
…
}
virtual server (虚拟服务器)的定义格式
virtual_server IP port #定义虚拟主机IP地址及其端口
virtual_server fwmark int #ipvs的防火墙打标,实现基于防火墙的负载均衡集群
virtual_server group string #使用虚拟服务器组
虚拟服务器组
将多个虚拟服务器定义成一个组,统一对外服务,如:http和https定义成一个虚拟服务器组
#参考文档:/usr/share/doc/keepalived/keepalived.conf.virtual_server_group
virtual_server_group <STRING> {
# Virtual IP Address and Port
<IPADDR> <PORT>
<IPADDR> <PORT>
...
# <IPADDR RANGE> has the form
...
虚拟服务器配置
virtual_server IP port { #VIP和PORT
delay_loop <INT> #检查后端服务器的时间间隔
lb_algo rr|wrr|lc|wlc|lblc|sh|dh #定义调度方法
lb_kind NAT|DR|TUN #集群的类型,注意要大写
persistence_timeout <INT> #持久连接时长
protocol TCP|UDP|SCTP #指定服务协议,一般为TCP
sorry_server <IPADDR> <PORT> #所有RS故障时,备用服务器地址
real_server <IPADDR> <PORT> { #RS的IP和PORT
weight <INT> #RS权重
notify_up <STRING>|<QUOTED-STRING> #RS上线通知脚本
notify_down <STRING>|<QUOTED-STRING> #RS下线通知脚本
HTTP_GET|SSL_GET|TCP_CHECK|SMTP_CHECK|MISC_CHECK { ... } #定义当前主机健康状态检测方法
}
}
#注意:括号必须分行写,两个括号写在同一行,如: }} 会出错
应用层监测
应用层检测:HTTP_GET|SSL_GET
HTTP_GET|SSL_GET {
url {
path <URL_PATH> #定义要监控的URL
status_code <INT> #判断上述检测机制为健康状态的响应码,一般为 200
}
connect_timeout <INTEGER> #客户端请求的超时时长, 相当于haproxy的timeout server
nb_get_retry <INT> #重试次数
delay_before_retry <INT> #重试之前的延迟时长
connect_ip <IP ADDRESS> #向当前RS哪个IP地址发起健康状态检测请求
connect_port <PORT> #向当前RS的哪个PORT发起健康状态检测请求
bindto <IP ADDRESS> #向当前RS发出健康状态检测请求时使用的源地址
bind_port <PORT> #向当前RS发出健康状态检测请求时使用的源端口
}
范例:
virtual_server 172.31.0.188 80 {
delay_loop 3
lb_algo rr
lb_kind DR
protocol TCP
sorry_server 127.0.0.1 80
real_server 172.31.0.17 80 {
weight 1
HTTP_GET {
url {
path /monitor.html
status_code 200
}
connect_timeout 1
nb_get_retry 3
delay_before_retry 1
}
}
real_server 172.31.0.27 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 1
nb_get_retry 3
delay_before_retry 1
}
}
}
TCP监测
传输层检测:TCP_CHECK
TCP_CHECK {
connect_ip <IP ADDRESS> #向当前RS的哪个IP地址发起健康状态检测请求
connect_port <PORT> #向当前RS的哪个PORT发起健康状态检测请求
bindto <IP ADDRESS> #发出健康状态检测请求时使用的源地址
bind_port <PORT> #发出健康状态检测请求时使用的源端口
connect_timeout <INTEGER> #客户端请求的超时时长, 等于haproxy的timeout server
}
范例:
virtual_server 172.31.0.188 80 {
delay_loop 6
lb_algo wrr
lb_kind DR
#persistence_timeout 120 #会话保持时间
protocol TCP
sorry_server 127.0.0.1 80
real_server 172.31.0.17 80 {
weight 1
TCP_CHECK {
connect_timeout 5
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 172.31.0.27 80 {
weight 1
TCP_CHECK {
connect_timeout 5
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
}
实战案例1:实现单主的 LVS-DR 模式
准备web服务器并使用脚本绑定VIP至web服务器lo网卡
#准备两台后端RS主机
[root@rs1 ~]# cat lvs_dr_rs.sh
#!/bin/bash
#Author:xuanlv
#Date:2021-06-13
vip=172.31.0.188
mask=‘255.255.255.255‘
dev=lo:1
rpm -q httpd &> /dev/null || yum -y install httpd &>/dev/null
service httpd start &> /dev/null && echo "The httpd Server is Ready!"
echo "<h1>`hostname`</h1>" > /var/www/html/index.html
case $1 in
start)
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
ifconfig $dev $vip netmask $mask #broadcast $vip up #route add -host $vip dev $dev
echo "The RS Server is Ready!"
;;
stop)
ifconfig $dev down
echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
echo "The RS Server is Canceled!"
;;
*)
echo "Usage: $(basename $0) start|stop"
exit 1
;;
esac
# 执行
[root@rs1 ~]# bash lvs_dr_rs.sh start
[root@rs1 ~]# bash lvs_dr_rs.sh start
#测试直接访问两台RS
配置keepalived
#ka1节点的配置
[root@ka1-centos8 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from keepalived@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id ka1.longxuan.vip
vrrp_mcast_group4 224.0.100.100
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 66
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
172.31.0.188 dev eth0 label eth0:1
}
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"
}
virtual_server 172.31.0.188 80 {
delay_loop 3
lb_algo rr
lb_kind DR
protocol TCP
sorry_server 127.0.0.1 80
real_server 172.31.0.27 80 {
weight 1
HTTP_GET { #应用层检测
url {
path /
status_code 200
}
connect_timeout 1
nb_get_retry 3
delay_before_retry 1
}
}
real_server 172.31.0.37 80 {
weight 1
TCP_CHECK { #另一台主机使用TCP检测
connect_timeout 5
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
}
#ka2节点的配置,配置和ka1基本相同,只需修改三行
[root@ka2-centos8 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from keepalived@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id ka2.longxuan.vip #修改此行
vrrp_mcast_group4 224.0.100.100
}
vrrp_instance VI_1 {
state BACKUP #修改此行
interface eth0
virtual_router_id 66
priority 80 #修改此行
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
172.31.0.188 dev eth0 label eth0:1
}
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"
}
virtual_server 172.31.0.188 80 {
delay_loop 3
lb_algo rr
lb_kind DR
protocol TCP
sorry_server 127.0.0.1 80
real_server 172.31.0.27 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 1
nb_get_retry 3
delay_before_retry 1
}
}
real_server 172.31.0.37 80 {
weight 1
TCP_CHECK {
connect_timeout 5
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
}
访问测试结果
root@long:~# curl 172.31.0.188
<h1>rs2 web</h1>
root@long:~# curl 172.31.0.188
<h1>rs1 web</h1>
安装ipvsadm
[root@ka1 ~]# yum install ipvsadm -y
# 查看ipvsadm
[root@ka1 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 172.31.0.188:80 rr
-> 172.31.0.27:80 Route 1 0 0
-> 172.31.0.37:80 Route 1 0 0
模拟故障
#第一台RS1故障,自动切换至RS2,这里一开始访问还会去RS1检测,经过检测几次发现真的检测不了,就不会往RS1调度,直接调度RS2
root@long:~# curl 172.31.0.188
<h1>rs2 web</h1>
root@long:~# curl 172.31.0.188
<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
<html><head>
<title>403 Forbidden</title>
</head><body>
<h1>Forbidden</h1>
<p>You don‘t have permission to access /index.html
on this server.</p>
</body></html>
root@long:~# curl 172.31.0.188
<h1>rs2 web</h1>
root@long:~# curl 172.31.0.188
#检查ipvsadm还有一台可以调度,说明机器不能调度后,ipvsadm会自动剔除掉
[root@ka2 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 172.31.0.188:80 rr
-> 172.31.0.37:80 Route 1 0 0
当后端两台RS都故障时,就会启用Sorry Server这台服务,前提是这台在ka1必须安装有web服务器,如果后端RS其中一台恢复了也不会访问Sorry Server服务
root@long:~# curl 172.31.0.188
Sorry Server
实战案例2:实现双主的 LVS-DR 模式
[root@ka1-centos8 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from keepalived@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id ka1 #另一个节点为ka2
vrrp_mcast_group4 224.0.100.10
}
vrrp_instance VI_1 {
state MASTER #在另一个结点上为BACKUP
interface eth0
virtual_router_id 66
priority 100 #在另一个结点上为80
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
172.31.0.188 dev eth0 label eth0:1 #指定VIP
}
}
vrrp_instance VI_2 {
state BACKUP #在另一个结点上为MASTER
interface eth0
virtual_router_id 88
priority 80 #在另一个结点上为100
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
172.31.0.200/24 dev eth0 label eth0:2 #指定VIP2
}
}
virtual_server 172.31.0.17 80 {
delay_loop 6
lb_algo rr
lb_kind DR
protocol TCP
sorry_server 127.0.0.1 80
real_server 172.31.0.27 80 { #指定RS1地址
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 172.31.0.37 80 { #指定RS2地址
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
virtual_server 172.31.0.200 80 { #指定VIP2
delay_loop 6
lb_algo rr
lb_kind DR
protocol TCP
sorry_server 127.0.0.1 80
real_server 172.31.0.27 80 { #指定RS3地址
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 172.31.0.37 80 { #指定RS4地址
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}