Keepalive之nginx调度架构

2021-12-31 16:39:21

author：JevonWei

版权声明：原创作品

单主模式Keepalive之Nginx调度

实验目的：实现Nginx调度的高可用，当一台Nginx调度器故障时，启用备用的Nginx调度，在架构中，启用了kepalive节点状态通知脚本机制，当keepalive的状态由MASTER变为BACKUP或由BACKUP变为MASTER时，以邮件通知用户，通过在通知脚本文件中定义BACKUP状态的节点启用nginx服务，是为了防止因Nginx服务故障导致节点权重降低从而变为BACKUP状态，为了避免此种现象，故在转换为BACKUP状态时需启动一次Nginx服务

实验环境

keepalive-A 172.16.253.108

keepalive-B 172.16.253.105

Nginx-A     172.16.253.108

Nginx-B     172.16.253.105

LVS-RS1     172.16.250.127

LVS-RS2     172.16.253.193

VIP         172.16.253.150

client      172.16.253.177

环境拓扑图

web服务集群

为了更好的观察实验结果，故在此将RS1和RS2的web页面内容设置不一致，以致可以更清晰的区分RS1服务端和RS2服务端

LVS-RS1

[root@LVS-RS1 ~]# systemctl restart chronyd  \\多台服务器时间同步

[root@LVS-RS1 ~]# iptables -F

[root@LVS-RS1 ~]# setenforce 0

[root@LVS-RS1 ~]# yum -y install nginx

[root@LVS-RS1 ~]# vim /usr/share/nginx/html/index.html

<h1> Web RS1 </h1>

[root@LVS-RS1 ~]# systemctl start nginx

LVS-RS2

[root@LVS-RS2 ~]# systemctl restart chronyd  \\多台服务器时间同步

[root@LVS-RS2 ~]# iptables -F

[root@LVS-RS2 ~]# setenforce 0

[root@LVS-RS2 ~]# yum -y install nginx

[root@LVS-RS2 ~]# vim /usr/share/nginx/html/index.html

<h1> Web RS2 </h1>

[root@LVS-RS2 ~]# systemctl start nginx

nginx调度集群

nginx-A

[root@nginx-A ~]# yum -y install nginx

[root@nginx-A ~]# vim /usr/share/nginx/html/index.html

</h1> sorry from keepalive-A </h1>

[root@nginx-A ~]# vim /etc/nginx/nginx.conf

http {

    upstream websrvs {

        server 172.16.250.127:80;

        server 172.16.253.193:80;

    }

}

server {

    listen       80 default_server;

    location / {

        proxy_pass http://websrvs;

    }

}

[root@nginx-A ~]# systemctl start nginx

nginx-B

[root@nginx-B ~]# yum -y install nginx

[root@nginx-B ~]# vim /usr/share/nginx/html/index.html

</h1> sorry from keepalive-B </h1>

[root@nginx-B ~]# vim /etc/nginx/nginx.conf

http {

    upstream websrvs {

        server 172.16.250.127:80;

        server 172.16.253.193:80;

    }

}

server {

    listen       80 default_server;

    location / {

        proxy_pass http://websrvs;

    }

}

[root@nginx-B ~]# systemctl start nginx

keepalive集群

keepalive-A

[root@keepalive-A ~]# yum -y install keepalived

[root@keepalive-A ~]# vim /etc/keepalived/notify.sh  \\定义状态检测通知脚本，当节点变为BACKUP时，启动nginx服务

#!/bin/bash

    #

    contact='root@localhost'  \\通知的联系者，即本地邮件服务器的通知邮件接受者

    notify() {

       local mailsubject="$(hostname) to be $1, vip floating" \\指定邮件标题

       local mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1"  \\指定邮件正文

       echo "$mailbody" | mail -s "$mailsubject" $contact  \\发送通知信息邮件到邮件接收者

    }

    case $1 in

        master)

           notify master

           ;;

        backup)

            notify backup

            systemctl start nginx

            ;;

        fault)

            notify fault

            ;;

        *)

            echo "Usage: $(basename $0) {master|backup|fault}"

            exit 1

            ;;

        esac

[root@keepaliveA ~]# vim /etc/keepalived/keepalived.conf

global_defs {

    notification_email {

        jevon@danran.com

    }

    notification_email_from ka_admin@danran.com

    smtp_server 127.0.0.1

    smtp_connect_timeout 30

    router_id keepaliveA

    vrrp_mcast_group4 224.103.5.5

}

vrrp_script chk_mant_down { \\定义检测脚本

    script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0" \\检测/etc/keepalived/down文件是否存在，若文件存在，则weight权重-10，若文件不存在，则每隔一秒检测

    interval 1 \\每隔一秒检测

    weight -10

    fall 2 \\检测两次故障则表示节点故障

    rise 1 \\检测一次正常则表示节点正常

}

vrrp_script chk_nginx { \\检测nginx服务是否运行脚本

    script "killall -0 nginx" \\-0为探测信号，当nginx进程存在时，返回值$?=0,当nginx进程不存在时，返回值$?为非零，则权重减10

    interval 2

    weight -10

    fall 2

    rise 2

}

vrrp_instance VI_A {

    state BACKUP

    interface ens33

    virtual_router_id 51

    priority 95

    advert_int 1

    authentication {

        auth_type PASS

        auth_pass qr8hQHuL

    }

    virtual_ipaddress {

    172.16.253.150/32

    notify_master "/etc/keepalived/notify.sh master"

    notify_backup "/etc/keepalived/notify.sh backup"

    notify_fault "/etc/keepalived/notify.sh fault"

    }

    track_script { \\脚本调用

        chk_mant_down

        chk_nginx \\调用chk_nginx脚本

    }

}

[root@keepalive-A ~]# systemctl start keepalived

[root@keepalive-A ~]# systemctl status keepalived

keepalive-B

[root@keepalive-B ~]# yum -y install keepalived

[root@keepalive-A ~]# vim /etc/keepalived/notify.sh  \\定义状态检测通知脚本

#!/bin/bash

    #

    contact='root@localhost'  \\通知的联系者，即本地邮件服务器的通知邮件接受者

    notify() {

       local mailsubject="$(hostname) to be $1, vip floating" \\指定邮件标题

       local mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1"  \\指定邮件正文

       echo "$mailbody" | mail -s "$mailsubject" $contact  \\发送通知信息邮件到邮件接收者

    }

    case $1 in

        master)

           notify master

           ;;

        backup)

            notify backup

            systemctl start nginx

            ;;

        fault)

            notify fault

            ;;

        *)

            echo "Usage: $(basename $0) {master|backup|fault}"

            exit 1

            ;;

        esac

[root@keepalive-B ~]# vim /etc/keepalived/keepalived.conf

global_defs {

    notification_email {

        jevon@danran.com

    }

    notification_email_from ka_admin@danran.com

    smtp_server 127.0.0.1

    smtp_connect_timeout 30

    router_id keepaliveA

    vrrp_mcast_group4 224.103.5.5

}

vrrp_script chk_mant_down { \\定义检测脚本

    script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0" \\检测/etc/keepalived/down文件是否存在，若文件存在，则weight权重-10，若文件不存在，则每隔一秒检测

    interval 1 \\每隔一秒检测

    weight -10

    fall 2 \\检测两次故障则表示节点故障

    rise 1 \\检测一次正常则表示节点正常

}

vrrp_script chk_nginx { \\检测nginx服务是否运行脚本

    script "killall -0 nginx" \\-0为探测信号，当nginx进程存在时，返回值$?=0,当nginx进程不存在时，返回值$?为非零，则权重减10

    interval 2

    weight -10

    fall 2

    rise 2

}

vrrp_instance VI_A {

    state BACKUP

    interface ens33

    virtual_router_id 51

    priority 95

    advert_int 1

    authentication {

        auth_type PASS

        auth_pass qr8hQHuL

    }

    virtual_ipaddress {

    172.16.253.150/32

    notify_master "/etc/keepalived/notify.sh master"

    notify_backup "/etc/keepalived/notify.sh backup"

    notify_fault "/etc/keepalived/notify.sh fault"

    }

    track_script { \\脚本调用

        chk_mant_down

        chk_nginx \\调用chk_nginx脚本

    }

}

[root@keepalive-B ~]# systemctl start keepalived

[root@keepalive-A ~]# systemctl start keepalived

测试

访问测试调度

[root@client ~]# for i in {1..10};do curl http://172.16.253.150;done

<h1> Web RS1 </h1>

<h1> Web RS2 </h1>

<h1> Web RS1 </h1>

<h1> Web RS2 </h1>

<h1> Web RS1 </h1>

测试检测监控Nginx服务资源的外部脚本

[root@keepalive-A ~]# ip a l

2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000

link/ether 00:0c:29:75:dc:3c brd ff:ff:ff:ff:ff:ff

    inet 172.16.253.150/32 scope global ens33

        valid_lft forever preferred_lft forever

手动停止Nginx服务，keepalive-A变为了BACKUP节点

[root@keepalive-A ~]# systemctl stop nginx

[root@keepalive-A ~]# ip a l

2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000

link/ether 00:0c:29:75:dc:3c brd ff:ff:ff:ff:ff:ff

几秒钟之后，Nginx服务再次自动启动，则Keepalive-A再次成为MASTER节点

[root@keepalive-A ~]# ip a l

2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000

link/ether 00:0c:29:75:dc:3c brd ff:ff:ff:ff:ff:ff

    inet 172.16.253.150/32 scope global ens33

        valid_lft forever preferred_lft forever

当nginx的80端口被占用时，Nginx服务停止之后不会再次启动，则keepalive-A变为BACKUP

[root@keepalive-A ~]# yum -y install httpd

[root@keepalive-A ~]# systemctl stop nginx && systemctl start httpd \\停止nginx的同时启动httpd

[root@keepalive-A ~]# ip a l

2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000

link/ether 00:0c:29:75:dc:3c brd ff:ff:ff:ff:ff:ff

[root@keepalive-A ~]# systemctl status keepalived.service

Sep 05 21:21:14 danran.com Keepalived_vrrp[24153]: VRRP_Script(chk_nginx) failed

在HTTPD进程停止之后，80端口被释放，则keepalive-A上的Nginx也不会再次重启

双主模式的Nginx调度(一般不做会话绑定)

双主模式的目的是为了访问web访问路径的单点故障，当一个web虚拟路由故障时，可使用另外一个web路径访问，最终是为了实现集群的高可用性

实验环境

keepalive-A 172.16.253.108

keepalive-B 172.16.253.105

Nginx-A     172.16.253.108

Nginx-B     172.16.253.105

LVS-RS1     172.16.250.127

LVS-RS2     172.16.253.193

VI-A        172.16.253.150

VI-B        172.16.253.140

client      172.16.253.177