Centos7使用kubeadm 安装多主高可用kubernets:v.1.11集群

2022-10-27 11:24:22

实验环境介绍：

本次实验环境是5个节点 3台master 2台node节点：

k8smaster01 192.168.111.128 软件：etcd k8smaster haproxy keepalived

k8smaster02 192.168.111.129 软件：etcd k8smaster haproxy keepalived

k8smaster03 192.168.111.130 软件：etcd k8smaster haproxy keepalived

k8snode01 192.168.111.131  软件：k8snode

k8snode02 192.168.111.132  软件：k8snode

VIP： 192.168.111.100

系统优化（在所有节点上操作）

关闭防火墙

systemctl stop firewalld.service

systemctl disable firewalld.service

关闭SELINUX和swap，优化内核参数

 sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config

 setenforce 0

 # 临时关闭swap
 # 永久关闭 注释/etc/fstab文件里swap相关的行
 swapoff -a

# 配置转发相关参数，否则可能会出错
cat <<EOF >  /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
vm.swappiness=0
EOF
sysctl --system

# 加载ipvs相关内核模块
# 如果重新开机，需要重新加载
modprobe ip_vs
modprobe ip_vs_rr
modprobe ip_vs_wrr
modprobe ip_vs_sh
modprobe nf_conntrack_ipv4
lsmod | grep ip_vs

配置yum源

wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo

mv /etc/yum.repos.d/epel.repo /etc/yum.repos.d/epel.repo.backup

mv /etc/yum.repos.d/epel-testing.repo /etc/yum.repos.d/epel-testing.repo.backup

wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo

yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

cat <<EOF > /etc/yum.repos.d/kubernetes.repo

[kubernetes]

name=Kubernetes

baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/

enabled=

gpgcheck=

repo_gpgcheck=

gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg

EOF

yum clean all && yum makecache 

sudo yum install -y yum-utils device-mapper-persistent-data lvm2

配置hosts解析

192.168.111.128 k8smaster01

192.168.111.129 k8smaster02

192.168.111.130 k8smaster03

192.168.111.131 k8snode01

192.168.111.132 k8snode02

安装docker

v1.11.1版本推荐使用docker v17.03,v1.11,v1.12,v1.13, 也可以使用，再高版本官网不推荐使用，但是可以忽略。

这里安装18.06.0-ce

yum -y install docker-ce

systemctl enable docker && systemctl restart docker

安装 kubeadm, kubelet 和 kubectl(所有节点)

yum install -y kubelet kubeadm kubectl ipvsadm

systemctl enable kubelet && systemctl start kubelet

配置haproxy代理和keepalived（如下操作在所有master节点上操作）

# 拉取haproxy镜像

docker pull haproxy:1.7.-alpine

cat >/etc/haproxy/haproxy.cfg<<EOF

global

  log 127.0.0.1 local0 err

  maxconn

  uid

  gid

  #daemon

  nbproc

  pidfile haproxy.pid

defaults

  mode http

  log 127.0.0.1 local0 err

  maxconn

  retries

  timeout connect 5s

  timeout client 30s

  timeout server 30s

  timeout check 2s

listen admin_stats

  mode http

  bind 0.0.0.0:

  log 127.0.0.1 local0 err

  stats refresh 30s

  stats uri     /haproxy-status

  stats realm   Haproxy\ Statistics

  stats auth    will:will

  stats hide-version

  stats admin if TRUE

frontend k8s-https

  bind 0.0.0.0:

  mode tcp

  #maxconn

  default_backend k8s-https

backend k8s-https

  mode tcp

  balance roundrobin

  server k8smaster01 192.168.111.128: weight  maxconn  check inter  rise  fall

  server k8smaster02 192.168.111.129: weight  maxconn  check inter  rise  fall

  server k8smaster03 192.168.111.130: weight  maxconn  check inter  rise  fall

EOF

# 启动haproxy

docker run -d --name my-haproxy \

-v /etc/haproxy:/usr/local/etc/haproxy:ro \

-p : \

-p : \

--restart always \

haproxy:1.7.-alpine

# 拉取keepalived镜像

docker pull osixia/keepalived:1.4.

# 启动

# 载入内核相关模块

lsmod | grep ip_vs

modprobe ip_vs

# 启动keepalived

# ens33为本次实验192.168.111./24网段的所在网卡

docker run --net=host --cap-add=NET_ADMIN \

-e KEEPALIVED_INTERFACE=ens33 \

-e KEEPALIVED_VIRTUAL_IPS="#PYTHON2BASH:['192.168.111.100']" \

-e KEEPALIVED_UNICAST_PEERS="#PYTHON2BASH:['192.168.111.128','192.168.111.129','192.168.111.130']" \

-e KEEPALIVED_PASSWORD=hello \

--name k8s-keepalived \

--restart always \

-d osixia/keepalived:1.4.

# 此时会配置 192.168.111.100 到其中一台机器

# ping测试

ping  192.168.111.100

# 如果失败后清理后，重新实验

#docker rm -f k8s-keepalived

#ip a del 192.168.111.100/ dev ens33

配置kubelet(所有节点操作)

# 配置kubelet使用国内pause镜像

# 配置kubelet的cgroups

cat >/etc/sysconfig/kubelet<<EOF

KUBELET_EXTRA_ARGS="--cgroup-driver=cgroupfs  --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause-amd64:3.1"

EOF

# 启动

systemctl daemon-reload

systemctl enable kubelet && systemctl restart kubelet

配置k8smaster01(192.168.111.128上操作)

cd /etc/kubernetes

# 生成配置文件

cat >kubeadm-master.config<<EOF

apiVersion: kubeadm.k8s.io/v1alpha2

kind: MasterConfiguration

kubernetesVersion: v1.11.1

imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers

apiServerCertSANs:

- "k8smaster01"

- "k8smaster02"

- "k8smaster03"

- "192.168.111.128"

- "192.168.111.129"

- "192.168.111.130"

- "192.168.111.100"

- "127.0.0.1"

api:

  advertiseAddress: 192.168.111.128

  controlPlaneEndpoint: 192.168.111.100:

etcd:

  local:

    extraArgs:

      listen-client-urls: "https://127.0.0.1:2379,https://192.168.111.128:2379"

      advertise-client-urls: "https://192.168.111.128:2379"

      listen-peer-urls: "https://192.168.111.128:2380"

      initial-advertise-peer-urls: "https://192.168.111.128:2380"

      initial-cluster: "k8smaster01=https://192.168.111.128:2380"

    serverCertSANs:

      - k8smaster01

      - 192.168.111.128

    peerCertSANs:

      - k8smaster01

      - 192.168.111.128

controllerManagerExtraArgs:

  node-monitor-grace-period: 10s

  pod-eviction-timeout: 10s

networking:

  podSubnet: 10.244.0.0/

kubeProxy:

  config:

    mode: ipvs

    # mode: iptables

EOF

# 提前拉取镜像

# 如果执行失败 可以多次执行

kubeadm config images pull --config kubeadm-master.config

# 初始化

# 注意保存返回的 join 命令

kubeadm init --config kubeadm-master.config

# 初始化失败时使用

#kubeadm reset

# 将ca相关文件传至其他master节点
cd /etc/kubernetes/pki/

USER=root

CONTROL_PLANE_IPS="k8smaster02 k8smaster03"

for host in ${CONTROL_PLANE_IPS}; do
　　 ssh "${USER}"@$host "mkdir -p /etc/kubernetes/pki/etcd"

    scp ca.crt ca.key  sa.key  sa.pub front-proxy-ca.crt front-proxy-ca.key  "${USER}"@$host:/etc/kubernetes/pki/

    scp etcd/ca.crt etcd/ca.key "${USER}"@$host:/etc/kubernetes/pki/etcd/
　　scp ../admin.conf "${USER}"@$host:/etc/kubernetes/

done

kubeadm init失败解决：

将阿里云image tag成官方的image，即可解决init失败问题。(v1.11.0有此问题)

docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver-amd64:v1.11.1 k8s.gcr.io/kube-apiserver-amd64:v1.11.1
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy-amd64:v1.11.1 k8s.gcr.io/kube-proxy-amd64:v1.11.1

docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/etcd-amd64:3.2. k8s.gcr.io/etcd-amd64:3.2.

docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler-amd64:v1.11.1 k8s.gcr.io/kube-scheduler-amd64:v1.11.1

docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager-amd64:v1.11.1 k8s.gcr.io/kube-controller-manager-amd64:v1.11.1

docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:1.1. k8s.gcr.io/coredns:1.1.

docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/pause-amd64:3.1 k8s.gcr.io/pause-amd64:3.1

docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.1 k8s.gcr.io/pause:3.1

配置k8smaster02(192.168.111.129上操作)

cd /etc/kubernetes

# 生成配置文件

cat >kubeadm-master.config<<EOF

apiVersion: kubeadm.k8s.io/v1alpha2

kind: MasterConfiguration

kubernetesVersion: v1.11.1

imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers

apiServerCertSANs:

- "k8smaster01"

- "k8smaster02"

- "k8smaster03"

- "192.168.111.128"

- "192.168.111.129"

- "192.168.111.130"

- "192.168.111.100"

- "127.0.0.1"

api:

  advertiseAddress: 192.168.111.129

  controlPlaneEndpoint: 192.168.111.100:

etcd:

  local:

    extraArgs:

      listen-client-urls: "https://127.0.0.1:2379,https://192.168.111.129:2379"

      advertise-client-urls: "https://192.168.111.129:2379"

      listen-peer-urls: "https://192.168.111.129:2380"

      initial-advertise-peer-urls: "https://192.168.111.129:2380"

      initial-cluster: "k8smaster01=https://192.168.111.128:2380,k8smaster02=https://192.168.111.129:2380"

      initial-cluster-state: existing

    serverCertSANs:

      - k8smaster02

      - 192.168.111.129

    peerCertSANs:

      - k8smaster02

      - 192.168.111.129

controllerManagerExtraArgs:

  node-monitor-grace-period: 10s

  pod-eviction-timeout: 10s

networking:

  podSubnet: 10.244.0.0/

kubeProxy:

  config:

    mode: ipvs

    # mode: iptables

EOF

# 配置kubelet

kubeadm alpha phase certs all --config kubeadm-master.config

kubeadm alpha phase kubelet config write-to-disk --config kubeadm-master.config

kubeadm alpha phase kubelet write-env-file --config kubeadm-master.config

kubeadm alpha phase kubeconfig kubelet --config kubeadm-master.config

systemctl restart kubelet

# 添加etcd到集群中

export KUBECONFIG=/etc/kubernetes/admin.conf

kubectl exec -n kube-system etcd-k8smaster01 -- etcdctl --ca-file /etc/kubernetes/pki/etcd/ca.crt --cert-file /etc/kubernetes/pki/etcd/peer.crt --key-file /etc/kubernetes/pki/etcd/peer.key --endpoints=https://192.168.111.128:2379 member add k8smaster02 https://192.168.111.129:2380

kubeadm alpha phase etcd local --config kubeadm-master.config

# 提前拉取镜像

kubeadm config images pull --config kubeadm-master.config

# 部署

kubeadm alpha phase kubeconfig all --config kubeadm-master.config

kubeadm alpha phase controlplane all --config kubeadm-master.config

kubeadm alpha phase mark-master --config kubeadm-master.config

配置k8smaster03(192.168.111.130上操作)

cd /etc/kubernetes

# 生成配置文件

cat >kubeadm-master.config<<EOF

apiVersion: kubeadm.k8s.io/v1alpha2

kind: MasterConfiguration

kubernetesVersion: v1.11.1

imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers

apiServerCertSANs:

- "k8smaster01"

- "k8smaster02"

- "k8smaster03"

- "192.168.111.128"

- "192.168.111.129"

- "192.168.111.130"

- "192.168.111.100"

- "127.0.0.1"

api:

  advertiseAddress: 192.168.111.130

  controlPlaneEndpoint: 192.168.111.100:

etcd:

  local:

    extraArgs:

      listen-client-urls: "https://127.0.0.1:2379,https://192.168.111.130:2379"

      advertise-client-urls: "https://192.168.111.130:2379"

      listen-peer-urls: "https://192.168.111.130:2380"

      initial-advertise-peer-urls: "https://192.168.111.130:2380"

      initial-cluster: "k8smaster01=https://192.168.111.128:2380,k8smaster02=https://192.168.111.129:2380,k8smaster03=https://192.168.111.130:2380"

      initial-cluster-state: existing

    serverCertSANs:

      - k8smaster03

      - 192.168.111.130

    peerCertSANs:

      - k8smaster03

      - 192.168.111.130

controllerManagerExtraArgs:

  node-monitor-grace-period: 10s

  pod-eviction-timeout: 10s

networking:

  podSubnet: 10.244.0.0/

kubeProxy:

  config:

    mode: ipvs

    # mode: iptables

EOF

# 配置kubelet

kubeadm alpha phase certs all --config kubeadm-master.config

kubeadm alpha phase kubelet config write-to-disk --config kubeadm-master.config

kubeadm alpha phase kubelet write-env-file --config kubeadm-master.config

kubeadm alpha phase kubeconfig kubelet --config kubeadm-master.config

systemctl restart kubelet

# 添加etcd到集群中

KUBECONFIG=/etc/kubernetes/admin.conf

kubectl exec -n kube-system etcd-k8smaster01 -- etcdctl --ca-file /etc/kubernetes/pki/etcd/ca.crt --cert-file /etc/kubernetes/pki/etcd/peer.crt --key-file /etc/kubernetes/pki/etcd/peer.key --endpoints=https://192.168.111.128:2379 member add k8smaster03 https://192.168.111.130:2380

kubeadm alpha phase etcd local --config kubeadm-master.config

# 提前拉取镜像

kubeadm config images pull --config kubeadm-master.config

# 部署

kubeadm alpha phase kubeconfig all --config kubeadm-master.config

kubeadm alpha phase controlplane all --config kubeadm-master.config

kubeadm alpha phase mark-master --config kubeadm-master.config

配置使用kubectl (master 任意节点执行)

rm -rf $HOME/.kube

mkdir -p $HOME/.kube

sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config

sudo chown $(id -u):$(id -g) $HOME/.kube/config
# 查看node节点
　kubectl get nodes
 
# 只有网络插件也安装配置完成之后，才能会显示为ready状态
# 设置master允许部署应用pod，参与工作负载，现在可以部署其他系统组件

配置使用网络插件（任意master节点上操作）

# 下载配置

cd /etc/kubernetes

mkdir flannel && cd flannel

wget https://raw.githubusercontent.com/coreos/flannel/v0.10.0/Documentation/kube-flannel.yml

# 修改配置

# 此处的ip配置要与上面kubeadm的pod-network一致

  net-conf.json: |

    {

      "Network": "10.244.0.0/16",

      "Backend": {

        "Type": "vxlan"

      }

    }

# 修改镜像

image: registry.cn-shanghai.aliyuncs.com/gcr-k8s/flannel:v0.10.0-amd64

# 如果Node有多个网卡的话，参考flannel issues ，

# https://github.com/kubernetes/kubernetes/issues/39701

# 目前需要在kube-flannel.yml中使用--iface参数指定集群主机内网网卡的名称，

# 否则可能会出现dns无法解析。容器无法通信的情况，需要将kube-flannel.yml下载到本地，

# flanneld启动参数加上--iface=<iface-name>

    containers:

      - name: kube-flannel

        image: registry.cn-shanghai.aliyuncs.com/gcr-k8s/flannel:v0.10.0-amd64

        command:

        - /opt/bin/flanneld

        args:

        - --ip-masq

        - --kube-subnet-mgr

        - --iface=ens33

# 启动

kubectl apply -f kube-flannel.yml

# 查看

kubectl get pods --namespace kube-system

kubectl get svc --namespace kube-system

配置node节点加入集群（所有的node节点上操作）

以下上master生成的，与你环境可能不符合  
kubeadm join 192.168.111.100: --token uf9oul.7k4csgxe5p7upvdb --discovery-token-ca-cert-hash sha256:36bc173b46eb0545fc30dd5db2d27dab70a257bd406fd791647d991a69454595

node节点报错处理办法：

tail -f /var/log/message
Jul 19 07:52:21 localhost kubelet: E0726 07:52:21.336281   10018 summary.go:102] Failed to get system container stats for "/system.slice/kubelet.service": failed to get cgroup stats for "/system.slice/kubelet.service": failed to get container info for "/system.slice/kubelet.service": unknown container "/system.slice/kubelet.service"

在kubelet配置文件追加以下配置

/etc/sysconfig/kubelet

# Append configuration in Kubelet

--runtime-cgroups=/systemd/system.slice --kubelet-cgroups=/systemd/system.slice

这样一个集群环境配置完成里，其余的是自己添加附件吧。

上一篇：springboot配置SSL自签名证书


下一篇：Oracle查询重复数据并删除，只保留一条记录