day02 K8s二进制安装
1、环境初始化
1.1、 软件系统版本
| 软件 | 版本 |
|---|---|
| CentOS | CentOS Linux release 7.7.1908 (Core) |
| Docker | 19.03.12 |
| Kubernetes | V1.18.8 |
| Flannel | V0.13.0 |
| Kernel-lm | 4.18+ |
| Kernel-lm-devel | 4.18+ |
| Haproxy | 1.5.18 |
| Keepalived | 1.3.5 |
1.2、节点规划
| hostname | ip |
|---|---|
| k8s-master-01 | 192.168.12.101 |
| k8s-master-02 | 192.168.12.102 |
| k8s-master-03 | 192.168.12.103 |
| k8s-node-01 | 192.168.12.104 |
| k8s-node-02 | 192.168.12.105 |
1.3、修改主机名称
hostnamectl set-hostname k8s-master-01
hostnamectl set-hostname k8s-master-02
hostnamectl set-hostname k8s-master-03
hostnamectl set-hostname k8s-node-01
hostnamectl set-hostname k8s-node-02
1.4、配置HOSTS解析(5台都要执行)
cat >> /etc/hosts <<EOF
192.168.12.101 k8s-master-01 m1
192.168.12.102 k8s-master-02 m2
192.168.12.103 k8s-master-03 m3
192.168.12.104 k8s-node-01 n1
192.168.12.105 k8s-node-02 n2
EOF
1.5、集群各节点免密登录(只有master节点做)
[root@k8s-master-01 ~]# ssh-keygen
[root@k8s-master-01 ~]# for i in 101 102 103 104 105; do ssh-copy-id -i ~/.ssh/id_rsa.pub root@192.168.12.$i; done
1.6、关闭selinux
[root@k8s-master-01 ~]# for i in 101 102 103 104 105; do setenforce 0; done
1.7、关闭防火墙
[root@k8s-master-01 ~]# for i in 101 102 103 104 105; do systemctl disable --now firewalld; done
1.8、关闭swap分区
swapoff -a
sed -i.bak 's/^.*centos-swap/#&/g' /etc/fstab
echo 'KUBELET_EXTRA_ARGS="--fail-swap-on=false"' > /etc/sysconfig/kubelet
1.9、配置国内yum源
mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup
curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
# 刷新缓存
yum makecache
# 更新系统
yum update -y --exclud=kernel*
1.10、升级系统内核
wget https://elrepo.org/linux/kernel/el7/x86_64/RPMS/kernel-lt-4.4.245-1.el7.elrepo.x86_64.rpm
wget https://elrepo.org/linux/kernel/el7/x86_64/RPMS/kernel-lt-devel-4.4.245-1.el7.elrepo.x86_64.rpm
yum localinstall -y kernel-lt*
grub2-set-default 0 && grub2-mkconfig -o /etc/grub2.cfg
grubby --default-kernel
reboot
1.11 、安装ipvs
yum install -y conntrack-tools ipvsadm ipset conntrack libseccomp
# 加载IPVS模块
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
ipvs_modules="ip_vs ip_vs_lc ip_vs_wlc ip_vs_rr ip_vs_wrr ip_vs_lblc ip_vs_lblcr ip_vs_dh ip_vs_sh ip_vs_fo ip_vs_nq ip_vs_sed ip_vs_ftp nf_conntrack"
for kernel_module in \${ipvs_modules}; do
/sbin/modinfo -F filename \${kernel_module} > /dev/null 2>&1
if [ $? -eq 0 ]; then
/sbin/modprobe \${kernel_module}
fi
done
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep ip_vs
1.12、设置内核参数
cat > /etc/sysctl.d/k8s.conf << EOF
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp.keepaliv.probes = 3
net.ipv4.tcp_keepalive_intvl = 15
net.ipv4.tcp.max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp.max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.top_timestamps = 0
net.core.somaxconn = 16384
EOF
# 立即生效
sysctl --system
1.13 安装基础软件
yum install wget expect vim net-tools ntp bash-completion ipvsadm ipset jq iptables conntrack sysstat libseccomp -y
1.14、安装docker
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum install docker-ce-19.03.9 -y
sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json <<-'EOF'
{
"registry-mirrors": ["https://8mh75mhz.mirror.aliyuncs.com"]
}
EOF
sudo systemctl daemon-reload ; systemctl restart docker;systemctl enable --now docker.service
1、流程
1、master
高可用
kube-apiserver
kube-controller-manager
kube-scheduler
kubelet
kube-proxy
etcd
cronDNS
flannel
2、node
kubelet
kube-proxy
cronDNS
flannel
3、免密登录
[root@k8s-master-01 ~]# ssh-keygen
[root@k8s-master-01 ~]# for i in 101 102 103 104 105 106 107 108; do ssh-copy-id -i ~/.ssh/id_rsa.pub root@192.168.11.$i; done
2、部署生成证书工具
下载路径：https://github.com/cloudflare/cfssl/releases
# 下载
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
# 设置执行权限
chmod +x cfssl_1.6.0_linux_amd64
chmod +x cfssljson_1.6.0_linux_amd64
# 移动到/usr/local/bin
mv cfssl_1.6.0_linux_amd64 /usr/local/bin/cfssl
mv cfssljson_1.6.0_linux_amd64 /usr/local/bin/cfssljson
3、部署Etcd
3.1 创建集群根证书
mkdir -p /opt/cert/ca
cat > /opt/cert/ca/ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "8760h"
},
"profiles": {
"kubernetes": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "8760h"
}
}
}
}
EOF
3.1、创建CA证书
cat > /opt/cert/ca/ca-csr.json << EOF
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names":[{
"C": "CN",
"ST": "ShangHai",
"L": "ShangHai"
}]
}
EOF
# 生成根证书
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
3.2、创建Etcd应用证书
mkdir -p /opt/cert/etcd
cd /opt/cert/etcd
cat > etcd-csr.json << EOF
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"192.168.11.101",
"192.168.11.102",
"192.168.11.103",
"192.168.11.104",
"192.168.11.105"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "ShangHai",
"L": "ShangHai"
}
]
}
EOF
# 生成证书
[root@k8s-master-01 etcd]# cfssl gencert -ca=../ca/ca.pem -ca-key=../ca/ca-key.pem -config=../ca/ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcd
3.3、分发证书（需要改下）
for ip in m1 m2 m3
do
ssh root@${ip} "mkdir -pv /etc/etcd/ssl"
scp ../ca/ca*.pem root@${ip}:/etc/etcd/ssl
scp ./etcd*.pem root@${ip}:/etc/etcd/ssl
done
3.4、部署etcd
# 下载ETCD安装包cd /optmkdir datacd data/wget https://mirrors.huaweicloud.com/etcd/v3.3.24/etcd-v3.3.24-linux-amd64.tar.gz# 解压[root@k8s-master-01 data]# tar xf etcd-v3.3.24-linux-amd64.tar.gz # 分发至其他节点for i in m1 m2 m3doscp ./etcd-v3.3.24-linux-amd64/etcd* root@$i:/usr/local/bin/done# 查看ETCD安装是否成功[root@k8s-master-01 data]# etcd --versionetcd Version: 3.3.24Git SHA: bdd57848dGo Version: go1.12.17Go OS/Arch: linux/amd64
3.5、注册ETCD服务（需要修改）（三台master节点都要执行）
mkdir -pv /etc/kubernetes/conf/etcd
ETCD_NAME=`hostname`
INTERNAL_IP=`hostname -i`
INITIAL_CLUSTER=k8s-master-01=https://192.168.12.101:2380,k8s-master-02=https://192.168.12.102:2380,k8s-master-03=https://192.168.12.103:2380
cat << EOF | sudo tee /usr/lib/systemd/system/etcd.service
[Unit]
Description=etcd
Documentation=https://github.com/coreos
[Service]
ExecStart=/usr/local/bin/etcd \\
--name ${ETCD_NAME} \\
--cert-file=/etc/etcd/ssl/etcd.pem \\
--key-file=/etc/etcd/ssl/etcd-key.pem \\
--peer-cert-file=/etc/etcd/ssl/etcd.pem \\
--peer-key-file=/etc/etcd/ssl/etcd-key.pem \\
--trusted-ca-file=/etc/etcd/ssl/ca.pem \\
--peer-trusted-ca-file=/etc/etcd/ssl/ca.pem \\
--peer-client-cert-auth \\
--client-cert-auth \\
--initial-advertise-peer-urls https://${INTERNAL_IP}:2380 \\
--listen-peer-urls https://${INTERNAL_IP}:2380 \\
--listen-client-urls https://${INTERNAL_IP}:2379,https://127.0.0.1:2379 \\
--advertise-client-urls https://${INTERNAL_IP}:2379 \\
--initial-cluster-token etcd-cluster \\
--initial-cluster ${INITIAL_CLUSTER} \\
--initial-cluster-state new \\
--data-dir=/var/lib/etcd
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
3.6、启动（三台master节点都要启动）
systemctl daemon-reload
systemctl enable --now etcd
systemctl status etcd
tail -f /var/log/messages
3.7、测试ETCD集群（需要改）
ETCDCTL_API=3 etcdctl \
--cacert=/etc/etcd/ssl/etcd.pem \
--cert=/etc/etcd/ssl/etcd.pem \
--key=/etc/etcd/ssl/etcd-key.pem \
--endpoints="https://192.168.12.101:2379,https://192.168.12.102:2379,https://192.168.12.103:2379" \
endpoint status --write-out='table'
ETCDCTL_API=3 etcdctl \
--cacert=/etc/etcd/ssl/etcd.pem \
--cert=/etc/etcd/ssl/etcd.pem \
--key=/etc/etcd/ssl/etcd-key.pem \
--endpoints="https://192.168.12.101:2379,https://192.168.12.102:2379,https://192.168.12.103:2379" \
member list --write-out='table'
4、Master节点
kube-apiserver
kube-controller-manager
kube-scheduler
kubelet
kube-proxyetcd
cronDNS
flannel
4.1、创建证书
- 创建CA证书
[root@k8s-master-01 ~]# mkdir /opt/cert/k8s
[root@k8s-master-01 ~]# cd /opt/cert/k8s
cat > ca-config.json << EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF
# 请求文件
cat > ca-csr.json << EOF
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "ShangHai",
"ST": "ShangHai"
}
]
}
EOF
# 生成ca证书
[root@k8s-master-01 k8s]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
- 生成apiserver的应用证书
cat > server-csr.json << EOF
{
"CN": "kubernetes",
"hosts": [
"127.0.0.1",
"192.168.11.101",
"192.168.11.102",
"192.168.11.103",
"192.168.11.104",
"192.168.11.105",
"10.96.0.1",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "ShangHai",
"ST": "ShangHai"
}
]
}
EOF
[root@k8s-master-01 k8s]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server
- kube-controller-manager证书
cat > kube-controller-manager-csr.json << EOF
{
"CN": "system:kube-controller-manager",
"hosts": [
"127.0.0.1",
"192.168.11.101",
"192.168.11.102",
"192.168.11.103",
"192.168.11.104",
"192.168.11.105",
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "system:kube-controller-manager",
"OU": "System"
}
]
}
EOF
[root@k8s-master-01 k8s]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager
- kube-scheduler证书
cat > kube-scheduler-csr.json << EOF
{
"CN": "system:kube-scheduler",
"hosts": [
"127.0.0.1",
"192.168.11.101",
"192.168.11.102",
"192.168.11.103",
"192.168.11.104",
"192.168.11.105",
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "system:kube-scheduler",
"OU": "System"
}
]
}
EOF
[root@k8s-master-01 k8s]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler
- kube-proxy证书
cat > kube-proxy-csr.json << EOF
{
"CN":"system:kube-proxy",
"hosts":[],
"key":{
"algo":"rsa",
"size":2048
},
"names":[
{
"C":"CN",
"L":"BeiJing",
"ST":"BeiJing",
"O":"system:kube-proxy",
"OU":"System"
}
]
}
EOF
[root@k8s-master-01 k8s]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
- 管理员证书
cat > admin-csr.json << EOF
{
"CN":"admin",
"key":{
"algo":"rsa",
"size":2048
},
"names":[
{
"C":"CN",
"L":"BeiJing",
"ST":"BeiJing",
"O":"system:masters",
"OU":"System"
}
]
}
EOF
[root@k8s-master-01 k8s]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin
- 颁发证书
mkdir -pv /etc/kubernetes/ssl
cp -p ./{ca*pem,server*pem,kube-controller-manager*pem,kube-scheduler*.pem,kube-proxy*pem,admin*.pem} /etc/kubernetes/ssl
for i in m1 m2 m3; do
ssh root@$i "mkdir -pv /etc/kubernetes/ssl"
scp /etc/kubernetes/ssl/* root@$i:/etc/kubernetes/ssl
done
4.2、创建证书配置文件
4.2.1、创建kube-controller-manager.kubeconfig文件
export KUBE_APISERVER="https://192.168.11.105:8443"
# 设置集群参数
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kube-controller-manager.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials "kube-controller-manager" \
--client-certificate=/etc/kubernetes/ssl/kube-controller-manager.pem \
--client-key=/etc/kubernetes/ssl/kube-controller-manager-key.pem \
--embed-certs=true \
--kubeconfig=kube-controller-manager.kubeconfig
# 设置上下文参数（在上下文参数中将集群参数和用户参数关联起来）
kubectl config set-context default \
--cluster=kubernetes \
--user="kube-controller-manager" \
--kubeconfig=kube-controller-manager.kubeconfig
# 配置默认上下文
kubectl config use-context default --kubeconfig=kube-controller-manager.kubeconfig
4.2.2、创建kube-scheduler.kubeconfig文件
export KUBE_APISERVER="https://192.168.11.105:8443"
# 设置集群参数
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kube-scheduler.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials "kube-scheduler" \
--client-certificate=/etc/kubernetes/ssl/kube-scheduler.pem \
--client-key=/etc/kubernetes/ssl/kube-scheduler-key.pem \
--embed-certs=true \
--kubeconfig=kube-scheduler.kubeconfig
# 设置上下文参数（在上下文参数中将集群参数和用户参数关联起来）
kubectl config set-context default \
--cluster=kubernetes \
--user="kube-scheduler" \
--kubeconfig=kube-scheduler.kubeconfig
# 配置默认上下文
kubectl config use-context default --kubeconfig=kube-scheduler.kubeconfig
4.2.3、创建kube-proxy.kubeconfig文件
export KUBE_APISERVER="https://192.168.11.109:8443"
# 设置集群参数
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kube-proxy.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials "kube-proxy" \
--client-certificate=/etc/kubernetes/ssl/kube-proxy.pem \
--client-key=/etc/kubernetes/ssl/kube-proxy-key.pem \
--embed-certs=true \
--kubeconfig=kube-proxy.kubeconfig
# 设置上下文参数（在上下文参数中将集群参数和用户参数关联起来）
kubectl config set-context default \
--cluster=kubernetes \
--user="kube-proxy" \
--kubeconfig=kube-proxy.kubeconfig
# 配置默认上下文
kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
4.2.4、创建admin.kubeconfig文件
export KUBE_APISERVER="https://192.168.11.109:8443"
# 设置集群参数
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=admin.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials "admin" \
--client-certificate=/etc/kubernetes/ssl/admin.pem \
--client-key=/etc/kubernetes/ssl/admin-key.pem \
--embed-certs=true \
--kubeconfig=admin.kubeconfig
# 设置上下文参数（在上下文参数中将集群参数和用户参数关联起来）
kubectl config set-context default \
--cluster=kubernetes \
--user="admin" \
--kubeconfig=admin.kubeconfig
# 配置默认上下文
kubectl config use-context default --kubeconfig=admin.kubeconfig
4.2.5、创建TLS Bootstrapping集群配置文件
# 必须要用自己机器创建的Token
TLS_BOOTSTRAPPING_TOKEN=`head -c 16 /dev/urandom | od -An -t x | tr -d ' '`
cat > token.csv << EOF
${TLS_BOOTSTRAPPING_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF
export KUBE_APISERVER="https://192.168.11.109:8443"
# 设置集群参数
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kubelet-bootstrap.kubeconfig
# 设置客户端认证参数,此处token必须用上叙token.csv中的token
kubectl config set-credentials "kubelet-bootstrap" \
--token=${TLS_BOOTSTRAPPING_TOKEN} \
--kubeconfig=kubelet-bootstrap.kubeconfig
# 设置上下文参数（在上下文参数中将集群参数和用户参数关联起来）
kubectl config set-context default \
--cluster=kubernetes \
--user="kubelet-bootstrap" \
--kubeconfig=kubelet-bootstrap.kubeconfig
# 配置默认上下文
kubectl config use-context default --kubeconfig=kubelet-bootstrap.kubeconfig
4.2.6、分发集群配置文件
for i in m1 m2 m3;do ssh root@$i "mkdir -p /etc/kubernetes/cfg"; scp token.csv kube-scheduler.kubeconfig kube-controller-manager.kubeconfig admin.kubeconfig kube-proxy.kubeconfig kubelet-bootstrap.kubeconfig root@$i:/etc/kubernetes/cfg;done
4.3、安装kube-apiserver
4.3.1、下载kube-apiserver的安装包
# 下载安装包
wget https://dl.k8s.io/v1.18.8/kubernetes-server-linux-amd64.tar.gz
# 解压
[root@k8s-master-01 data]# tar -xf kubernetes-server-linux-amd64.tar.gz
# 分发安装包
for i in m1 m2 m3; do scp kube-apiserver kube-controller-manager kube-scheduler kubectl kube-proxy kubelet root@$i:/usr/local/bin/; done
4.3.2、编写apiserver的配置文件
KUBE_APISERVER_IP=`hostname -i`
cat > /etc/kubernetes/cfg/kube-apiserver.conf << EOF
KUBE_APISERVER_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/var/log/kubernetes \\
--advertise-address=${KUBE_APISERVER_IP} \\
--default-not-ready-toleration-seconds=360 \\
--default-unreachable-toleration-seconds=360 \\
--max-mutating-requests-inflight=2000 \\
--max-requests-inflight=4000 \\
--default-watch-cache-size=200 \\
--delete-collection-workers=2 \\
--bind-address=0.0.0.0 \\
--secure-port=6443 \\
--allow-privileged=true \\
--service-cluster-ip-range=10.96.0.0/16 \\
--service-node-port-range=30000-60000 \\
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \\
--authorization-mode=RBAC,Node \\
--enable-bootstrap-token-auth=true \\
--token-auth-file=/etc/kubernetes/cfg/token.csv \\
--kubelet-client-certificate=/etc/kubernetes/ssl/server.pem \\
--kubelet-client-key=/etc/kubernetes/ssl/server-key.pem \\
--tls-cert-file=/etc/kubernetes/ssl/server.pem \\
--tls-private-key-file=/etc/kubernetes/ssl/server-key.pem \\
--client-ca-file=/etc/kubernetes/ssl/ca.pem \\
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \\
--audit-log-maxage=30 \\
--audit-log-maxbackup=3 \\
--audit-log-maxsize=100 \\
--audit-log-path=/var/log/kubernetes/k8s-audit.log \\
--etcd-servers=https://192.168.11.101:2379,https://192.168.11.102:2379,https://192.168.11.103:2379 \\
--etcd-cafile=/etc/etcd/ssl/ca.pem \\
--etcd-certfile=/etc/etcd/ssl/etcd.pem \\
--etcd-keyfile=/etc/etcd/ssl/etcd-key.pem"
EOF
4.3.3、注册apiserver的服务
cat > /usr/lib/systemd/system/kube-apiserver.service << EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-apiserver.conf
ExecStart=/usr/local/bin/kube-apiserver \$KUBE_APISERVER_OPTS
Restart=on-failure
RestartSec=10
Type=notify
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
4.3.4、启动apiserver服务
# 创建kubernetes日志目录
mkdir -p /var/log/kubernetes/
systemctl daemon-reload
systemctl enable --now kube-apiserver
4.4、安装高可用
4.4.1、安装高可用软件
yum install -y keepalived haproxy
4.4.2、配置高可用软件
cat > /etc/haproxy/haproxy.cfg <<EOF
global
maxconn 2000
ulimit-n 16384
log 127.0.0.1 local0 err
stats timeout 30s
defaults
log global
mode http
option httplog
timeout connect 5000
timeout client 50000
timeout server 50000
timeout http-request 15s
timeout http-keep-alive 15s
frontend monitor-in
bind *:33305
mode http
option httplog
monitor-uri /monitor
listen stats
bind *:8006
mode http
stats enable
stats hide-version
stats uri /stats
stats refresh 30s
stats realm Haproxy\ Statistics
stats auth admin:admin
frontend k8s-master
bind 0.0.0.0:8443
bind 127.0.0.1:8443
mode tcp
option tcplog
tcp-request inspect-delay 5s
default_backend k8s-master
backend k8s-master
mode tcp
option tcplog
option tcp-check
balance roundrobin
default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
server k8s-master-01 192.168.11.101:6443 check inter 2000 fall 2 rise 2 weight 100
server k8s-master-02 192.168.11.102:6443 check inter 2000 fall 2 rise 2 weight 100
server k8s-master-03 192.168.11.103:6443 check inter 2000 fall 2 rise 2 weight 100
EOF
mv /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf_bak
cd /etc/keepalived
KUBERNETES_HOSTNAME=`hostname`
KUBERNETES_IP=`hostname -i`
cat > /etc/keepalived/keepalived.conf <<EOF
! Configuration File for keepalived
global_defs {
router_id ${KUBERNETES_HOSTNAME}
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
mcast_src_ip ${KUBERNETES_IP}
virtual_router_id 51
priority 80
advert_int 2
authentication {
auth_type PASS
auth_pass K8SHA_KA_AUTH
}
virtual_ipaddress {
192.168.11.109
}
}
EOF
4.4.3、启动
[root@k8s-master-01 keepalived]# systemctl enable --now keepalived haproxy
4.5、安装kube-controller-manager
4.5.1、编写配置文件
cat > /etc/kubernetes/cfg/kube-controller-manager.conf << EOF
KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/var/log/kubernetes \\
--leader-elect=true \\
--cluster-name=kubernetes \\
--bind-address=127.0.0.1 \\
--allocate-node-cidrs=true \\
--cluster-cidr=10.244.0.0/12 \\
--service-cluster-ip-range=10.96.0.0/16 \\
--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \\
--cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \\
--root-ca-file=/etc/kubernetes/ssl/ca.pem \\
--service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \\
--kubeconfig=/etc/kubernetes/cfg/kube-controller-manager.kubeconfig \\
--tls-cert-file=/etc/kubernetes/ssl/kube-controller-manager.pem \\
--tls-private-key-file=/etc/kubernetes/ssl/kube-controller-manager-key.pem \\
--experimental-cluster-signing-duration=87600h0m0s \\
--controllers=*,bootstrapsigner,tokencleaner \\
--use-service-account-credentials=true \\
--node-monitor-grace-period=10s \\
--horizontal-pod-autoscaler-use-rest-clients=true"
EOF
4.5.2、注册服务
cat > /usr/lib/systemd/system/kube-controller-manager.service << EOF
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-controller-manager.conf
ExecStart=/usr/local/bin/kube-controller-manager \$KUBE_CONTROLLER_MANAGER_OPTS
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
4.5.3、启动
systemctl daemon-reload
systemctl enable --now kube-controller-manager
systemctl status kube-controller-manager
4.6、安装kube-scheduler
4.6.1、编写配置文件
cat > /etc/kubernetes/cfg/kube-scheduler.conf << EOF
KUBE_SCHEDULER_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/var/log/kubernetes \\
--kubeconfig=/etc/kubernetes/cfg/kube-scheduler.kubeconfig \\
--leader-elect=true \\
--master=http://127.0.0.1:8080 \\
--bind-address=127.0.0.1 "
EOF
4.6.2、注册服务
cat > /usr/lib/systemd/system/kube-scheduler.service << EOF
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-scheduler.conf
ExecStart=/usr/local/bin/kube-scheduler \$KUBE_SCHEDULER_OPTS
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
4.6.3、启动
# 分别在三台master节点上启动
systemctl daemon-reload
systemctl enable --now kube-scheduler
4.7、安装kube-proxy
4.7.1、编写配置
cat > /etc/kubernetes/cfg/kube-proxy.conf << EOF
KUBE_PROXY_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/var/log/kubernetes \\
--config=/etc/kubernetes/cfg/kube-proxy-config.yml"
EOF
KUBE_APISERVER_IP=`hostname -i`
KUBE_APISERVER_HOSTNAME=`hostname`
cat > /etc/kubernetes/cfg/kube-proxy-config.yml << EOF
kind: KubeProxyConfiguration
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: ${KUBE_APISERVER_IP}
healthzBindAddress: ${KUBE_APISERVER_IP}:10256
metricsBindAddress: ${KUBE_APISERVER_IP}:10249
clientConnection:
burst: 200
kubeconfig: /etc/kubernetes/cfg/kube-proxy.kubeconfig
qps: 100
hostnameOverride: ${KUBE_APISERVER_HOSTNAME}
clusterCIDR: 10.96.0.0/16
enableProfiling: true
mode: "ipvs"
kubeProxyIPTablesConfiguration:
masqueradeAll: false
kubeProxyIPVSConfiguration:
scheduler: rr
excludeCIDRs: []
EOF
4.7.2、注册服务
cat > /usr/lib/systemd/system/kube-proxy.service << EOF
[Unit]
Description=Kubernetes Proxy
After=network.target
[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-proxy.conf
ExecStart=/usr/local/bin/kube-proxy \$KUBE_PROXY_OPTS
Restart=on-failure
RestartSec=10
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
4.7.3、启动
systemctl daemon-reload; systemctl enable --now kube-proxy; systemctl status kube-proxy
4.8、安装kubelet
4.8.1、部署TLS bootstrapping
kubectl create clusterrolebinding kubelet-bootstrap \
--clusterrole=system:node-bootstrapper \
--user=kubelet-bootstrap
4.8.2、编写kubelet配置文件
KUBE_HOSTNAME=`hostname`
KUBE_HOSTNAME_IP=`hostname -i`
cat > /etc/kubernetes/cfg/kubelet.conf << EOF
KUBELET_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/var/log/kubernetes \\
--hostname-override=${KUBE_HOSTNAME} \\
--container-runtime=docker \\
--kubeconfig=/etc/kubernetes/cfg/kubelet.kubeconfig \\
--bootstrap-kubeconfig=/etc/kubernetes/cfg/kubelet-bootstrap.kubeconfig \\
--config=/etc/kubernetes/cfg/kubelet-config.yml \\
--cert-dir=/etc/kubernetes/ssl \\
--image-pull-progress-deadline=15m \\
--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/k8sos/pause:3.2"
EOF
cat > /etc/kubernetes/cfg/kubelet-config.yml << EOF
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: ${KUBE_HOSTNAME_IP}
port: 10250
readOnlyPort: 10255
cgroupDriver: cgroupfs
clusterDNS:
- 10.96.0.2
clusterDomain: cluster.local
failSwapOn: false
authentication:
anonymous:
enabled: false
webhook:
cacheTTL: 2m0s
enabled: true
x509:
clientCAFile: /etc/kubernetes/ssl/ca.pem
authorization:
mode: Webhook
webhook:
cacheAuthorizedTTL: 5m0s
cacheUnauthorizedTTL: 30s
evictionHard:
imagefs.available: 15%
memory.available: 100Mi
nodefs.available: 10%
nodefs.inodesFree: 5%
maxOpenFiles: 1000000
maxPods: 110
EOF
4.8.3、注册kubelet的服务
cat > /usr/lib/systemd/system/kubelet.service << EOF
[Unit]
Description=Kubernetes Kubelet
After=docker.service
[Service]
EnvironmentFile=/etc/kubernetes/cfg/kubelet.conf
ExecStart=/usr/local/bin/kubelet \$KUBELET_OPTS
Restart=on-failure
RestartSec=10
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
4.8.4、启动kubelet
systemctl daemon-reload;systemctl enable --now kubelet;systemctl status kubelet.service
4.9、将节点加入集群
4.9.1、查看apiserver状态
[root@k8s-master-01 keepalived]# kubectl get cs
NAME STATUS MESSAGE ERROR
controller-manager Healthy ok
scheduler Healthy ok
etcd-0 Healthy {"health":"true"}
etcd-2 Healthy {"health":"true"}
etcd-1 Healthy {"health":"true"}
4.9.2、查看集群加入请求
[root@k8s-master-01 keepalived]# kubectl get csr
4.9.3、批准加入
kubectl certificate approve `kubectl get csr | grep "Pending" | awk '{print $1}'`
4.9.4、查看
[root@k8s-master-01 keepalived]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master-01 NotReady <none> 2s v1.18.8
k8s-master-03 NotReady <none> 2s v1.18.8
4.9.5、创建一个超管用户
kubectl create clusterrolebinding cluster-system-anonymous --clusterrole=cluster-admin --user=kubernetes
4.10、安装DNS
4.10.1、下载DNS代码
# 下载代码的链接：https://github.com/coredns/deployment/tree/coredns-1.14.0
[root@k8s-master-01 kubernetes]# pwd
/root/deployment-coredns-1.14.0/kubernetes
# 部署
[root@k8s-master-01 kubernetes]# ./deploy.sh -i 10.96.0.2 -s | kubectl apply -f -
serviceaccount/coredns created
clusterrole.rbac.authorization.k8s.io/system:coredns created
clusterrolebinding.rbac.authorization.k8s.io/system:coredns created
configmap/coredns created
deployment.apps/coredns created
service/kube-dns created
# 检测
[root@k8s-master-01 kubernetes]# kubectl get pods -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system coredns-8686db44f5-xlw46 1/1 Running 0 73s
4.11、安装flannel网络插件
4.11.1、下载网络查看
[root@kubernetes-master-01 /opt]# wget https://github.com/coreos/flannel/releases/download/v0.13.1-rc1/flannel-v0.13.1-rc1-linux-amd64.tar.gz
4.11.2、解压
[root@k8s-master-01 ~]# tar -xf flannel-v0.13.1-rc1-linux-amd64.tar.gz
4.11.3、分发flannel
for i in m1 m2 m3 n1 n2 n3 n4 n5; do scp flanneld mk-docker-opts.sh root@$i:/usr/local/bin; done
4.11.4、修改Docker启动文件
sed -i '/ExecStart/s/\(.*\)/#\1/' /usr/lib/systemd/system/docker.service
sed -i '/ExecReload/a ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS -H fd:// --containerd=/run/containerd/containerd.sock' /usr/lib/systemd/system/docker.service
sed -i '/ExecReload/a EnvironmentFile=-/run/flannel/subnet.env' /usr/lib/systemd/system/docker.service
for i in 101 102 103 104 105 106 107 108
do
ssh root@192.168.11.$i "sed -i '/ExecStart/s/\(.*\)/#\1/' /usr/lib/systemd/system/docker.service"
ssh root@192.168.11.$i "sed -i '/ExecReload/a ExecStart=/usr/bin/dockerd \$DOCKER_NETWORK_OPTIONS -H fd:// --containerd=/run/containerd/containerd.sock' /usr/lib/systemd/system/docker.service"
ssh root@192.168.11.$i "sed -i '/ExecReload/a EnvironmentFile=-/run/flannel/subnet.env' /usr/lib/systemd/system/docker.service"
done
4.11.5、将flannel的配置文件写入ETCD
etcdctl \
--ca-file=/etc/etcd/ssl/ca.pem \
--cert-file=/etc/etcd/ssl/etcd.pem \
--key-file=/etc/etcd/ssl/etcd-key.pem \
--endpoints="https://192.168.11.101:2379,https://192.168.11.102:2379,https://192.168.11.103:2379" \
mk /coreos.com/network/config '{"Network":"10.244.0.0/12", "SubnetLen": 21, "Backend": {"Type": "vxlan", "DirectRouting": true}}'
4.11.6、注册flannel服务
cat > /usr/lib/systemd/system/flanneld.service << EOF
[Unit]
Description=Flanneld address
After=network.target
After=network-online.target
Wants=network-online.target
After=etcd.service
Before=docker.service
[Service]
Type=notify
ExecStart=/usr/local/bin/flanneld \\
-etcd-cafile=/etc/etcd/ssl/ca.pem \\
-etcd-certfile=/etc/etcd/ssl/etcd.pem \\
-etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \\
-etcd-endpoints=https://192.168.11.101:2379,https://192.168.11.102:2379,https://192.168.11.103:2379 \\
-etcd-prefix=/coreos.com/network \\
-ip-masq
ExecStartPost=/usr/local/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/subnet.env
Restart=always
RestartSec=5
StartLimitInterval=0
[Install]
WantedBy=multi-user.target
RequiredBy=docker.service
EOF
4.11.7、启动flannel
systemctl daemon-reload
systemctl start flanneld
systemctl restart docker
4.12、测试集群
[root@k8s-master-01 ~]# kubectl run test -it --rm --image=busybox:1.28.3
If you don't see a command prompt, try pressing enter.
/ # nslookup kubernetes
Server: 10.96.0.2
Address 1: 10.96.0.2 kube-dns.kube-system.svc.cluster.local
Name: kubernetes
Address 1: 10.96.0.1 kubernetes.default.svc.cluster.local