K8s安装部署

K8S集群安装部署

一、系统基本环境搭建

角色 IP 组件
node1 192.168.24.136 kube-apiserver, kube-controller-manager, kube-scheduler, etcd
node2 192.168.24.137 kubelet,kube-proxy,docker etcd
node3 192.168.24.138 kubelet,kube-proxy,docker etcd

1.关闭swap

[root@node1 ~]# swapoff -a
[root@node1 ~]# vim /etc/fstab

2.关闭SELINUX

[root@node1 ~]# setenforce 0 && sed -i 's/^SELINUX=. */SELINUX=disabled/' /etc/selinux/config

3.调整内核参数

[root@node1 ~]# cat > /etc/sysctl.d/k8s.conf <<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
EOF
# 生效
[root@node1 ~]# sysctl --system

4.调整系统时区

#设置系统时区为中国/上海
[root@node1 ~]# timedatectl set-timezone Asia/Shanghai
#将当前的UTC时间写入硬件时钟
[root@node1 ~]# timedatectl set-local-rtc 0
#重启依赖于系统时间的服务
[root@node1 ~]# systemctl restart rsyslog
[root@node1 ~]# systemctl restart crond

5.关闭系统不需要的服务

[root@node1 ~]# systemctl stop postfix && systemctl disable postfix

二、安装etcd

1.创建证书文件

[root@localhost home]# mkdir /opt/TLS/etcd && cd /opt/TLS/etcd
[root@localhost etcd]# wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
[root@localhost etcd]# wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
[root@localhost etcd]# wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd6
[root@localhost etcd]# mkdir /usr/local/cfssl/
[root@localhost etcd]# mv cfssl_linux-amd64 /usr/local/cfssl/
[root@localhost etcd]# mv cfssljson_linux-amd64 /usr/local/cfssl/
[root@localhost etcd]# mv cfssl-certinfo_linux-amd6 /usr/local/cfssl/
[root@localhost etcd]#  vim > ca-config.json<< EOF 
{
	"signing": {
		"default": {
			"expiry": "87600h"
		},
		"profiles": {
			"www": {
				"expiry": "87600h",
				"usages": [
					"signing",
					"key encipherment",
					"server auth",
					"client auth"
				]
			}
		}
	}
}
[root@localhost etcd]#  vim ca-csr.json
{
        "CN": "etcd CA",
        "key": {
                "algo": "rsa",
                "size": 2048
        },
        "names": [
                {
                        "C": "CN",
                        "L": "Beijing",
                        "ST": "Beijing"
                }
        ]
}
[root@localhost etcd]# vim server-csr.json
{
	"CN": "etcd",
	"hosts": [
		"192.168.24.136",
		"192.168.24.137",
		"192.168.24.138"
	],
	"key": {
		"algo": "rsa",
		"size": 2048
	},
	"names": [
		{
			"C": "CN",
			"L": "BeiJing",
			"ST": "BeiJing"
		}
	]
}
[root@localhost etcd]# /usr/local/cfssl/cfssl_linux-amd64 gencert -initca ca-csr.json | /usr/local/cfssl/cfssljson_linux-amd64 -bare ca -
2021/06/30 23:30:35 [INFO] generating a new CA key and certificate from CSR
2021/06/30 23:30:35 [INFO] generate received request
2021/06/30 23:30:35 [INFO] received CSR
2021/06/30 23:30:35 [INFO] generating key: rsa-2048
2021/06/30 23:30:36 [INFO] encoded CSR
2021/06/30 23:30:36 [INFO] signed certificate with serial number 725059738840226310954172302162231768550676521910
[root@localhost etcd]# 

[root@localhost etcd]# /usr/local/cfssl/cfssl_linux-amd64 gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | /usr/local/cfssl/cfssljson_linux-amd64 -bare server
2021/06/30 23:39:25 [INFO] generate received request
2021/06/30 23:39:25 [INFO] received CSR
2021/06/30 23:39:25 [INFO] generating key: rsa-2048
2021/06/30 23:39:25 [INFO] encoded CSR
2021/06/30 23:39:25 [INFO] signed certificate with serial number 411396881075107735287957499160567737895798023863
2021/06/30 23:39:25 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@localhost etcd]# 

[root@localhost etcd]# ll
total 36
-rw-r--r-- 1 root root  226 Jun 30 23:26 ca-config.json
-rw-r--r-- 1 root root  956 Jun 30 23:30 ca.csr
-rw-r--r-- 1 root root  143 Jun 30 23:28 ca-csr.json
-rw------- 1 root root 1679 Jun 30 23:30 ca-key.pem
-rw-r--r-- 1 root root 1265 Jun 30 23:30 ca.pem
-rw-r--r-- 1 root root 1013 Jun 30 23:39 server.csr
-rw-r--r-- 1 root root  215 Jun 30 23:35 server-csr.json
-rw------- 1 root root 1679 Jun 30 23:39 server-key.pem
-rw-r--r-- 1 root root 1338 Jun 30 23:39 server.pem
[root@localhost etcd]# 

2. 安装etcd

2.1 node1上操作

[root@node1 opt]# wget https://github.com/etcd-io/etcd/releases/download/v3.4.9/etcd-v3.4.9-linux-amd64.tar.gz
[root@node1 opt]# tar -zxvf etcd-v3.4.9-linux-amd64.tar.gz
[root@node1 opt]# mkdir -p /opt/etcd/{bin,cfg,ssl}
[root@node1 opt]# mv etcd-v3.4.9-linux-amd64/{etcd,etcdctl} /opt/etcd/bin/
[root@node1 etcd]# scp -r /opt/TLS/etcd/{ca.pem,server-key.pem,server.pem} /opt/etcd/ssl/
[root@node1 opt]# vim /opt/etcd/cfg/etcd.conf
#[Member] 
ETCD_NAME="etcd-1" 
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.24.136:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.24.136:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.24.136:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.24.136:2379" 
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.24.136:2380,etcd-2=https://192.168.24.137:2380,etcd-3=https://192.168.24.138:2380" 
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" 
ETCD_INITIAL_CLUSTER_STATE="new"

#[Security]
ETCD_CERT_FILE="/opt/etcd/ssl/server.pem"
ETCD_KEY_FILE="/opt/etcd/ssl/server-key.pem"
ETCD_TRUSTED_CA_FILE="/opt/etcd/ssl/ca.pem"
ETCD_CLIENT_CERT_AUTH="true"
ETCD_PEER_CERT_FILE="/opt/etcd/ssl/server.pem"
ETCD_PEER_KEY_FILE="/opt/etcd/ssl/server-key.pem"
ETCD_PEER_TRUSTED_CA_FILE="/opt/etcd/ssl/ca.pem"
ETCD_PEER_CLIENT_CERT_AUTH="true"

[root@node1 opt]# vim /usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server 
After=network.target 
After=network-online.target 
Wants=network-online.target 
[Service]
Type=notify
EnvironmentFile=/opt/etcd/cfg/etcd.conf
ExecStart=/opt/etcd/bin/etcd
Restart=on-failure
LimitNOFILE=65536
[Install] 
WantedBy=multi-user.target
[root@node1 etcd]# scp -r /opt/etcd root@node2:/opt/
[root@node1 etcd]# scp -r /opt/etcd root@node3:/opt/
[root@node1 etcd]# scp -r /usr/lib/systemd/system/etcd.service root@node1:/usr/lib/systemd/system/
[root@node1 etcd]# scp -r /usr/lib/systemd/system/etcd.service root@node2:/usr/lib/systemd/system/
[root@node1 etcd]# systemctl daemon-reload 

2.2 node2上操作

[root@node2 ~]# vim /opt/etcd/cfg/etcd.conf
#[Member] 
ETCD_NAME="etcd-2"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.24.137:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.24.137:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.24.137:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.24.137:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.24.136:2380,etcd-2=https://192.168.24.137:2380,etcd-3=https://192.168.24.138:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

#[Security]
ETCD_CERT_FILE="/opt/etcd/ssl/server.pem"
ETCD_KEY_FILE="/opt/etcd/ssl/server-key.pem"
ETCD_TRUSTED_CA_FILE="/opt/etcd/ssl/ca.pem"
ETCD_CLIENT_CERT_AUTH="true"
ETCD_PEER_CERT_FILE="/opt/etcd/ssl/server.pem"
ETCD_PEER_KEY_FILE="/opt/etcd/ssl/server-key.pem"
ETCD_PEER_TRUSTED_CA_FILE="/opt/etcd/ssl/ca.pem"
ETCD_PEER_CLIENT_CERT_AUTH="true"

[root@node2 etcd]# systemctl daemon-reload

2.3 node3上操作

[root@node3 ~]# vim /opt/etcd/cfg/etcd.conf
#[Member] 
ETCD_NAME="etcd-3" 
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.24.138:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.24.138:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.24.138:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.24.138:2379" 
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.24.136:2380,etcd-2=https://192.168.24.137:2380,etcd-3=https://192.168.24.138:2380" 
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" 
ETCD_INITIAL_CLUSTER_STATE="new"

#[Security]
ETCD_CERT_FILE="/opt/etcd/ssl/server.pem"
ETCD_KEY_FILE="/opt/etcd/ssl/server-key.pem"
ETCD_TRUSTED_CA_FILE="/opt/etcd/ssl/ca.pem"
ETCD_CLIENT_CERT_AUTH="true"
ETCD_PEER_CERT_FILE="/opt/etcd/ssl/server.pem"
ETCD_PEER_KEY_FILE="/opt/etcd/ssl/server-key.pem"
ETCD_PEER_TRUSTED_CA_FILE="/opt/etcd/ssl/ca.pem"
ETCD_PEER_CLIENT_CERT_AUTH="true"
[root@node3 etcd]# systemctl daemon-reload

2.4 分别启动etcd

[root@node1 etcd]# systemctl start etcd 
[root@node1 etcd]# systemctl enable etcd
# 启动node1的etcd后需要立即启动node2、node3上的etcd,否则node1的etcd访问node2、node3不通,一段时候会报启动失败

备注说明

  • ETCD3.4版本ETCDCTL_API=3 etcdctl 和 etcd --enable-v2=false 成为了默认配置,如要使用v2版本,执行etcdctl时候需要设置ETCDCTL_API环境变量,例如:ETCDCTL_API=2 etcdctl
  • ETCD3.4版本会自动读取环境变量的参数,所以EnvironmentFile文件中有的参数,不需要再次在ExecStart启动参数中添加,二选一,如同时配置,会触发以下类似报错“etcd: conflicting environment variable - - “ETCD_NAME” is shadowed by corresponding command-line flag (either unset environment variable or disable flag)”
  • flannel操作etcd使用的是v2的API,而kubernetes操作etcd使用的v3的API

/opt/etcd/cfg/etcd.conf 配置文件参数说明:

参数 含义
ETCD_NAME 节点名称,集群中唯一
ETCD_DATA_DIR 数据目录
ETCD_LISTEN_PEER_URLS 集群通信监听地址
ETCD_LISTEN_CLIENT_URLS 客户端访问监听地址
ETCD_INITIAL_ADVERTISE_PEER_URLS 集群通告地址
ETCD_ADVERTISE_CLIENT_URLS 客户端通告地址
ETCD_INITIAL_CLUSTER 集群节点地址
ETCD_INITIAL_CLUSTER_TOKEN 集群 Token
ETCD_INITIAL_CLUSTER_STATE 加入集群的当前状态,new 是新集群,existing 表示加入

三、安装docker

node1节点

[root@node1 opt]# wget https://download.docker.com/linux/static/stable/x86_64/docker-19.03.9.tgz
[root@node1 opt]# mv docker /usr/local/
[root@node1 opt]# vim /usr/lib/systemd/system/docker.service
[Unit]
Description=Docker Application Container Engine 
Documentation=https://docs.docker.com 
After=network-online.target firewalld.service 
Wants=network-online.target

[Service] 
Type=notify 
ExecStart=/usr/bin/dockerd
ExecReload=/bin/kill -s HUP $MAINPID
LimitNOFILE=infinity 
LimitNPROC=infinity 
LimitCORE=infinity 
TimeoutStartSec=0 
Delegate=yes 
KillMode=process 
Restart=on-failure 
StartLimitBurst=3 
StartLimitInterval=60s

[Install] 
WantedBy=multi-user.target
[root@node1 opt]# systemctl daemon-reload
[root@node1 opt]# systemctl start docker
[root@node1 opt]# systemctl enable docker

[root@node1 opt]# scp -r docker/* root@node2:/usr/bin/
[root@node1 opt]# scp -r docker/* root@node3:/usr/bin/
[root@node1 opt]# scp -r /usr/lib/systemd/system/docker.service root@node2:/usr/lib/systemd/system/ 
[root@node1 opt]# scp -r /usr/lib/systemd/system/docker.service root@node3:/usr/lib/systemd/system/

node2节点

[root@node2 opt]# systemctl daemon-reload && systemctl start docker && systemctl enable docker

node3节点

[root@node2 opt]# systemctl daemon-reload && systemctl start docker && systemctl enable docker

四、部署api-server(Master Node)

1.创建apiserver自签证书

[root@node1 k8s]# vim /opt/TLS/k8s/ca-config.json
{
	"signing": {
		"default": {
			"expiry": "87600h"
		},
		"profiles": {
			"kubernetes": {
				"expiry": "87600h",
				"usages": [
					"signing",
					"key encipherment",
					"server auth",
					"client auth"
				]
			}
		}
	}
}

[root@node1 k8s]# vim /opt/TLS/k8s/ca-csr.json
{
	"CN": "kubernetes",
	"key": {
		"algo": "rsa",
		"size": 2048
	},
	"names": [
		{
			"C": "CN",
			"L": "Beijing",
			"ST": "Beijing",
			"O": "k8s",
			"OU": "System"
		}
	]
}

[root@node1 k8s]# vim /opt/TLS/k8s/server-csr.json
{
	"CN": "kubernetes",
	"hosts": [
		"10.0.0.1",
		"127.0.0.1",
		"kubernetes",
		"kubernetes.default",
		"kubernetes.default.svc",
		"kubernetes.default.svc.cluster",
		"kubernetes.default.svc.cluster.local",
		"192.168.24.136",
		"192.168.24.137",
		"192.168.24.138"
	],
	"key": {
		"algo": "rsa",
		"size": 2048
	},
	"names": [
		{
			"C": "CN",
			"L": "BeiJing",
			"ST": "BeiJing",
			"O": "k8s",
			"OU": "System"
		}
	]
}

# 生成证书文件
[root@node1 k8s]# /usr/local/cfssl/cfssl gencert -initca ca-csr.json | /usr/local/cfssl/cfssljson -bare ca -
2021/07/03 17:50:47 [INFO] generating a new CA key and certificate from CSR
2021/07/03 17:50:47 [INFO] generate received request
2021/07/03 17:50:47 [INFO] received CSR
2021/07/03 17:50:47 [INFO] generating key: rsa-2048
2021/07/03 17:50:47 [INFO] encoded CSR
2021/07/03 17:50:47 [INFO] signed certificate with serial number 112636923636866601548066763582581017478124700260
[root@node1 k8s]# 

#使用自签 CA 签发 kube-apiserver HTTPS 证书
[root@node1 k8s]# /usr/local/cfssl/cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | /usr/local/cfssl/cfssljson -bare server
2021/07/03 17:53:04 [INFO] generate received request
2021/07/03 17:53:04 [INFO] received CSR
2021/07/03 17:53:04 [INFO] generating key: rsa-2048
2021/07/03 17:53:05 [INFO] encoded CSR
2021/07/03 17:53:05 [INFO] signed certificate with serial number 60072710308510964584407608453253056896368493951
2021/07/03 17:53:05 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

[root@node1 k8s]# ls server*.pem
server-key.pem  server.pem
[root@node1 k8s]# 

2.下载安装包

K8s安装部署
K8s安装部署
K8s安装部署

[root@node1 opt]# wget https://dl.k8s.io/v1.19.0/kubernetes-server-linux-amd64.tar.gz
[root@node1 opt]# tar -zxvf kubernetes-server-linux-amd64.tar.gz
[root@node1 opt]# mv kubernetes kubernetes_package
[root@node1 opt]# mkdir -p /opt/kubernetes/{bin,cfg,ssl,logs}
[root@node1 opt]# cp kubernetes_package/kubernetes/server/bin/{kube-apiserver,kube-scheduler,kube-controller-manager,kubectl} /opt/kubernetes/bin/
[root@node1 opt]# ll /opt/kubernetes/bin/
total 300412
-rwxr-xr-x 1 root root 115245056 Jul  3 18:13 kube-apiserver
-rwxr-xr-x 1 root root 107249664 Jul  3 18:13 kube-controller-manager
-rwxr-xr-x 1 root root  43003904 Jul  3 18:15 kubectl
-rwxr-xr-x 1 root root  42123264 Jul  3 18:13 kube-scheduler
[root@node1 opt]# ln -s /opt/kubernetes/bin/kubectl /usr/bin/kubectl
[root@node1 opt]# 
[root@node1 opt]# cp /opt/TLS/k8s/{ca-key.pem,ca.pem,server-key.pem,server.pem} /opt/kubernetes/ssl/
[root@node1 opt]# ll /opt/kubernetes/ssl/
total 16
-rw------- 1 root root 1679 Jul  3 18:20 ca-key.pem
-rw-r--r-- 1 root root 1359 Jul  3 18:20 ca.pem
-rw------- 1 root root 1679 Jul  3 18:20 server-key.pem
-rw-r--r-- 1 root root 1627 Jul  3 18:20 server.pem
[root@node1 opt]#

3.配置api-server配置文件

[root@node1 opt]# vim /opt/kubernetes/cfg/kube-apiserver.conf
KUBE_APISERVER_OPTS="--logtostderr=false \
--v=2 \
--log-dir=/opt/kubernetes/logs \
--etcd-servers=https://192.168.24.136:2379,https://192.168.24.137:2379,https://192.168.24.138:2379 \
--bind-address=192.168.24.136 \
--secure-port=6443 \
--advertise-address=192.168.24.136 \
--allow-privileged=true \
--service-cluster-ip-range=10.0.0.0/24 \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction  \
--authorization-mode=RBAC,Node \
--enable-bootstrap-token-auth=true \
--token-auth-file=/opt/kubernetes/cfg/token.csv \
--service-node-port-range=30000-32767 \
--kubelet-client-certificate=/opt/kubernetes/ssl/server.pem \
--kubelet-client-key=/opt/kubernetes/ssl/server-key.pem \
--tls-cert-file=/opt/kubernetes/ssl/server.pem \
--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \
--client-ca-file=/opt/kubernetes/ssl/ca.pem \
--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \
--etcd-cafile=/opt/etcd/ssl/ca.pem \
--etcd-certfile=/opt/etcd/ssl/server.pem \
--etcd-keyfile=/opt/etcd/ssl/server-key.pem \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/opt/kubernetes/logs/k8s-audit.log
参数 参数解释
–logtostderr 启用日志
–v 日志等级
–log-dir 日志目录
–etcd-servers etcd 集群地址
–bind-address 监听地址
–secure-port https 安全端口
–advertise-address 集群通告地址
–allow-privileged 启用授权
–service-cluster-ip-range Service 虚拟 IP 地址段
–enable-admission-plugins 准入控制模块
–authorization-mode 认证授权,启用 RBAC 授权和节点自管理
–enable-bootstrap-token-auth 启用 TLS bootstrap 机制
–token-auth-file bootstrap token 文件
–service-node-port-range Service nodeport 类型默认分配端口范围
–kubelet-client-xxx apiserver 访问 kubelet 客户端证书
–tls-xxx-file apiserver https 证书
–etcd-xxxfile 连接 Etcd 集群证书
–audit-log-xxx 审计日志
  • 启用 TLS Bootstrapping
    TLS Bootstrapping机制 TLS Bootstraping:Master apiserver 启用 TLS 认证后,Node 节点 kubelet 和 kube- proxy 要与 kube-apiserver 进行通信,必须使用 CA 签发的有效证书才可以,当 Node 节点很多时,这种客户端证书颁发需要大量工作,同样也会增加集群扩展复杂度。为了 简化流程,Kubernetes 引入了 TLS bootstraping 机制来自动颁发客户端证书,kubelet 会以一个低权限用户自动向 apiserver 申请证书,kubelet 的证书由 apiserver 动态签署。
    所以强烈建议在 Node 上使用这种方式,目前主要用于 kubelet,kube-proxy 还是由我 们统一颁发一个证书。

4.创建token文件

格式:token,用户名,UID,用户组

[root@node1 ~]# head -c 16 /dev/urandom | od -An -t x | tr -d ' '
ef80e12f86c4e342f449e64e3f94f3a9
[root@node1 opt]# vim /opt/kubernetes/cfg/token.csv
c47ffb939f5ca36231d9e3121a252940,kubelet-bootstrap,10001,"system:node- bootstrapper"

5.systemd 管理 apiserver

[root@node1 opt]# vim /usr/lib/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes API Server 
Documentation=https://github.com/kubernetes/kubernetes 

[Service]
EnvironmentFile=/opt/kubernetes/cfg/kube-apiserver.conf
ExecStart=/opt/kubernetes/bin/kube-apiserver $KUBE_APISERVER_OPTS 
Restart=on-failure

[Install]
WantedBy=multi-user.target
[root@node1 opt]# 

6.启动并设置开机启动

[root@node1 opt]# systemctl daemon-reload
[root@node1 opt]# systemctl start kube-apiserver
[root@node1 opt]# systemctl enable kube-apiserver

7.授权 kubelet-bootstrap 用户允许请求证书

[root@node1 opt]# kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap

K8s安装部署

五、部署kube-controller-manager(Master Node)

1. 创建配置文件

[root@node1 opt]# vim /opt/kubernetes/cfg/kube-controller-manager.conf
KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=false \
--v=2 \
--log-dir=/opt/kubernetes/logs \
--leader-elect=true \
--master=127.0.0.1:8080 \
--bind-address=127.0.0.1 \
--allocate-node-cidrs=true \
--cluster-cidr=10.244.0.0/16 \
--service-cluster-ip-range=10.0.0.0/24 \
--cluster-signing-cert-file=/opt/kubernetes/ssl/ca.pem \
--cluster-signing-key-file=/opt/kubernetes/ssl/ca-key.pem \
--root-ca-file=/opt/kubernetes/ssl/ca.pem \
--service-account-private-key-file=/opt/kubernetes/ssl/ca-key.pem \
--experimental-cluster-signing-duration=87600h0m0s"

2. systemd 管理 controller-manager

[root@node1 opt]# vim /usr/lib/systemd/system/kube-controller-manager.service
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=/opt/kubernetes/cfg/kube-controller-manager.conf 
ExecStart=/opt/kubernetes/bin/kube-controller-manager $KUBE_CONTROLLER_MANAGER_OPTS
Restart=on-failure

[Install]
WantedBy=multi-user.target

3. 启动并设置开机启动

[root@node1 opt]# systemctl daemon-reload 
[root@node1 opt]# systemctl start kube-controller-manager 
[root@node1 opt]# systemctl enable kube-controller-manager

六、部署 kube-scheduler(Master Node)

1. 创建配置文件

[root@node1 opt]# vim /opt/kubernetes/cfg/kube-scheduler.conf

2. systemd 管理 kube-scheduler

[root@node1 opt]# vim /usr/lib/systemd/system/kube-scheduler.service
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=/opt/kubernetes/cfg/kube-scheduler.conf
ExecStart=/opt/kubernetes/bin/kube-scheduler $KUBE_SCHEDULER_OPTS
Restart=on-failure

[Install]
WantedBy=multi-user.target

3. 启动并设置开机启动

[root@node1 opt]# systemctl daemon-reload
[root@node1 opt]# systemctl status kube-scheduler
[root@node1 opt]# systemctl enable kube-scheduler

七、查看集群状态

[root@node1 opt]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS    MESSAGE             ERROR
controller-manager   Healthy   ok                  
scheduler            Healthy   ok                  
etcd-1               Healthy   {"health":"true"}   
etcd-0               Healthy   {"health":"true"}   
etcd-2               Healthy   {"health":"true"}   
[root@node1 opt]# 

八、部署 kubelet(Worker Node)

1. 创建kubelet.conf配置文件

[root@node2 ~]# mkdir -p /opt/kubernetes/{bin,cfg,ssl,logs}
[root@node2 ~]# vim /opt/kubernetes/cfg/kubelet.conf
KUBELET_OPTS="--logtostderr=false \
--v=2 \
--log-dir=/opt/kubernetes/logs \
--hostname-override=node2 \
--network-plugin=cni \
--kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \
--bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig \
--config=/opt/kubernetes/cfg/kubelet-config.yml \
--cert-dir=/opt/kubernetes/ssl \
--pod-infra-container-image=lizhenliang/pause-amd64:3.0"

2.配置kubelet-config.yml参数文件

[root@node2 ~]# vim /opt/kubernetes/cfg/kubelet-config.yml
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: 0.0.0.0
port: 10250
readOnlyPort: 10255
cgroupDriver: cgroupfs
clusterDNS:
- 10.0.0.2
clusterDomain: cluster.local
failSwapOn: false
authentication:
	anonymous:
		enabled: false 
	webhook: 
		cacheTTL: 2m0s 
		enabled: true 
	x509: 
		clientCAFile: /opt/kubernetes/ssl/ca.pem 
authorization: 
	mode: Webhook 
	webhook: 
		cacheAuthorizedTTL: 5m0s
		cacheUnauthorizedTTL: 30s 
evictionHard: 
imagefs.available: 15% 
memory.available: 100Mi
nodefs.available: 10% 
nodefs.inodesFree: 5%
maxOpenFiles: 1000000
maxPods: 110

3.生成bootstrap.kubeconfig 配置文件

[root@node1 ~]# scp -r /opt/kubernetes_package/kubernetes/server/bin/{kubectl,kubelet,kube-proxy} root@node2:/opt/kubernetes/bin/
[root@node1 ~]# scp -r /opt/kubernetes/ssl/ca.pem root@node2:/opt/kubernetes/ssl/  
[root@node2 ]# cd /opt/kubernetes/cfg 
[root@node2 cfg]# export KUBE_APISERVER="https://192.168.24.136:6443"
  145 
[root@node2 cfg]# export PORT TOKEN="c47ffb939f5ca36231d9e3121a252940"
[root@node2 cfg]# /opt/kubernetes/bin/kubectl config set-cluster kubernetes \
--certificate-authority=/opt/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=bootstrap.kubeconfig
[root@node2 cfg]# /opt/kubernetes/bin/kubectl config set-credentials "kubelet-bootstrap" \
--token=${TOKEN} \
--kubeconfig=bootstrap.kubeconfig 
[root@node2 cfg]# 
[root@node2 cfg]# /opt/kubernetes/bin/kubectl config set-context default \
--cluster=kubernetes \
--user="kubelet-bootstrap" \
--kubeconfig=bootstrap.kubeconfig
[root@node2 cfg]# /opt/kubernetes/bin/kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
[root@node2 cfg]# cat bootstrap.kubeconfig 
apiVersion: v1
clusters:
- cluster:
    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUR2akNDQXFhZ0F3SUJBZ0lVRTdyUWVKS2wrNGdxOXNPMzQvclBKdDRpZ21Rd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1pURUxNQWtHQTFVRUJoTUNRMDR4RURBT0JnTlZCQWdUQjBKbGFXcHBibWN4RURBT0JnTlZCQWNUQjBKbAphV3BwYm1jeEREQUtCZ05WQkFvVEEyczRjekVQTUEwR0ExVUVDeE1HVTNsemRHVnRNUk13RVFZRFZRUURFd3ByCmRXSmxjbTVsZEdWek1CNFhEVEl4TURjd016QTVORFl3TUZvWERUSTJNRGN3TWpBNU5EWXdNRm93WlRFTE1Ba0cKQTFVRUJoTUNRMDR4RURBT0JnTlZCQWdUQjBKbGFXcHBibWN4RURBT0JnTlZCQWNUQjBKbGFXcHBibWN4RERBSwpCZ05WQkFvVEEyczRjekVQTUEwR0ExVUVDeE1HVTNsemRHVnRNUk13RVFZRFZRUURFd3ByZFdKbGNtNWxkR1Z6Ck1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBcXdQcXFUM0RTZE1odVlGQUExVngKSFpkU1NBQ0pRVkN3Mmdvb2JHM0dTaGh3bmY3dzc2MDF0eE5NbUsraEpGNVRBWUNnS05PU2svQ3lUT0wwY0NuNgprcmQ3TEVwNkVzZzl6cXQzN2VaRzExN0MzRXZhR25lZmY0NHUyM2xJSUJQUEJMam84c2Myd2d4SFI4bGREbWlECng0bWhyY3VoVzBHdjluUllUbDdFR2Z0OHlFRFh4SXdFNDFhejgxR0dHN29jd0tZK3BoaUx4TjVkbWZEWFpCWU8KQmdmdEtBOHBjNU9QVVlrUHhXOUcyQStNdkhnTlhBa0xha2lrc0h1MGkxZUxobWQyRUhDeEttbVJLWW1ZUkYxYwpRcy9yd0ZQS1ZHRUpRUyt1NkVDbHZvTGpqZ1VPcE41Ty9aYWxIdjA0aUdkTGd5a2x3ZjJ3RlRqY1BwalYxSFJXCnBRSURBUUFCbzJZd1pEQU9CZ05WSFE4QkFmOEVCQU1DQVFZd0VnWURWUjBUQVFIL0JBZ3dCZ0VCL3dJQkFqQWQKQmdOVkhRNEVGZ1FVdzIzZWN2UzJ6VVNrY0pSOEtPcG83eVRZaytzd0h3WURWUjBqQkJnd0ZvQVV3MjNlY3ZTMgp6VVNrY0pSOEtPcG83eVRZaytzd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFIUzE0ZVB5dGJ2VHZMc0E0Z3d3CmZqR09MeElRNi9iQ2hESml1Q3RxUG5XNld0VC9NdHBqbVFKN0xlMlFFejVMMitLbERsSmdWcE9HWFdqSkY2NHoKZUFDTHhyYlM0TEhhVFZZdnBHM3V0by8rdVNQRkYwTlU3QUdGZnBOVEtMK0I0TTdidUVuYUFBSnpKQnNYSXhqeQowdDdPeEljUWNlN05IRTNTUkhlaHVPbkw1Ymk0K09KbkpwaVYzK1I4ZUsvU3g3ZTRrRXFLdG9mYmY3M0dhbE9vClQzZEh3TzVkeFNkT1B4SzFCbUtHb2pZUEdtUTNrSnM2Nmw0ZmxFZGpKV2prSDd3MWdXcTZHaGx1ZXRFVFlUTzIKQzRlUVpJSTJjZnRRTkh6Q2hJcHM1cXVoNGxackYzaStITHJpbjlxR3Y2N0YwREtwRlAwMXgzVkFTUkwxK2xHcgpobUE9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    server: https://192.168.24.136:6443
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: kubelet-bootstrap
  name: default
current-context: ""
kind: Config
preferences: {}
users:
- name: kubelet-bootstrap
  user:
    token: c47ffb939f5ca36231d9e3121a252940
[root@node2 cfg]# 

4. systemd 管理 kubelet

[root@node2 cfg]# vim /usr/lib/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet
After=docker.service

[Service]
EnvironmentFile=/opt/kubernetes/cfg/kubelet.conf
ExecStart=/opt/kubernetes/bin/kubelet $KUBELET_OPTS

Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

5. 启动并设置开机启动

[root@node2 cfg]# systemctl daemon-reload
[root@node2 cfg]# systemctl start kubelet
[root@node2 cfg]# systemctl enable kubelet

6.批准 kubelet 证书申请并加入集群

[root@node1 cfg]# kubectl get csr
[root@node1 cfg]# kubectl certificate approve node-csr-i-zpjwJ6gET9K_AN0-M8Wb7XpZxrqyBcNlUKGZLv35s
[root@node1 cfg]# kubectl get node
NAME    STATUS     ROLES    AGE   VERSION
node2   NotReady   <none>   80s   v1.19.0
[root@node1 cfg]# 
注:由于网络插件还没有部署,节点会没有准备就绪 NotReady

K8s安装部署K8s安装部署

九、部署 kube-proxy

1.创建kube-proxy.conf

[root@node2 cfg]# vim /opt/kubernetes/cfg/kube-proxy.conf
KUBE_PROXY_OPTS="--logtostderr=false \
--v=2 \
--log-dir=/opt/kubernetes/logs \
--config=/opt/kubernetes/cfg/kube-proxy-config.yml"

2.配置参数文件

[root@node2 cfg]# vim /opt/kubernetes/cfg/kube-proxy-config.yml
kind: KubeProxyConfiguration
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
metricsBindAddress: 0.0.0.0:10249
clientConnection:
        kubeconfig: /opt/kubernetes/cfg/kube-proxy.kubeconfig
hostnameOverride: node2
clusterCIDR: 10.0.0.0/24

3.生成 kube-proxy.kubeconfig

# master节点上生成
[root@node1 k8s]# cd /opt/TLS/k8s
[root@node1 k8s]# vim kube-proxy-csr.json 
{
	"CN": "system:kube-proxy",
	"hosts": [],
	"key": {
		"algo": "rsa",
		"size": 2048
	},
	"names": [
		{
			"C": "CN",
			"L": "BeiJing",
			"ST": "BeiJing",
			"O": "k8s",
			"OU": "System"
		}
	]
}

[root@node1 k8s]# /usr/local/cfssl/cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | /usr/local/cfssl/cfssljson -bare kube-proxy
[root@node1 k8s]# scp -r {kube-proxy-key.pem,kube-proxy.pem} root@node2:/opt/kubernetes/ssl/

4.生成kubeconfig

[root@node2 cfg]# export KUBE_APISERVER="https://192.168.24.136:6443"
[root@node2 cfg]# /opt/kubernetes/bin/kubectl config set-cluster kubernetes \
--certificate-authority=/opt/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kube-proxy.kubeconfig 

[root@node2 cfg]# /opt/kubernetes/bin/kubectl config set-credentials kube-proxy \
--client-certificate=/opt/kubernetes/ssl/kube-proxy.pem \
--client-key=/opt/kubernetes/ssl/kube-proxy-key.pem \
--embed-certs=true \
--kubeconfig=kube-proxy.kubeconfig 

[root@node2 cfg]# /opt/kubernetes/bin/kubectl config set-context default \
--cluster=kubernetes \
--user=kube-proxy \
--kubeconfig=kube-proxy.kubeconfig 

[root@node2 cfg]# /opt/kubernetes/bin/kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

5. systemd 管理 kube-proxy

[root@node2 cfg]# vim /usr/lib/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Proxy
After=network.target

[Service]
EnvironmentFile=/opt/kubernetes/cfg/kube-proxy.conf
ExecStart=/opt/kubernetes/bin/kube-proxy $KUBE_PROXY_OPTS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

6. 启动并设置开机启动

[root@node2 cfg]# systemctl daemon-reload
[root@node2 cfg]# systemctl start kube-proxy
[root@node2 cfg]# systemctl enable kube-proxy

十、部署 CNI 网络(Master Node)

master下载cni包,解压后,部署好网络插件,/opt/cni需要拷贝到其他节点上,其他节点也需要

[root@node1 opt]# wget https://github.com/containernetworking/plugins/releases/download/v0.8.6/cni-plugins-linux-amd64-v0.8.6.tgz
[root@node1 opt]# tar -zxvf cni-plugins-linux-amd64-v0.8.6.tgz -C /opt/cni/bin/
[root@node1 opt]# cd /opt/kubernetes/cfg 
[root@node1 cfg]# wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
#默认镜像地址无法访问,修改为 docker hub 镜像仓库。
[root@node1 cfg]# sed -i -r "s#quay.io/coreos/flannel:.*-amd64#lizhenliang/flannel:v0.12.0-amd64#g" kube-flannel.yml
[root@node1 cfg]# scp -r /opt/cni root@node2:/opt/
[root@node1 cfg]# kubectl apply -f kube-flannel.yml
[root@node1 cfg]# kubectl get pods -n kube-system
NAME                    READY   STATUS    RESTARTS   AGE
kube-flannel-ds-jcp6n   1/1     Running   0          6m22s
[root@node1 cfg]# kubectl get node
NAME    STATUS   ROLES    AGE   VERSION
node2   Ready    <none>   91m   v1.19.0
[root@node1 cfg]# 

K8s安装部署

1.授权 apiserver 访问 kubelet

[root@node1 cfg]# vim apiserver-to-kubelet-rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata: 
        annotations: 
                rbac.authorization.kubernetes.io/autoupdate: "true" 
        labels: 
                kubernetes.io/bootstrapping: rbac-defaults 
        name: system:kube-apiserver-to-kubelet 
rules: 
        - apiGroups:
                - "" 
          resources: 
                - nodes/proxy 
                - nodes/stats 
                - nodes/log 
                - nodes/spec 
                - nodes/metrics 
                - pods/log 
          verbs: 
            - "*" 
--- 
apiVersion: rbac.authorization.k8s.io/v1 
kind: ClusterRoleBinding 
metadata: 
        name: system:kube-apiserver 
        namespace: "" 
roleRef: 
        apiGroup: rbac.authorization.k8s.io
        kind: ClusterRole 
        name: system:kube-apiserver-to-kubelet 
subjects:
        - apiGroup: rbac.authorization.k8s.io 
        kind: User 
        name: kubernetes

十一、部署node3节点

1. 拷贝已部署好的 Node 相关文件到新节点

[root@node2 opt]# scp -r /opt/kubernetes root@node3:/opt/
[root@node2 opt]# scp -r /usr/lib/systemd/system/{kubelet,kube-proxy}.service root@node3:/usr/lib/systemd/system/

2. 删除 kubelet 证书和 kubeconfig 文件

[root@node3 cfg]# rm -rf /opt/kubernetes/cfg/kubelet.kubeconfig /opt/kubernetes/ssl/kubelet*
[root@node3 cfg]# 

注:这几个文件是证书申请审批后自动生成的,每个 Node 不同,必须删除重新生成。

3. 修改主机名

[root@node3 cfg]# vim /opt/kubernetes/cfg/kubelet.conf 
--hostname-override=node3
[root@node3 cfg]# vim /opt/kubernetes/cfg/kube-proxy-config.yml 
hostnameOverride: node3
[root@node3 cfg]# 

4. 启动并设置开机启动

[root@node3 cfg]# systemctl daemon-reload
[root@node3 cfg]# systemctl start kubelet
[root@node3 cfg]# systemctl enable kubelet
[root@node3 cfg]# systemctl start kube-proxy
[root@node3 cfg]# systemctl enable kube-proxy

5. 在 Master 上批准新 Node kubelet 证书申请

[root@node1 cfg]# kubectl certificate approve node-csr-V5KygJs8u7OWvFB2QI2FyiqdLJNwVpDyHnBhVmrC4oQ
certificatesigningrequest.certificates.k8s.io/node-csr-V5KygJs8u7OWvFB2QI2FyiqdLJNwVpDyHnBhVmrC4oQ approved

十二、验证测试

[root@node1 logs]# kubectl create deployment nginx --image=nginx
deployment.apps/nginx created
[root@node1 logs]# kubectl expose deployment nginx --port=80 --type=NodePort
service/nginx exposed

K8s安装部署

总结说明

1.kubectl get node 不是Ready状态

K8s安装部署
我这里的问题是,/opt/cni/ 插件从节点机器上没有部署

上一篇:【蓝牙数据采集模块】-02-Sensor Controller 编写CC2650STK外设驱动


下一篇:最新二进制k8s安装