本文采用3个master方式搭建k8s,所使用的应用版本如下：
etcd：v3.4.13
kubernetes：v1.19.0
docker：v20.10.0
flanneld
二进制安装Kubernetes/k8s
安装etcd
#生成证书
#下载cfssl
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -O cfssl -P /usr/local/bin/
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -O cfssljson -P /usr/local/bin/
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -O cfssl-certinfo -P /usr/local/bin/
chmod +x /usr/local/bin/cfssl*
mkdir /root/ssl #创建一个目录，用于存放证书
#生成证书
cat > /root/ssl/ca-config.json << EOF
{
"signing": {
"default": {
"expiry": "43800h"
},
"profiles": {
"server": {
"expiry": "43800h",
"usages": [
"signing",
"key encipherment",
"server auth"
]
},
"client": {
"expiry": "43800h",
"usages": [
"signing",
"key encipherment",
"client auth"
]
},
"etcd": {
"expiry": "43800h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF
cat > /root/ssl/ca-csr.json << EOF
{
"CN": "etcd",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "etcd",
"OU": "System"
}
]
}
EOF
cfssl gencert -initca ca-csr.json | cfssljson -bare ca #生成根证书
ca.csr、ca-key.pem、ca.pem #生成三个文件，保存备用
cat > /root/ssl/etcd-csr.json << EOF
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"192.168.75.129",
"192.168.75.130",
"192.168.75.131",
"localhost"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "etcd",
"OU": "System"
}
]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=etcd etcd-csr.json | cfssljson -bare etcd #生成etcd证书
etcd.pem etcd-key.pem #生成的etcd证书
mkdir /etc/kubernetes/pki -p #创建文件夹，用于存放证书和kubernetes相关配置文件
mkdir -p /etc/etcd/pki #创建文件夹，存放etcd服务及etcd证书
cp ca.csr ca-key.pem ca.pem /etc/kubernetes/pki #将生成的根证书拷贝到/etc/kubernetes/pki
cp etcd.pem etcd-key.pem /etc/etcd/pki #将生成的etcd证书拷贝到/etc/etcd/pki
安装etcd
wget https://github.com/etcd-io/etcd/releases/download/v3.4.13/etcd-v3.4.13-linux-amd64.tar.gz
tar zxvf etcd-v3.4.13-linux-amd64.tar.gz
cp etcd-v3.4.13-linux-amd64/etcd* /usr/bin #将etcd工具拷贝到/usr/bin
#为etcd添加systemd启动
cat > /usr/lib/systemd/syste/etcd.service << EOF
[Unit]
Description=etcd key-value store
[Unit]
Description=etcd key-value store
Documentation=https://github.com/etcd-io/etcd
After=network.target
[Service]
EnvironmentFile=/etc/etcd/etcd.conf
ExecStart=/usr/bin/etcd
Restart=always
[Install]
WantedBy=multi-user.target
EOF
#添加etcd配置文件
cat > /etc/etcd/etcd.conf << EOF
ETCD_NAME=etcd1 #分别设置etcd1，etcd2，etcd3
ETCD_DATA_DIR="/etc/etcd/data"
ETCD_CERT_FILE="etc/etcd/pki/etcd.pem"
ETCD_KEY_FILE="/etc/etcd/pki/etcd-key.pem"
ETCD_TRUSTED_CA_FILE="/etc/kubernetes/pki/ca.pem"
ETCD_CLIENT_CERT_AUTH="true"
ETCD_ENABLE_V2=true
ETCD_LISTEN_CLIENT_URLS="https://192.168.75.129:2379" #修改当前master IP地址
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.75.129:2379" #修改当前master IP地址
ETCD_PEER_CERT_FILE="etc/etcd/pki/etcd.pem"
ETCD_PEER_KEY_FILE="/etc/etcd/pki/etcd-key.pem"
ETCD_PEER_TRUSTED_CA_FILE="/etc/kubernetes/pki/ca.pem"
ETCD_LISTEN_PEER_URLS="https://192.168.75.129:2380" #修改当前master IP地址
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.75.129:2380" #修改当前master IP地址
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER="etcd1=https://192.168.75.129:2380,etcd2=https://192.168.75.130:2380,etcd3=https://192.168.75.131:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF
启动etcd
systemctl daemon-reload
systemctl start etcd.service
携带etcd证书验证集群是否正常
etcdctl --cacert=/etc/kubernetes/pki/ca.pem --cert=/etc/etcd/pki/etcd.pem --key=/etc/etcd/pki/etcd-key.pem --endpoints=https://192.168.75.129:2379,https://192.168.75.130:2379,https://192.168.75.131:2379 endpoint health
安装kubernetes
安装kube-apiserver
#下载kubernetes二进制包
wget https://dl.k8s.io/v1.19.0/kubernetes-server-linux-amd64.tar.gz
tar zxvf kubernetes-server-linux-amd64.tar.gz
cd kubernetes/server/bin
cp kube-apiserver kube-controller-manager kube-scheduler kubeadm kube-aggregator kubelet kubectl kube-proxy /usr/bin/
生成apiserver证书
cat > /root/ssl/api-config.json << EOF
{
"signing": {
"default": {
"expiry": "8760h"
},
"profiles": {
"kubernetes": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "8760h"
}
}
}
}
EOF
cat > /root/ssl/apiserver-csr.json << EOF
{
"CN": "kubernetes",
"hosts": [ #host处写明master的IP、主机名等
"127.0.0.1",
"192.168.75.129",
"192.168.75.130",
"192.168.75.131",
"192.168.75.100",
"169.169.0.1",
"k8s-1",
"k8s-2",
"k8s-3",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "kuai",
"OU": "kuai"
}]
}
EOF
生成apiserver证书
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem --config=api-config.json -profile=kubernetes apiserver-csr.json | cfssljson -bare server
server-key.pem server.pem server.csr #生成三个证书
#将生成的证书拷贝到/etc/kubernetes/pki/
cp server-key.pem server.pem server.csr /etc/kubernetes/k8s/pki/
将apiserver添加到systemd启动、创建apiserver配置文件和kube
#添加apiserver到systemd启动
cat > /usr/lib/systemd/system/kube-apiserver.service << EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=/etc/kubernetes/apiserver
ExecStart=/usr/bin/kube-apiserver \$KUBE_API_ARGS
Restart=always
[Install]
WantedBy=multi-user.target
EOF
#创建apiserver配置文件
cat > /etc/kubernetes/apiserver << EOF
KUBE_API_ARGS="--secure-port=6443 \
--tls-cert-file=/etc/kubernetes/k8s/pki/server.pem \
--tls-private-key-file=/etc/kubernetes/k8s/pki/server-key.pem \
--client-ca-file=/etc/kubernetes/k8s/pki/ca.pem \
--apiserver-count=3 --endpoint-reconciler-type=master-count \
--etcd-servers=https://192.168.75.129:2379,https://192.168.75.130;2379,https://192.168.75.131:2379 \
--etcd-cafile=/etc/kubernetes/pki/ca.pem \
--etcd-certfile=/etc/etcd/pki/etcd.pem \
--etcd-keyfile=/etc/etcd/pki/etcd-key.pem \
--service-cluster-ip-range=169.169.0.1/16 \
--service-node-port-range=30000-32767 \
--allow-privileged=true \
--logtostderr=false --log-dir=/var/log/kubernetes --v=0"
EOF
#创建客户端链接kube-apiserver服务所需的kubeconfig配置文件
cat > /etc/kubernetes/kubeconfig << EOF
apiVersion: v1
kind: Config
clusters:
- name: default
cluster:
server: https://192.168.75.100:9443 #VIP地址
certificate-authority: /etc/kubernetes/pki/ca.pem
users:
- name: admin
user:
client-certificate: /etc/kubernetes/pki/client.crt
client-key: /etc/kubernetes/pki/client.key
contexts:
- context:
cluster: default
user: admin
name: default
current-context: default
EOF
启动apiserver
systemctl daemon-reload
systemctl start kube-apiserver.service
安装controller和scheduler
生成controller和scheduler使用的证书
openssl genrsa -out client.key 2048
openssl req -new -key client.key -subj "/CN=admin" -out client.csr
openssl x509 -req -in client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out client.crt -days 36500
cp client.crt client.key /etc/kubernetes/pki/ #拷贝证书到/etc/kubernetes/pki/
将controller添加到systemd启动
cat > /usr/lib/systemd/system/kube-contrller-manager.service << EOF
[Unit]
Description=Kubernetes Controller Server
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=/etc/kubernetes/controller-manager
ExecStart=/usr/bin/kube-controller-manager $KUBE_CONTROLLER_MANAGER_ARGS
Restart=always
[Install]
WantedBy=multi-user.target
EOF
#配置controller启动参数
cat > /etc/kubernetes/controller-manager << EOF
KUBE_CONTROLLER_MANAGER_ARGS="--kubeconfig=/etc/kubernetes/kubeconfig \
--leader-elect=true \
--service-cluster-ip-range=169.169.0.0/16 \
--service-account-private-key-file=/etc/kubernetes/k8s/pki/server-key.pem \
--root-ca-file=/etc/kubernetes/k8s/pki/ca.pem \
--log-dir=/var/log/kubernetes --logtostderr=false --v=0"
EOF
配置scheduler启动
# 将scheduler添加到systemd启动
cat > /usr/lib/systemd/system/kube-scheduler.service << EOF
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=/etc/kubernetes/scheduler
ExecStart=/usr/bin/kube-scheduler $KUBE_SCHEDULER_ARGS
Restart=always
[Install]
WantedBy=multi-user.target
EOF
#添加schduler启动文件
cat > /etc/kubernetes/scheduler << EOF
KUBE_SCHEDULER_ARGS="--kubeconfig=/etc/kubernetes/kubeconfig \
--leader-elect=true \
--logtostderr=false --log-dir=/var/log/kubernetes --v=0"
EOF