下载安装包
wget https://dl.k8s.io/v1.23.0-rc.0/kubernetes-server-linux-amd64.tar.gz
解压并拷贝二进制程序到环境变量目录
tar xf kubernetes-server-linux-amd64.tar.gz cd kubernetes/server/bin cp kube-apiserver kube-controller-manager kube-scheduler kubectl /usr/local/bin/
分发其他节点
scp kube-apiserver kube-controller-manager kube-scheduler kubectl master-2:/usr/local/bin/ scp kube-apiserver kube-controller-manager kube-scheduler kubectl master-2:/usr/local/bin/
所有节点创建配置文件目录及日志目录
mkdir -p /etc/kubernetes/ssl mkdir /var/log/kubernetes
#启动TLS Bootstrapping 机制
Master apiserver启用TLS认证后，每个节点的 kubelet 组件都要使用由 apiserver 使用的 CA 签发的有效证书才能与 apiserver 通讯，当Node节点很多时，这种客户端证书颁发需要大量工作，同样也会增加集群扩展复杂度。
为了简化流程，Kubernetes引入了TLS bootstraping机制来自动颁发客户端证书，kubelet会以一个低权限用户自动向apiserver申请证书，kubelet的证书由apiserver动态签署。
Bootstrap 是很多系统中都存在的程序，比如 Linux 的bootstrap，bootstrap 一般都是作为预先配置在开启或者系统启动的时候加载，这可以用来生成一个指定环境。Kubernetes 的 kubelet 在启动时同样可以加载一个这样的配置文件，这个文件的内容类似如下形式：
apiVersion: v1
clusters: null
contexts:
- context:
cluster: kubernetes
user: kubelet-bootstrap
name: default
current-context: default
kind: Config
preferences: {}
users:
- name: kubelet-bootstrap
user: {}
#TLS bootstrapping 具体引导过程
1.TLS 作用
TLS 的作用就是对通讯加密，防止中间人窃听；同时如果证书不信任的话根本就无法与 apiserver 建立连接，更不用提有没有权限向apiserver请求指定内容。
2. RBAC 作用
当 TLS 解决了通讯问题后，那么权限问题就应由 RBAC 解决(可以使用其他权限模型，如 ABAC)；RBAC 中规定了一个用户或者用户组(subject)具有请求哪些 api 的权限；在配合 TLS 加密的时候，实际上 apiserver 读取客户端证书的 CN 字段作为用户名，读取 O字段作为用户组.
以上说明：第一，想要与 apiserver 通讯就必须采用由 apiserver CA 签发的证书，这样才能形成信任关系，建立 TLS 连接；第二，可以通过证书的 CN、O 字段来提供 RBAC 所需的用户与用户组。
#kubelet 首次启动流程
TLS bootstrapping 功能是让 kubelet 组件去 apiserver 申请证书，然后用于连接 apiserver；那么第一次启动时没有证书如何连接 apiserver ?
在apiserver 配置中指定了一个 token.csv 文件，该文件中是一个预设的用户配置；同时该用户的Token 和 由apiserver 的 CA签发的用户被写入了 kubelet 所使用的 bootstrap.kubeconfig 配置文件中；这样在首次请求时，kubelet 使用 bootstrap.kubeconfig 中被 apiserver CA 签发证书时信任的用户来与 apiserver 建立 TLS 通讯，使用 bootstrap.kubeconfig 中的用户 Token 来向 apiserver 声明自己的 RBAC 授权身份.
token.csv格式:
3940fd7fbb391d1b4d861ad17a1f0613,kubelet-bootstrap,10001,"system:kubelet-bootstrap"
首次启动时，可能与遇到 kubelet 报 401 无权访问 apiserver 的错误；这是因为在默认情况下，kubelet 通过 bootstrap.kubeconfig 中的预设用户 Token 声明了自己的身份，然后创建 CSR 请求；但是不要忘记这个用户在我们不处理的情况下他没任何权限的，包括创建 CSR 请求；所以需要创建一个 ClusterRoleBinding，将预设用户 kubelet-bootstrap 与内置的 ClusterRole system:node-bootstrapper 绑定到一起，使其能够发起 CSR 请求。稍后安装kubelet的时候演示。
#创建token.csv文件
[root@master-1 work]# pwd /data/work [root@master-1 work]# ls ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem etcd.csr etcd-csr.json etcd-key.pem etcd.pem [root@master-1 work]# cat > token.csv << EOF > $(head -c 16 /dev/urandom | od -An -t x | tr -d ' '),kubelet-bootstrap,10001,"system:kubelet-bootstrap" > EOF [root@master-1 work]# cat token.csv 7dbeda43ed70c5db077891df43115dca,kubelet-bootstrap,10001,"system:kubelet-bootstrap"
#创建csr请求文件
[root@master-1 work]# cat kube-apiserver-csr.json
{
"CN": "kubernetes",
"hosts": [
"127.0.0.1",
"192.168.10.28",
"192.168.10.29",
"192.168.10.30",
"192.168.10.31",
"10.255.0.1",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Hebei",
"L": "shijiazhuang",
"O": "k8s",
"OU": "system"
}
]
}
将证书文件复制对应目录
[root@master-1 work]# ls ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem etcd.csr etcd-csr.json etcd-key.pem etcd.pem kube-apiserver.csr kube-apiserver-csr.json kube-apiserver-key.pem kube-apiserver.pem token.csv [root@master-1 work]# cp ca*.pem kube-apiserver*.pem /etc/kubernetes/ssl/ 您在 /var/spool/mail/root 中有新邮件 [root@master-1 work]# ls ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem etcd.csr etcd-csr.json etcd-key.pem etcd.pem kube-apiserver.csr kube-apiserver-csr.json kube-apiserver-key.pem kube-apiserver.pem token.csv [root@master-1 work]# cp token.csv /etc/kubernetes/
创建api 配置文件
vim kube-apiserver.conf KUBE_APISERVER_OPTS="--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \ --anonymous-auth=false \ --bind-address=192.168.10.29 \ --secure-port=6443 \ --advertise-address=192.168.10.29 \ --insecure-port=0 \ --authorization-mode=Node,RBAC \ --runtime-config=api/all=true \ --enable-bootstrap-token-auth \ --service-cluster-ip-range=10.255.0.0/16 \ --token-auth-file=/etc/kubernetes/token.csv \ --service-node-port-range=30000-50000 \ --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem \ --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem \ --client-ca-file=/etc/kubernetes/ssl/ca.pem \ --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem \ --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem \ --service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \ --service-account-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \ --service-account-issuer=https://kubernetes.default.svc.cluster.local \ --etcd-cafile=/etc/etcd/ssl/ca.pem \ --etcd-certfile=/etc/etcd/ssl/etcd.pem \ --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \ --etcd-servers=https://192.168.10.29:2379,https://192.168.10.30:2379,https://192.168.10.31:2379 \ --enable-swagger-ui=true \ --allow-privileged=true \ --apiserver-count=3 \ --audit-log-maxage=30 \ --audit-log-maxbackup=3 \ --audit-log-maxsize=100 \ --audit-log-path=/var/log/kube-apiserver-audit.log \ --event-ttl=1h \ --alsologtostderr=true \ --logtostderr=false \ --log-dir=/var/log/kubernetes \ --v=4"
#注：
--logtostderr：启用日志
--v：日志等级
--log-dir：日志目录
--etcd-servers：etcd集群地址
--bind-address：监听地址
--secure-port：https安全端口
--advertise-address：集群通告地址
--allow-privileged：启用授权
--service-cluster-ip-range：Service虚拟IP地址段
--enable-admission-plugins：准入控制模块
--authorization-mode：认证授权，启用RBAC授权和节点自管理
--enable-bootstrap-token-auth：启用TLS bootstrap机制
--token-auth-file：bootstrap token文件
--service-node-port-range：Service nodeport类型默认分配端口范围
--kubelet-client-xxx：apiserver访问kubelet客户端证书
--tls-xxx-file：apiserver https证书
--etcd-xxxfile：连接Etcd集群证书 –
-audit-log-xxx：审计日志
创建启动文件
[root@master-1 work]# vim kube-apiserver.service [Unit] Description=Kubernetes API Server Documentation=https://github.com/kubernetes/kubernetes After=etcd.service Wants=etcd.service [Service] EnvironmentFile=-/etc/kubernetes/kube-apiserver.conf ExecStart=/usr/local/bin/kube-apiserver $KUBE_APISERVER_OPTS Restart=on-failure RestartSec=5 Type=notify LimitNOFILE=65536 [Install] WantedBy=multi-user.target
分发配置文件与启动文件
[root@master-1 work]# scp -r /etc/kubernetes master-2:/etc/ ca-key.pem 100% 1675 440.6KB/s 00:00 ca.pem 100% 1346 429.8KB/s 00:00 kube-apiserver-key.pem 100% 1679 557.7KB/s 00:00 kube-apiserver.pem 100% 1635 478.7KB/s 00:00 token.csv 100% 84 25.0KB/s 00:00 kube-apiserver.conf 100% 1611 1.2MB/s 00:00 [root@master-1 work]# scp -r /etc/kubernetes master-3:/etc/ ca-key.pem 100% 1675 911.5KB/s 00:00 ca.pem 100% 1346 765.9KB/s 00:00 kube-apiserver-key.pem 100% 1679 990.1KB/s 00:00 kube-apiserver.pem 100% 1635 914.3KB/s 00:00 token.csv 100% 84 69.5KB/s 00:00 kube-apiserver.conf 100% 1611 1.1MB/s 00:00 [root@master-1 work]# scp /usr/lib/systemd/system/kube-apiserver.service master-2:/usr/lib/systemd/system/ kube-apiserver.service 100% 361 203.5KB/s 00:00 您在 /var/spool/mail/root 中有新邮件 [root@master-1 work]# scp /usr/lib/systemd/system/kube-apiserver.service master-2:/usr/lib/systemd/system/ kube-apiserver.service 100% 361 242.1KB/s 00:00 [root@master-1 work]# scp /usr/lib/systemd/system/kube-apiserver.service master-3:/usr/lib/systemd/system/ kube-apiserver.service
修改master-2与master-3的配置文件的IP
vim /etc/kubernetes/kube-apiserver.conf KUBE_APISERVER_OPTS="--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \ --anonymous-auth=false \ --bind-address=192.168.10.30 \ --secure-port=6443 \ --advertise-address=192.168.10.30 \ --insecure-port=0 \ --authorization-mode=Node,RBAC \ --runtime-config=api/all=true \ --enable-bootstrap-token-auth \ --service-cluster-ip-range=10.255.0.0/16 \ --token-auth-file=/etc/kubernetes/token.csv \ --service-node-port-range=30000-50000 \ --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem \ --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem \ --client-ca-file=/etc/kubernetes/ssl/ca.pem \ --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem \ --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem \ --service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \ --service-account-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \ --service-account-issuer=https://kubernetes.default.svc.cluster.local \ --etcd-cafile=/etc/etcd/ssl/ca.pem \ --etcd-certfile=/etc/etcd/ssl/etcd.pem \ --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \ --etcd-servers=https://192.168.10.29:2379,https://192.168.10.30:2379,https://192.168.10.31:2379 \ --enable-swagger-ui=true \ --allow-privileged=true \ --apiserver-count=3 \ --audit-log-maxage=30 \ --audit-log-maxbackup=3 \ --audit-log-maxsize=100 \ --audit-log-path=/var/log/kube-apiserver-audit.log \ --event-ttl=1h \ --alsologtostderr=true \ --logtostderr=false \ --log-dir=/var/log/kubernetes \ --v=4"
与master-3的配置文件的IP
root@master-3 modules]# vim /etc/kubernetes/kube-apiserver.conf KUBE_APISERVER_OPTS="--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \ --anonymous-auth=false \ --bind-address=192.168.10.31 \ --secure-port=6443 \ --advertise-address=192.168.10.31 \ --insecure-port=0 \ --authorization-mode=Node,RBAC \ --runtime-config=api/all=true \ --enable-bootstrap-token-auth \ --service-cluster-ip-range=10.255.0.0/16 \ --token-auth-file=/etc/kubernetes/token.csv \ --service-node-port-range=30000-50000 \ --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem \ --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem \ --client-ca-file=/etc/kubernetes/ssl/ca.pem \ --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem \ --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem \ --service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \ --service-account-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \ --service-account-issuer=https://kubernetes.default.svc.cluster.local \ --etcd-cafile=/etc/etcd/ssl/ca.pem \ --etcd-certfile=/etc/etcd/ssl/etcd.pem \ --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \ --etcd-servers=https://192.168.10.29:2379,https://192.168.10.30:2379,https://192.168.10.31:2379 \ --enable-swagger-ui=true \ --allow-privileged=true \ --apiserver-count=3 \ --audit-log-maxage=30 \ --audit-log-maxbackup=3 \ --audit-log-maxsize=100 \ --audit-log-path=/var/log/kube-apiserver-audit.log \ --event-ttl=1h \ --alsologtostderr=true \ --logtostderr=false \ --log-dir=/var/log/kubernetes \ --v=4"
启动
[root@master-3 modules]# systemctl daemon-reload
[root@master-3 modules]# systemctl enable kube-apiserver
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-apiserver.service to /usr/lib/systemd/system/kube-apiserver.service.
[root@master-3 modules]# systemctl start kube-apiserver
您在 /var/spool/mail/root 中有新邮件
[root@master-3 modules]# systemctl status kube-apiserver
● kube-apiserver.service - Kubernetes API Server
Loaded: loaded (/usr/lib/systemd/system/kube-apiserver.service; enabled; vendor preset: disabled)
Active: active (running) since 五 2022-01-14 10:15:17 CST; 16s ago
Docs: https://github.com/kubernetes/kubernetes
Main PID: 26553 (kube-apiserver)
Tasks: 11
Memory: 258.7M
CGroup: /system.slice/kube-apiserver.service
└─26553 /usr/local/bin/kube-apiserver --enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota --anonymous-auth=false --bind-addr...
1月 14 10:15:18 master-3 kube-apiserver[26553]: W0114 10:15:18.756424 26553 lease.go:233] Resetting endpoints for master service "kubernetes" to [192.168.10.29 192.168.10.30 192.168.10.31]
1月 14 10:15:18 master-3 kube-apiserver[26553]: I0114 10:15:18.758353 26553 controller.go:611] quota admission added evaluator for: endpoints
1月 14 10:15:18 master-3 kube-apiserver[26553]: I0114 10:15:18.760229 26553 httplog.go:129] "HTTP" verb="PUT" URI="/api/v1/namespaces/default/endpoints/kubernetes" latency="3.416816ms" userA...088" resp=200
1月 14 10:15:18 master-3 kube-apiserver[26553]: I0114 10:15:18.762632 26553 httplog.go:129] "HTTP" verb="GET" URI="/apis/discovery.k8s.io/v1/namespaces/default/endpointslices/kubernetes" latency="1.84777...
1月 14 10:15:18 master-3 kube-apiserver[26553]: I0114 10:15:18.763525 26553 controller.go:611] quota admission added evaluator for: endpointslices.discovery.k8s.io
1月 14 10:15:18 master-3 kube-apiserver[26553]: I0114 10:15:18.765955 26553 httplog.go:129] "HTTP" verb="PUT" URI="/apis/discovery.k8s.io/v1/namespaces/default/endpointslices/kubernetes" latency="3.00586...
1月 14 10:15:28 master-3 kube-apiserver[26553]: I0114 10:15:28.747329 26553 httplog.go:129] "HTTP" verb="GET" URI="/api/v1/namespaces/default" latency="2.212795ms" userAgent="kube-apiserver/...088" resp=200
1月 14 10:15:28 master-3 kube-apiserver[26553]: I0114 10:15:28.749026 26553 httplog.go:129] "HTTP" verb="GET" URI="/api/v1/namespaces/default/services/kubernetes" latency="1.384208ms" userAg...088" resp=200
1月 14 10:15:28 master-3 kube-apiserver[26553]: I0114 10:15:28.755391 26553 httplog.go:129] "HTTP" verb="GET" URI="/api/v1/namespaces/default/endpoints/kubernetes" latency="1.578857ms" userA...088" resp=200
1月 14 10:15:28 master-3 kube-apiserver[26553]: I0114 10:15:28.758525 26553 httplog.go:129] "HTTP" verb="GET" URI="/apis/discovery.k8s.io/v1/namespaces/default/endpointslices/kubernetes" latency="1.56392...
Hint: Some lines were ellipsized, use -l to show in full.
部署kubectl组件
Kubectl是客户端工具，操作k8s资源的，如增删改查等。
Kubectl操作资源的时候，怎么知道连接到哪个集群，需要一个文件/etc/kubernetes/admin.conf，kubectl会根据这个文件的配置，去访问k8s资源。/etc/kubernetes/admin.con文件记录了访问的k8s集群，和用到的证书。
#创建csr请求文件
{
"CN": "admin",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Hubei",
"L": "Wuhan",
"O": "system:masters",
"OU": "system"
}
]
}
#生成证书
[root@master-1 work]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin
2022/01/15 08:28:44 [INFO] generate received request
2022/01/15 08:28:44 [INFO] received CSR
2022/01/15 08:28:44 [INFO] generating key: rsa-2048
2022/01/15 08:28:44 [INFO] encoded CSR
2022/01/15 08:28:44 [INFO] signed certificate with serial number 381703224104814716044969405758384157623879899724
2022/01/15 08:28:44 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
证书拷贝对应目录‘
[root@master-1 work]# cp admin*.pem /etc/kubernetes/ssl/
#创建kubeconfig配置文件，比较重要
1.设置集群参数
[root@master-1 work]# kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true --server=https://192.168.10.29:6443 --kubeconfig=kube.config Cluster "kubernetes" set. 您在 /var/spool/mail/root 中有新邮件 [root@master-1 work]# kubectl config set-credentials admin --client-certificate=admin.pem --client-key=admin-key.pem --embed-certs=true --kubeconfig=kube.config User "admin" set. [root@master-1 work]# kubectl config set-context kubernetes --cluster=kubernetes --user=admin --kubeconfig=kube.config Context "kubernetes" created. [root@master-1 work]# kubectl config use-context kubernetes --kubeconfig=kube.config Switched to context "kubernetes". [root@master-1 work]# mkdir ~/.kube -p 您在 /var/spool/mail/root 中有新邮件 [root@master-1 work]# cp kube.config ~/.kube/config [root@master-1 work]# kubectl create clusterrolebinding kube-apiserver:kubelet-apis --clusterrole=system:kubelet-api-admin --user kubernetes clusterrolebinding.rbac.authorization.k8s.io/kube-apiserver:kubelet-apis created
测试
[root@master-1 work]# kubectl cluster-info
Kubernetes control plane is running at https://192.168.10.29:6443
To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'.
[root@master-1 work]# kubectl get componentstatuses
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME STATUS MESSAGE ERROR
controller-manager Unhealthy Get "https://127.0.0.1:10257/healthz": dial tcp 127.0.0.1:10257: connect: connection refused
scheduler Unhealthy Get "https://127.0.0.1:10259/healthz": dial tcp 127.0.0.1:10259: connect: connection refused
etcd-0 Healthy {"health":"true","reason":""}
etcd-2 Healthy {"health":"true","reason":""}
etcd-1 Healthy {"health":"true","reason":""}
您在 /var/spool/mail/root 中有新邮件
[root@master-1 work]#
同步其他节点
[root@master-1 ~]# scp -r .kube master-2:/root [root@master-1 ~]# scp -r .kube master-3:/root
设置自动补全
yum install -y bash-completion&& source /usr/share/bash-completion/bash_completion&& source <(kubectl completion bash)&& kubectl completion bash > ~/.kube/completion.bash.inc&&source '/root/.kube/completion.bash.inc' &&source $HOME/.bash_profile