0. 环境
本次演示环境使用metersphere版本：v1.13.0
设置k8s节点包括1台master和1台worker；
应用部署控制管理器使用deployment；
持久化存储使用hostPath写入到NFS挂载目录。
NFS挂载方式如下：
在master配置了nfs服务，并将nfs共享目录/nfs_storage挂在到了两个节点中的/opt/nfs;
如下图所示：
1. 导入镜像
将MeterSphere安装包中的镜像一次导入到系统中，或导入到harbor中；
此次演示环境没有配置harbor，直接导入到两个k8s节点中：
cd /opt/metersphere-offline-installer-v1.13.0/
for i in `ls images`; do docker load < $i; done
2. 创建共享目录
mkdir /opt/nfs/metersphere/
#将conf目录提前拷贝到共享目录中
cp -r /opt/metersphere-offline-installer-v1.13.0/metersphere/conf /opt/nfs/metersphere/
3. 创建应用
3.1 创建MySQL
kubectl create -f mysql-configmap.yaml
#mysql-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: ms-mysql-config
namespace: ms
data:
my.cnf: |
[mysqld]
datadir=/var/lib/mysql
default-storage-engine=INNODB
character_set_server=utf8mb4
lower_case_table_names=1
table_open_cache=128
max_connections=2000
max_connect_errors=6000
innodb_file_per_table=1
innodb_buffer_pool_size=1G
max_allowed_packet=64M
transaction_isolation=READ-COMMITTED
innodb_flush_method=O_DIRECT
innodb_lock_wait_timeout=1800
innodb_flush_log_at_trx_commit=0
sync_binlog=0
server-id=1
log-bin=mysql-bin
expire_logs_days = 2
binlog_format=mixed
sql_mode=STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION
skip-name-resolve
[mysql]
default-character-set=utf8mb4
[mysql.server]
default-character-set=utf8mb4
kubectl create -f mysql-deploy.yaml
#mysql-deploy.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: mysql
namespace: ms
spec:
selector:
matchLabels:
app: mysql
replicas: 1
template:
metadata:
labels:
app: mysql
spec:
containers:
- name: mysql
image: registry.cn-qingdao.aliyuncs.com/metersphere/mysql:5.7.33
imagePullPolicy: IfNotPresent
env:
- name: MYSQL_DATABASE
value: metersphere
- name: MYSQL_ROOT_PASSWORD
value: "Password123@mysql"
ports:
- containerPort: 3306
protocol: TCP
volumeMounts:
- name: mysql-persistent-storage
mountPath: /var/lib/mysql
- name: config-volume
mountPath: /etc/mysql/conf.d
volumes:
- name: config-volume
configMap:
name: ms-mysql-config
- name: mysql-persistent-storage
hostPath:
path: /opt/nfs/metersphere/data/mysql
---
apiVersion: v1
kind: Service
metadata:
name: mysql
namespace: ms
spec:
ports:
- name: mysql
protocol: TCP
port: 3306
targetPort: 3306
nodePort: 31306
type: NodePort
selector:
app: mysql
3.2 创建zookeeper
kubectl create -f zookeeper-deploy.yaml
#zookeeper-deploy.yaml
kind: Deployment
apiVersion: apps/v1
metadata:
name: zookeeper-deployment-1
namespace: ms
spec:
replicas: 1
selector:
matchLabels:
app: zookeeper-1
name: zookeeper-1
template:
metadata:
labels:
app: zookeeper-1
name: zookeeper-1
spec:
containers:
- name: zoo1
image: uhub.service.ucloud.cn/metersphere/zookeeper:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 2181
env:
- name: ZOOKEEPER_ID
value: "1"
- name: ZOOKEEPER_SERVER_1
value: zoo1
- name: ZOOKEEPER_SERVER_2
value: zoo2
- name: ZOOKEEPER_SERVER_3
value: zoo3
volumeMounts:
- name: ms-zookeeper1-data
mountPath: /var/lib/zookeeper
volumes:
- name: ms-zookeeper1-data
hostPath:
path: /opt/nfs/metersphere/data/zookeeper/zookeeper1
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: zookeeper-deployment-2
namespace: ms
spec:
replicas: 1
selector:
matchLabels:
app: zookeeper-2
name: zookeeper-2
template:
metadata:
labels:
app: zookeeper-2
name: zookeeper-2
spec:
containers:
- name: zoo2
image: uhub.service.ucloud.cn/metersphere/zookeeper:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 2181
env:
- name: ZOOKEEPER_ID
value: "2"
- name: ZOOKEEPER_SERVER_1
value: zoo1
- name: ZOOKEEPER_SERVER_2
value: zoo2
- name: ZOOKEEPER_SERVER_3
value: zoo3
volumeMounts:
- name: ms-zookeeper2-data
mountPath: /var/lib/zookeeper
volumes:
- name: ms-zookeeper2-data
hostPath:
path: /opt/nfs/metersphere/data/zookeeper/zookeeper2
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: zookeeper-deployment-3
namespace: ms
spec:
replicas: 1
selector:
matchLabels:
app: zookeeper-3
name: zookeeper-3
template:
metadata:
labels:
app: zookeeper-3
name: zookeeper-3
spec:
containers:
- name: zoo3
image: uhub.service.ucloud.cn/metersphere/zookeeper:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 2181
env:
- name: ZOOKEEPER_ID
value: "3"
- name: ZOOKEEPER_SERVER_1
value: zoo1
- name: ZOOKEEPER_SERVER_2
value: zoo2
- name: ZOOKEEPER_SERVER_3
value: zoo3
volumeMounts:
- name: ms-zookeeper3-data
mountPath: /var/lib/zookeeper
volumes:
- name: ms-zookeeper3-data
hostPath:
path: /opt/nfs/metersphere/data/zookeeper/zookeeper3
---
apiVersion: v1
kind: Service
metadata:
name: zoo1
labels:
app: zookeeper-1
namespace: ms
spec:
ports:
- name: client
port: 2181
protocol: TCP
- name: follower
port: 2888
protocol: TCP
- name: leader
port: 3888
protocol: TCP
selector:
app: zookeeper-1
---
apiVersion: v1
kind: Service
metadata:
name: zoo2
labels:
app: zookeeper-2
namespace: ms
spec:
ports:
- name: client
port: 2181
protocol: TCP
- name: follower
port: 2888
protocol: TCP
- name: leader
port: 3888
protocol: TCP
selector:
app: zookeeper-2
---
apiVersion: v1
kind: Service
metadata:
name: zoo3
labels:
app: zookeeper-3
namespace: ms
spec:
ports:
- name: client
port: 2181
protocol: TCP
- name: follower
port: 2888
protocol: TCP
- name: leader
port: 3888
protocol: TCP
selector:
app: zookeeper-3
此时需要将zookeeper目录权限改为777
chmod 777 /opt/nfs/metersphere/data/zookeeper/zookeeper*
3.3 创建kafka
kubectl create -f kafka-deploy.yaml
#kafka-deploy.yaml
kind: Deployment
apiVersion: apps/v1
metadata:
name: kafka-deployment-1
namespace: ms
spec:
replicas: 1
selector:
matchLabels:
name: kafka-service-1
template:
metadata:
labels:
name: kafka-service-1
app: kafka-service-1
spec:
containers:
- name: kafka-1
image: uhub.service.ucloud.cn/metersphere/kafka:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 9092
env:
- name: KAFKA_ADVERTISED_PORT
value: "9092"
- name: KAFKA_ADVERTISED_HOST_NAME
value: kafka-service-1
- name: KAFKA_ZOOKEEPER_CONNECT
value: zoo1:2181,zoo2:2181,zoo3:2181
- name: KAFKA_BROKER_ID
value: "1"
- name: KAFKA_CREATE_TOPICS
value: mytopic:2:1
volumeMounts:
- name: ms-kafka1-data
mountPath: /kafka
securityContext:
runAsUser: 0
volumes:
- name: ms-kafka1-data
hostPath:
path: /opt/nfs/metersphere/data/kafka/kafka1
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: kafka-deployment-2
namespace: ms
spec:
replicas: 1
selector:
selector:
matchLabels:
name: kafka-service-2
template:
metadata:
labels:
name: kafka-service-2
app: kafka-service-2
spec:
containers:
- name: kafka-2
image: uhub.service.ucloud.cn/metersphere/kafka:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 9092
env:
- name: KAFKA_ADVERTISED_PORT
value: "9092"
- name: KAFKA_ADVERTISED_HOST_NAME
value: kafka-service-2
- name: KAFKA_ZOOKEEPER_CONNECT
value: zoo1:2181,zoo2:2181,zoo3:2181
- name: KAFKA_BROKER_ID
value: "2"
volumeMounts:
- name: ms-kafka2-data
mountPath: /kafka
securityContext:
runAsUser: 0
volumes:
- name: ms-kafka2-data
hostPath:
path: /opt/nfs/metersphere/data/kafka/kafka2
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: kafka-deployment-3
namespace: ms
spec:
replicas: 1
selector:
selector:
matchLabels:
name: kafka-service-3
template:
metadata:
labels:
name: kafka-service-3
app: kafka-service-3
spec:
containers:
- name: kafka-3
image: uhub.service.ucloud.cn/metersphere/kafka:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 9092
env:
- name: KAFKA_ADVERTISED_PORT
value: "9092"
- name: KAFKA_ADVERTISED_HOST_NAME
value: kafka-service-3
- name: KAFKA_ZOOKEEPER_CONNECT
value: zoo1:2181,zoo2:2181,zoo3:2181
- name: KAFKA_BROKER_ID
value: "3"
volumeMounts:
- name: ms-kafka3-data
mountPath: /kafka
securityContext:
runAsUser: 0
volumes:
- name: ms-kafka3-data
hostPath:
path: /opt/nfs/metersphere/data/kafka/kafka3
---
apiVersion: v1
kind: Service
metadata:
name: kafka-service-1
labels:
app: kafka-service-1
namespace: ms
spec:
type: ClusterIP
ports:
- port: 9092
name: kafka-service-1
targetPort: 9092
protocol: TCP
selector:
app: kafka-service-1
---
apiVersion: v1
kind: Service
metadata:
name: kafka-service-2
labels:
app: kafka-service-2
namespace: ms
spec:
type: ClusterIP
ports:
- port: 9092
name: kafka-service-2
targetPort: 9092
protocol: TCP
selector:
app: kafka-service-2
---
apiVersion: v1
kind: Service
metadata:
name: kafka-service-3
labels:
app: kafka-service-3
namespace: ms
spec:
type: ClusterIP
ports:
- port: 9092
name: kafka-service-3
targetPort: 9092
protocol: TCP
selector:
app: kafka-service-3
3.4 创建ms-node-controller
kubectl create -f ms-node-controller-deploy.yaml
#ms-node-controller-deploy.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: ms-node-controller
namespace: ms
spec:
selector:
matchLabels:
app: ms-node-controller
replicas: 1
template:
metadata:
labels:
app: ms-node-controller
spec:
containers:
- name: ms-node-controller
image: registry.cn-qingdao.aliyuncs.com/metersphere/ms-node-controller:v1.14.1
imagePullPolicy: IfNotPresent
env:
- name: HOST_HOSTNAME
value: ms-node-controller
ports:
- containerPort: 8082
- containerPort: 9100
resources:
limits:
memory: "1073741824"
volumeMounts:
- name: ms-conf
mountPath: /opt/metersphere/conf
- name: ms-data
mountPath: /opt/metersphere/data
- name: ms-logs
mountPath: /opt/metersphere/logs
- name: ms-node-controller-sock
mountPath: /var/run/docker.sock
- name: nodeexporter
image: registry.cn-qingdao.aliyuncs.com/metersphere/node-exporter:latest
args:
- --path.procfs=/host/proc
- --path.sysfs=/host/sys
- --collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($|/)
imagePullPolicy: IfNotPresent
securityContext:
privileged: true
volumeMounts:
- name: nodeexporter-proc
mountPath: /host/proc
readOnly: true
- name: nodeexporter-sys
mountPath: /host/sys
readOnly: true
- name: nodeexporter-rootfs
mountPath: /rootfs
readOnly: true
volumes:
- name: nodeexporter-proc
hostPath:
path: /proc
- name: nodeexporter-sys
hostPath:
path: /sys
- name: nodeexporter-rootfs
hostPath:
path: /
- name: ms-conf
hostPath:
path: /opt/nfs/metersphere/conf
- name: ms-data
hostPath:
path: /opt/nfs/metersphere/data
- name: ms-logs
hostPath:
path: /opt/nfs/metersphere/logs
- name: ms-node-controller-sock
hostPath:
path: /var/run/docker.sock
---
apiVersion: v1
kind: Service
metadata:
name: ms-node-controller
namespace: ms
spec:
ports:
- name: ms-node-8082
protocol: TCP
port: 8082
targetPort: 8082
- name: ms-node-9100
protocol: TCP
port: 9100
targetPort: 9100
type: ClusterIP
selector:
app: ms-node-controller
3.5 创建ms-server
kubectl create -f ms-server-deploy.yaml
#ms-server-deploy.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: ms-server
namespace: ms
spec:
selector:
matchLabels:
app: ms-server
replicas: 1
template:
metadata:
labels:
app: ms-server
spec:
containers:
- name: ms-server
image: registry.cn-qingdao.aliyuncs.com/metersphere/metersphere:v1.14.1
imagePullPolicy: IfNotPresent
env:
- name: JMETER_IMAGE
value: registry.cn-qingdao.aliyuncs.com/metersphere/jmeter-master:5.4.1-ms11-jdk8
- name: K8S_NODE-CONTROLLER-IMAGE
value: registry.cn-qingdao.aliyuncs.com/metersphere/ms-node-controller:v1.14.1
- name: KAFKA_BOOTSTRAP-SERVERS
value: kafka-service-1:9092,kafka-service-2:9092,kafka-service-3:9092
- name: KAFKA_LOG_TOPIC
value: JMETER_LOGS
- name: KAFKA_PARTITIONS
value: "4"
- name: KAFKA_REPLICAS
value: "1"
- name: KAFKA_REPORT_TOPIC
value: JMETER_REPORTS
- name: KAFKA_TEST_TOPIC
value: LOAD_TESTS
- name: KAFKA_TOPIC
value: JMETER_METRICS
- name: SESSION_TIMEOUT
value: "21600"
- name: SPRING_DATASOURCE_PASSWORD
value: Password123@mysql
- name: SPRING_DATASOURCE_URL
value: jdbc:mysql://mysql:3306/metersphere?autoReconnect=false&useUnicode=true&characterEncoding=UTF-8&characterSetResults=UTF-8&zeroDateTimeBehavior=convertToNull&allowPublicKeyRetrieval=true&useSSL=false&sessionVariables=sql_mode=%27STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION%27
- name: SPRING_DATASOURCE_USERNAME
value: root
- name: TCP_MOCK_PORT
value: 10000-10010
ports:
- containerPort: 8081
protocol: TCP
volumeMounts:
- mountPath: /opt/metersphere/conf
name: ms-conf
- mountPath: /opt/metersphere/logs
name: ms-logs
- mountPath: /opt/metersphere/data
name: ms-data
#restartPolicy: Always
- name: ms-data-streaming
image: registry.cn-qingdao.aliyuncs.com/metersphere/ms-data-streaming:v1.14.1
imagePullPolicy: IfNotPresent
env:
- name: HOST_HOSTNAME
value: ms-data-streaming
- name: KAFKA_BOOTSTRAP-SERVERS
value: kafka-service-1:9092,kafka-service-2:9092,kafka-service-3:9092
- name: KAFKA_LOG_TOPIC
value: JMETER_LOGS
- name: KAFKA_PARTITIONS
value: "4"
- name: KAFKA_REPLICAS
value: "1"
- name: KAFKA_REPORT_TOPIC
value: JMETER_REPORTS
- name: KAFKA_TEST_TOPIC
value: LOAD_TESTS
- name: KAFKA_TOPIC
value: JMETER_METRICS
- name: SPRING_DATASOURCE_PASSWORD
value: Password123@mysql
- name: SPRING_DATASOURCE_URL
value: jdbc:mysql://mysql:3306/metersphere?autoReconnect=false&useUnicode=true&characterEncoding=UTF-8&characterSetResults=UTF-8&zeroDateTimeBehavior=convertToNull&allowPublicKeyRetrieval=true&useSSL=false
- name: SPRING_DATASOURCE_USERNAME
value: root
volumeMounts:
- name: ms-data-streaming-conf
mountPath: /opt/metersphere/conf/metersphere.properties
- name: ms-data-streaming-logs
mountPath: /opt/metersphere/logs/data-streaming
volumes:
- name: ms-conf
hostPath:
path: /opt/nfs/metersphere/conf
- name: ms-logs
hostPath:
path: /opt/nfs/metersphere/logs
- name: ms-data
hostPath:
path: /opt/nfs/metersphere/data
- name: ms-data-streaming-conf
hostPath:
path: /opt/nfs/metersphere/conf/metersphere.properties
- name: ms-data-streaming-logs
hostPath:
path: /opt/nfs/metersphere/logs/data-streaming
---
apiVersion: v1
kind: Service
metadata:
name: ms-server
namespace: ms
spec:
ports:
- name: ms-server
protocol: TCP
port: 8081
targetPort: 8081
nodePort: 30081
- name: ms-data-streaming
protocol: TCP
port: 8084
targetPort: 8084
type: NodePort
selector:
app: ms-server
3.6 创建prometheus
kubectl create -f prometheus-configmap.yaml
#prometheus-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: ms-prometheus
namespace: ms
labels:
app: ms-prometheus
data:
prometheus.yml: |-
global:
scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
# scrape_timeout is set to the global default (10s).
alerting:
alertmanagers:
- static_configs:
- targets:
# - alertmanager:9093
rule_files:
# - "first_rules.yml"
# - "second_rules.yml"
scrape_configs:
# The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
- job_name: 'prometheus'
# metrics_path defaults to '/metrics'
# scheme defaults to 'http'.
static_configs:
- targets: ['prometheus:9090']
- job_name: 'metersphere'
consul_sd_configs:
- server: 'ms-server:8081'
services: []
kubectl create -f prometheus-deploy.yaml
# prometheus-deploy.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: prometheus
namespace: ms
spec:
selector:
matchLabels:
app: prometheus
replicas: 1
template:
metadata:
labels:
app: prometheus
spec:
containers:
- name: ms-prometheus
image: registry.cn-qingdao.aliyuncs.com/metersphere/prometheus:latest
args:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --web.console.libraries=/etc/prometheus/console_libraries
- --web.console.templates=/etc/prometheus/consoles
- --storage.tsdb.retention=200h
- --web.enable-lifecycle
ports:
- containerPort: 9090
protocol: TCP
volumeMounts:
- name: prometheus-data
mountPath: /prometheus
- name: prometheus-conf
mountPath: /etc/prometheus/
securityContext:
runAsUser: 0
volumes:
- name: prometheus-data
hostPath:
path: /opt/nfs/metersphere/data/prometheus
- name: prometheus-conf
configMap:
name: ms-prometheus
---
apiVersion: v1
kind: Service
metadata:
name: prometheus
namespace: ms
spec:
selector:
app: prometheus
ports:
- name: prometheus
protocol: TCP
port: 9090
nodePort: 30090
type: NodePort
4. 检查部署状态
[root@demo-cluster-master-1 ~]# kubectl get pod,svc -n ms
NAME READY STATUS RESTARTS AGE
pod/kafka-deployment-1-569cf8bf48-pflc6 1/1 Running 1 61m
pod/kafka-deployment-2-668f99864c-d52p5 1/1 Running 1 67m
pod/kafka-deployment-3-6fc8c4dc55-kxqzf 1/1 Running 1 67m
pod/ms-node-controller-5777b9c5c8-hfjdk 2/2 Running 0 67m
pod/ms-server-64f66bbd5c-cf5wx 2/2 Running 0 67m
pod/mysql-c98cc44bd-mvlfw 1/1 Running 0 68m
pod/prometheus-54969cf98f-w7dm4 1/1 Running 0 67m
pod/zookeeper-deployment-1-6c9557bf57-wwr6d 1/1 Running 0 62m
pod/zookeeper-deployment-2-6b9994df86-hw28x 1/1 Running 0 62m
pod/zookeeper-deployment-3-5d777d955-95gqf 1/1 Running 0 62m
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kafka-service-1 ClusterIP 172.20.244.167 <none> 9092/TCP 67m
service/kafka-service-2 ClusterIP 172.20.117.47 <none> 9092/TCP 67m
service/kafka-service-3 ClusterIP 172.20.60.55 <none> 9092/TCP 67m
service/ms-node-controller ClusterIP 172.20.141.206 <none> 8082/TCP,9100/TCP 67m
service/ms-server NodePort 172.20.5.7 <none> 8081:30081/TCP,8084:31095/TCP 67m
service/mysql NodePort 172.20.32.24 <none> 3306:31306/TCP 68m
service/prometheus NodePort 172.20.13.104 <none> 9090:30090/TCP 67m
service/zoo1 ClusterIP 172.20.230.192 <none> 2181/TCP,2888/TCP,3888/TCP 68m
service/zoo2 ClusterIP 172.20.13.139 <none> 2181/TCP,2888/TCP,3888/TCP 68m
service/zoo3 ClusterIP 172.20.164.27 <none> 2181/TCP,2888/TCP,3888/TCP 68m
5. 访问mstersphere页面
http://nodeIP:30081