简介：
本次部署采用的是prometheus+grafana+cadvisor+Node exporter+blackbox exporter+Alertmanager的架构来实现服务器和docker容器及宿主机硬件层面的监控，并接入钉钉告警，分为c/s端，所有应用均为容器化部署。
部署前准备（所有服务器）：
安装docker
关闭防火墙及selinux（最好做到开机禁用）
server端：
组件：
prometheus
node-exporter
webhook-dingtalk
alertmanager
cadvisor
grafana
docker-compose.yml如下：
version: '3.7'
services:
prometheus:
image: prom/prometheus:latest
container_name: prometheus
restart: unless-stopped
ports:
- '9090:9090'
volumes:
- /data/prometheus.yml:/etc/prometheus/prometheus.yml
- /data/alert-rules.yml:/etc/prometheus/alert-rules.yml
depends_on:
- cadvisor
node-exporter:
image: prom/node-exporter:latest
container_name: node-exporter
restart: unless-stopped
ports:
- '9100:9100'
command:
- '--path.procfs=/host/proc'
- '--path.sysfs=/host/sys'
- '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'
volumes:
- /proc:/host/proc
- /sys:/host/sys
- /:/rootfs
webhook-dingtalk:
image: timonwong/prometheus-webhook-dingtalk
container_name: alertdingtalk
restart: unless-stopped
ports:
- '8060:8060'
volumes:
- /data/config.yml:/etc/prometheus-webhook-dingtalk/config.yml
alertmanager:
image: prom/alertmanager:latest
container_name: alertmanager
restart: unless-stopped
ports:
- '9093:9093'
- '9094:9094'
volumes:
- /data/alertmanager.yml:/etc/alertmanager/alertmanager.yml
blackbox_exporter:
image: prom/blackbox-exporter
network_mode: 'host'
container_name: blackbox_exporter
restart: unless-stopped
ports:
- '9115:9115'
cadvisor:
image: google/cadvisor:latest
container_name: cadvisor
restart: unless-stopped
ports:
- '8080:8080'
volumes:
- /:/rootfs:ro
- /var/run:/var/run:rw
- /sys:/sys:ro
- /var/lib/docker/:/var/lib/docker:ro
grafana:
image: grafana/grafana:latest
container_name: grafana
restart: unless-stopped
ports:
- '3000:3000'
user: '0'
volumes:
- ./data/grafana:/var/lib/grafana
environment:
- GF_SECURITY_ADMIN_PASSWORD=admin
- GF_SERVER_ROOT_URL=http://192.168.13.192:3000
配置文件：
alertmanager.yml
global:
resolve_timeout: 1m
route:
receiver: 'webhook'
group_wait: 10s
group_interval: 1m
repeat_interval: 1h
group_by: ['alertname']
inhibit_rules:
- source_match:
severity: 'critical'
target_match:
severity: 'warning'
equal: ['alertname', 'instance']
receivers:
- name: 'webhook'
webhook_configs:
- url: 'http://192.168.13.192:8060/dingtalk/webhook/send'
send_resolved: true
alert-rules.yml
groups:
- name: node-alert
rules:
- alert: DockerDown
expr: up{job="node"} == 0
for: 5m
labels:
severity: critical
instance: "{{ $labels.instance }}"
annotations:
summary: "instance: {{ $labels.instance }} down"
description: "Instance: {{ $labels.instance }} 已经宕机 5分钟"
value: "{{ $value }}"
- alert: Nodedown
expr: probe_success{job="node2"} == 0
for: 1m
labels:
severity: critical
instance: "{{ $labels.instance }}"
annotations:
summery: "instance: {{ $labels.instance }} down"
description: "Instance: {{ $labels.instance }} 已经宕机 5分钟"
value: "{{ $value }}"
- alert: NodeCpuHigh
expr: (1 - avg by (instance) (irate(node_cpu_seconds_total{job="node",mode="idle"}[5m]))) * 100 > 80
for: 5m
labels:
severity: critical
instance: "{{ $labels.instance }}"
annotations:
summary: "instance: {{ $labels.instance }} cpu使用率过高"
description: "CPU 使用率超过 80%"
value: "{{ $value }}"
- alert: NodeCpuIowaitHigh
expr: avg by (instance) (irate(node_cpu_seconds_total{job="node",mode="iowait"}[5m])) * 100 > 50
for: 5m
labels:
severity: warning
instance: "{{ $labels.instance }}"
annotations:
summary: "instance: {{ $labels.instance }} cpu iowait 使用率过高"
description: "CPU iowait 使用率超过 50%"
value: "{{ $value }}"
- alert: NodeLoad5High
expr: node_load5 > (count by (instance) (node_cpu_seconds_total{job="node",mode='system'})) * 1.2
for: 5m
labels:
severity: warning
instance: "{{ $labels.instance }}"
annotations:
summary: "instance: {{ $labels.instance }} load(5m) 过高"
description: "Load(5m) 过高，超出cpu核数 1.2倍"
value: "{{ $value }}"
- alert: NodeMemoryHigh
expr: (1 - node_memory_MemAvailable_bytes{job="node"} / node_memory_MemTotal_bytes{job="node"}) * 100 > 90
for: 5m
labels:
severity: critical
instance: "{{ $labels.instance }}"
annotations:
summary: "instance: {{ $labels.instance }} memory 使用率过高"
description: "Memory 使用率超过 90%"
value: "{{ $value }}"
- alert: NodeDiskRootHigh
expr: (1 - node_filesystem_avail_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/"} / node_filesystem_size_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/"}) * 100 > 90
for: 5m
labels:
severity: critical
instance: "{{ $labels.instance }}"
annotations:
summary: "instance: {{ $labels.instance }} disk(/ 分区) 使用率过高"
description: "Disk(/ 分区) 使用率超过 90%"
value: "{{ $value }}"
- alert: NodeDiskBootHigh
expr: (1 - node_filesystem_avail_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/boot"} / node_filesystem_size_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/boot"}) * 100 > 80
for: 10m
labels:
severity: critical
instance: "{{ $labels.instance }}"
annotations:
summary: "instance: {{ $labels.instance }} disk(/boot 分区) 使用率过高"
description: "Disk(/boot 分区) 使用率超过 80%"
value: "{{ $value }}"
- alert: NodeDiskReadHigh
expr: irate(node_disk_read_bytes_total{job="node"}[5m]) > 20 * (1024 ^ 2)
for: 5m
labels:
severity: warning
instance: "{{ $labels.instance }}"
annotations:
summary: "instance: {{ $labels.instance }} disk 读取字节数 速率过高"
description: "Disk 读取字节数 速率超过 20 MB/s"
value: "{{ $value }}"
- alert: NodeDiskWriteHigh
expr: irate(node_disk_written_bytes_total{job="node"}[5m]) > 20 * (1024 ^ 2)
for: 5m
labels:
severity: warning
instance: "{{ $labels.instance }}"
annotations:
summary: "instance: {{ $labels.instance }} disk 写入字节数 速率过高"
description: "Disk 写入字节数 速率超过 20 MB/s"
value: "{{ $value }}"
- alert: NodeDiskReadRateCountHigh
expr: irate(node_disk_reads_completed_total{job="node"}[5m]) > 3000
for: 5m
labels:
severity: warning
instance: "{{ $labels.instance }}"
annotations:
summary: "instance: {{ $labels.instance }} disk iops 每秒读取速率过高"
description: "Disk iops 每秒读取速率超过 3000 iops"
value: "{{ $value }}"
- alert: NodeDiskWriteRateCountHigh
expr: irate(node_disk_writes_completed_total{job="node"}[5m]) > 3000
for: 5m
labels:
severity: warning
instance: "{{ $labels.instance }}"
annotations:
summary: "instance: {{ $labels.instance }} disk iops 每秒写入速率过高"
description: "Disk iops 每秒写入速率超过 3000 iops"
value: "{{ $value }}"
- alert: NodeInodeRootUsedPercentHigh
expr: (1 - node_filesystem_files_free{job="node",fstype=~"ext4|xfs",mountpoint="/"} / node_filesystem_files{job="node",fstype=~"ext4|xfs",mountpoint="/"}) * 100 > 80
for: 10m
labels:
severity: warning
instance: "{{ $labels.instance }}"
annotations:
summary: "instance: {{ $labels.instance }} disk(/ 分区) inode 使用率过高"
description: "Disk (/ 分区) inode 使用率超过 80%"
value: "{{ $value }}"
- alert: NodeInodeBootUsedPercentHigh
expr: (1 - node_filesystem_files_free{job="node",fstype=~"ext4|xfs",mountpoint="/boot"} / node_filesystem_files{job="node",fstype=~"ext4|xfs",mountpoint="/boot"}) * 100 > 80
for: 10m
labels:
severity: warning
instance: "{{ $labels.instance }}"
annotations:
summary: "instance: {{ $labels.instance }} disk(/boot 分区) inode 使用率过高"
description: "Disk (/boot 分区) inode 使用率超过 80%"
value: "{{ $value }}"
- alert: NodeFilefdAllocatedPercentHigh
expr: node_filefd_allocated{job="node"} / node_filefd_maximum{job="node"} * 100 > 80
for: 10m
labels:
severity: warning
instance: "{{ $labels.instance }}"
annotations:
summary: "instance: {{ $labels.instance }} filefd 打开百分比过高"
description: "Filefd 打开百分比 超过 80%"
value: "{{ $value }}"
config.yml
targets:
webhook:
url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
mention:
all: true
prometheus.yml
global:
scrape_interval: 15s
evaluation_interval: 15s
external_labels:
monitor: 'nodecontainer-monitor'
scrape_configs:
- job_name: 'node'
scrape_interval: 5s
static_configs:
- targets: ['192.168.13.192:9090','192.168.13.192:8080','192.168.13.192:9100','192.168.13.192:3000','192.168.13.192:9093','192.168.13.209:8081','192.168.13.209:9100']
- job_name: 'node2'
scrape_interval: 5s
metrics_path: /probe
params:
module: [icmp]
static_configs:
- targets:
- 192.168.13.192
labels:
instance: 192.168.13.192
- targets:
- 192.168.13.209
labels:
instance: 192.168.13.209
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- target_label: __address__
replacement: 192.168.13.192:9115
alerting:
alertmanagers:
- static_configs:
- targets:
- 192.168.13.192:9093
rule_files:
- alert-rules.yml
直接docker-compose up -d 运行即可
client端:（被监控端）
组件：
node-exporter
cadvisor
version: '3.7'
services:
node-exporter:
image: prom/node-exporter:latest
restart: unless-stopped
ports:
- '9100:9100'
command:
- '--path.procfs=/host/proc'
- '--path.sysfs=/host/sys'
- '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'
volumes:
- /proc:/host/proc
- /sys:/host/sys
- /:/rootfs
cadvisor:
image: google/cadvisor:latest
container_name: cadvisor
restart: unless-stopped
ports:
- '8080:8080'
volumes:
- /:/rootfs:ro
- /var/run:/var/run:rw
- /sys:/sys:ro
- /var/lib/docker/:/var/lib/docker:ro
运行 docker-compose up -d
grafana配置及导入dashboards
此处导入下载好的docker-monitoring_rev1.json和node-exporter-full_rev13.json、blackbox-exporter_rev1.json用于展示container和host的监控
下载地址：https://grafana.com/dashboards?dataSource=prometheus&category=docker
https://grafana.com/grafana/plugins/grafana-piechart-panel/?tab=installation（需要安装饼图插件）
注意事项：
1、expr: up{job="node"} == 0 alert-rule.yml里面的job=“xx”一定要与prometheus.yml里面的job_name相同，否则无法触发告警
2、docker-compose.yml里面注意映射的端口号不要被占用，映射的路径要正确
3、部分机器docker-compose版本过低，需要升级。注意容器网络模式，最好统一设置成 network_mode 'host'