通过docker stack部署prometheus、node-exporter、alertmanager和grafana。prometheus最新版本：2.19.2

swarm集群（一个节点）：

manager     192.168.30.135

mkdir -p /home/prom/{prometheus,prometheus/data,alertmanager,grafana}chmod 777 /home/prom/{prometheus/data,grafana}cd /home/prom

tree ..├── alertmanager
│   ├── alertmanager.yml
│   └── config.yml
├── docker-stack.yml
├── grafana
└── prometheus
    ├── alert-rules.yml
    ├── data
    └── prometheus.yml

4 directories, 5 files

---

Prometheus

vim /home/prom/prometheus/alert-rules.yml

groups:
  - name: node-alert    rules:
    - alert: NodeDown      expr: up{job="node"} == 0      for: 5m      labels:
        severity: critical        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} down"        description: "Instance: {{ $labels.instance }} 已经宕机 5分钟"        value: "{{ $value }}"
        
    - alert: NodeCpuHigh      expr: (1 - avg by (instance) (irate(node_cpu_seconds_total{job="node",mode="idle"}[5m]))) * 100 > 80      for: 5m      labels:
        severity: warning        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} cpu使用率过高"        description: "CPU 使用率超过 80%"
        value: "{{ $value }}"

    - alert: NodeCpuIowaitHigh      expr: avg by (instance) (irate(node_cpu_seconds_total{job="node",mode="iowait"}[5m])) * 100 > 50      for: 5m      labels:
        severity: warning        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} cpu iowait 使用率过高"        description: "CPU iowait 使用率超过 50%"
        value: "{{ $value }}"

    - alert: NodeLoad5High      expr: node_load5 > (count by (instance) (node_cpu_seconds_total{job="node",mode='system'})) * 1.2      for: 5m      labels:
        severity: warning        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} load(5m) 过高"        description: "Load(5m) 过高，超出cpu核数 1.2倍"
        value: "{{ $value }}"

    - alert: NodeMemoryHigh      expr: (1 - node_memory_MemAvailable_bytes{job="node"} / node_memory_MemTotal_bytes{job="node"}) * 100 > 90      for: 5m      labels:
        severity: warning        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} memory 使用率过高"        description: "Memory 使用率超过 90%"
        value: "{{ $value }}"

    - alert: NodeDiskRootHigh      expr: (1 - node_filesystem_avail_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/"} / node_filesystem_size_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/"}) * 100 > 90      for: 10m      labels:
        severity: warning        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} disk(/ 分区) 使用率过高"        description: "Disk(/ 分区) 使用率超过 90%"
        value: "{{ $value }}"

    - alert: NodeDiskBootHigh      expr: (1 - node_filesystem_avail_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/boot"} / node_filesystem_size_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/boot"}) * 100 > 80      for: 10m      labels:
        severity: warning        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} disk(/boot 分区) 使用率过高"        description: "Disk(/boot 分区) 使用率超过 80%"
        value: "{{ $value }}"

    - alert: NodeDiskReadHigh      expr: irate(node_disk_read_bytes_total{job="node"}[5m]) > 20 * (1024 ^ 2)      for: 5m      labels:
        severity: warning        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} disk 读取字节数 速率过高"        description: "Disk 读取字节数 速率超过 20 MB/s"
        value: "{{ $value }}"

    - alert: NodeDiskWriteHigh      expr: irate(node_disk_written_bytes_total{job="node"}[5m]) > 20 * (1024 ^ 2)      for: 5m      labels:
        severity: warning        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} disk 写入字节数 速率过高"        description: "Disk 写入字节数 速率超过 20 MB/s"
        value: "{{ $value }}"
        
    - alert: NodeDiskReadRateCountHigh      expr: irate(node_disk_reads_completed_total{job="node"}[5m]) > 3000      for: 5m      labels:
        severity: warning        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} disk iops 每秒读取速率过高"        description: "Disk iops 每秒读取速率超过 3000 iops"
        value: "{{ $value }}"

    - alert: NodeDiskWriteRateCountHigh      expr: irate(node_disk_writes_completed_total{job="node"}[5m]) > 3000      for: 5m      labels:
        severity: warning        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} disk iops 每秒写入速率过高"        description: "Disk iops 每秒写入速率超过 3000 iops"
        value: "{{ $value }}"

    - alert: NodeInodeRootUsedPercentHigh      expr: (1 - node_filesystem_files_free{job="node",fstype=~"ext4|xfs",mountpoint="/"} / node_filesystem_files{job="node",fstype=~"ext4|xfs",mountpoint="/"}) * 100 > 80      for: 10m      labels:
        severity: warning        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} disk(/ 分区) inode 使用率过高"        description: "Disk (/ 分区) inode 使用率超过 80%"
        value: "{{ $value }}"

    - alert: NodeInodeBootUsedPercentHigh      expr: (1 - node_filesystem_files_free{job="node",fstype=~"ext4|xfs",mountpoint="/boot"} / node_filesystem_files{job="node",fstype=~"ext4|xfs",mountpoint="/boot"}) * 100 > 80      for: 10m      labels:
        severity: warning        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} disk(/boot 分区) inode 使用率过高"        description: "Disk (/boot 分区) inode 使用率超过 80%"
        value: "{{ $value }}"
        
    - alert: NodeFilefdAllocatedPercentHigh      expr: node_filefd_allocated{job="node"} / node_filefd_maximum{job="node"} * 100 > 80      for: 10m      labels:
        severity: warning        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} filefd 打开百分比过高"        description: "Filefd 打开百分比 超过 80%"
        value: "{{ $value }}"

    - alert: NodeNetworkNetinBitRateHigh      expr: avg by (instance) (irate(node_network_receive_bytes_total{device=~"eth0|eth1|ens33|ens37"}[1m]) * 8) > 20 * (1024 ^ 2) * 8      for: 3m      labels:
        severity: warning        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} network 接收比特数 速率过高"        description: "Network 接收比特数 速率超过 20MB/s"
        value: "{{ $value }}"

    - alert: NodeNetworkNetoutBitRateHigh      expr: avg by (instance) (irate(node_network_transmit_bytes_total{device=~"eth0|eth1|ens33|ens37"}[1m]) * 8) > 20 * (1024 ^ 2) * 8      for: 3m      labels:
        severity: warning        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} network 发送比特数 速率过高"        description: "Network 发送比特数 速率超过 20MB/s"
        value: "{{ $value }}"
        
    - alert: NodeNetworkNetinPacketErrorRateHigh      expr: avg by (instance) (irate(node_network_receive_errs_total{device=~"eth0|eth1|ens33|ens37"}[1m])) > 15      for: 3m      labels:
        severity: warning        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} 接收错误包 速率过高"        description: "Network 接收错误包 速率超过 15个/秒"
        value: "{{ $value }}"

    - alert: NodeNetworkNetoutPacketErrorRateHigh      expr: avg by (instance) (irate(node_network_transmit_packets_total{device=~"eth0|eth1|ens33|ens37"}[1m])) > 15      for: 3m      labels:
        severity: warning        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} 发送错误包 速率过高"        description: "Network 发送错误包 速率超过 15个/秒"
        value: "{{ $value }}"

    - alert: NodeProcessBlockedHigh      expr: node_procs_blocked{job="node"} > 10      for: 10m      labels:
        severity: warning        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} 当前被阻塞的任务的数量过多"        description: "Process 当前被阻塞的任务的数量超过 10个"
        value: "{{ $value }}"

    - alert: NodeTimeOffsetHigh      expr: abs(node_timex_offset_seconds{job="node"}) > 3 * 60      for: 2m      labels:
        severity: info        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} 时间偏差过大"        description: "Time 节点的时间偏差超过 3m"
        value: "{{ $value }}"

vim /home/prom/prometheus/prometheus.yml

global:
  scrape_interval:     15s  evaluation_interval: 15salerting:
  alertmanagers:
  - static_configs:
    - targets:
      - alertmanager:9093rule_files:
  - "*rules.yml"
  scrape_configs:
  - job_name: 'prometheus'
    static_configs:
    - targets: ['prometheus:9090']

  - job_name: 'node'
    static_configs:
    - targets: ['192.168.30.135:9100']

  - job_name: 'alertmanager'
    static_configs:
    - targets: ['alertmanager:9093']

---

Alertmanager

vim /home/prom/alertmanager/config.yml

targets:
  webhook:
    url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxx             #修改为钉钉机器人的webhook
    mention:
      all: true

vim /home/prom/alertmanager/alertmanager.yml

global:
  resolve_timeout: 5m  smtp_smarthost: 'smtp.163.com:465'             #邮箱smtp服务器代理，启用SSL发信, 端口一般是465
  smtp_from: 'alert@163.com'              #发送邮箱名称
  smtp_auth_username: 'alert@163.com'              #邮箱名称
  smtp_auth_password: 'password'                #邮箱密码或授权码
  smtp_require_tls: falseroute:
  receiver: 'default'
  group_wait: 10s  group_interval: 1m  repeat_interval: 1h  group_by: ['alertname']inhibit_rules:- source_match:
    severity: 'critical'
  target_match:
    severity: 'warning'
  equal: ['alertname', 'instance']
  receivers:- name: 'default'
  email_configs:
  - to: 'receiver@163.com'
    send_resolved: true
  webhook_configs:
  - url: 'http://dingtalk:8060/dingtalk/webhook/send'
    send_resolved: true

---

docker-stack.yml

vim /home/prom/docker-stack.yml

version: '3.7'services:
  dingtalk:
    image: timonwong/prometheus-webhook-dingtalk:latest    ports:
      - "8060:8060"
    configs:
      - source: dingtalk_config        target: /etc/prometheus-webhook-dingtalk/config.yml    networks:
      - prom    deploy:
      mode: replicated      replicas: 1

  alertmanager:
    image: prom/alertmanager:latest    ports:
      - "9093:9093"
      - "9094:9094"
    configs:
      - source: alertmanager_config        target: /etc/alertmanager/alertmanager.yml    networks:
      - prom    deploy:
      mode: replicated      replicas: 1

  prometheus:
    image: prom/prometheus:latest    ports:
      - "9090:9090"
    configs:
      - source: prometheus_config        target: /etc/prometheus/prometheus.yml      - source: alert_rules        target: /etc/prometheus/alert-rules.yml    networks:
      - prom    deploy:
      mode: replicated      replicas: 1
      placement:
        constraints:
          - node.role == manager  grafana:
    image: grafana/grafana:latest    ports:
      - "3000:3000"
    volumes:
      - type: volume        source: grafana        target: /var/lib/grafana    networks:
      - prom    deploy:
      mode: replicated      replicas: 1
      configs:
  dingtalk_config:
    file: ./alertmanager/config.yml    
  alertmanager_config:
    file: ./alertmanager/alertmanager.yml    
  prometheus_config:
    file: ./prometheus/prometheus.yml    
  alert_rules:
    file: ./prometheus/alert-rules.yml    
volumes:
  prometheus:
    driver: local    driver_opts:
      type: none      o: bind      device: /home/prom/prometheus/data  grafana:
    driver: local    driver_opts:
      type: none      o: bind      device: /home/prom/grafana      
networks:
  prom:
    driver: overlay

version: '3.7'services:
  node-exporter:
    image: prom/node-exporter:latest    ports:
      - "9100:9100"
    networks:
      - prom  dingtalk:
    image: timonwong/prometheus-webhook-dingtalk:latest    volumes:
      - type: bind        source: ./alertmanager/config.yml        target: /etc/prometheus-webhook-dingtalk/config.yml        read_only: true
    ports:
      - "8060:8060"
    networks:
      - prom  alertmanager:
    depends_on:
      - dingtalk    image: prom/alertmanager:latest    volumes:
      - type: bind        source: ./alertmanager/alertmanager.yml        target: /etc/alertmanager/alertmanager.yml        read_only: true
    ports:
      - "9093:9093"
      - "9094:9094"
    networks:
      - prom  prometheus:
    depends_on:
      - alertmanager    image: prom/prometheus:latest    volumes:
      - type: bind        source: ./prometheus/prometheus.yml        target: /etc/prometheus/prometheus.yml        read_only: true
      - type: bind        source: ./prometheus/alert-rules.yml        target: /etc/prometheus/alert-rules.yml        read_only: true
      - type: volume        source: prometheus        target: /prometheus    ports:
      - "9090:9090"
    networks:
      - prom  grafana:
    depends_on:
      - prometheus    image: grafana/grafana:latest    volumes:
      - type: volume        source: grafana        target: /var/lib/grafana    ports:
      - "3000:3000"
    networks:
      - promvolumes:
  prometheus:
    driver: local    driver_opts:
      type: none      o: bind      device: /home/prom/prometheus/data  grafana:
    driver: local    driver_opts:
      type: none      o: bind      device: /home/prom/grafana      
networks:
  prom:
    driver: bridge

docker stack deploy prom --compose-file docker-stack.yml

docker stack lsNAME                SERVICES            ORCHESTRATOR
prom                4                   Swarm

docker service lsID                  NAME                MODE                REPLICAS            IMAGE                                          PORTS
f72uewsvc8os        prom_alertmanager   replicated          1/1                 prom/alertmanager:latest                       *:9093-9094->9093-9094/tcp
qonjcrm8pf8o        prom_dingtalk       replicated          1/1                 timonwong/prometheus-webhook-dingtalk:latest   *:8060->8060/tcp
u376krlzd9o6        prom_grafana        replicated          1/1                 grafana/grafana:latest                         *:3000->3000/tcp
kjj909up7ptd        prom_prometheus     replicated          1/1                 prom/prometheus:latest                         *:9090->9090/tcp

docker psCONTAINER ID        IMAGE                                          COMMAND                  CREATED              STATUS              PORTS               NAMES
daf3f972ceea        timonwong/prometheus-webhook-dingtalk:latest   "/bin/prometheus-web…"   About a minute ago   Up About a minute   8060/tcp            prom_dingtalk.1.76ick5qr2fquysl6noztepypa
bcd8f36c78dc        grafana/grafana:latest                         "/run.sh"                About a minute ago   Up About a minute   3000/tcp            prom_grafana.1.ybv3yqburoc6olwys0xh2pqlk
160b53a9f51e        prom/prometheus:latest                         "/bin/prometheus --c…"   About a minute ago   Up About a minute   9090/tcp            prom_prometheus.1.wo8gjnlqlup2nd0ejb88pca85
709ee8176696        prom/alertmanager:latest                       "/bin/alertmanager -…"   About a minute ago   Up About a minute   9093/tcp            prom_alertmanager.1.5beu8aeyt1towanyj9wixtggr

容器启动正常，访问ip:9090，

因为docker-stack.yml不包含node-exporter，所以状态是DOWN。访问ip:3000，

可以看到，prometheus各组件状态正常。

---

node-exporter

通过docker单独启动node-exporter：

docker pull prom/node-exporter:latest

docker run -d -p 9100:9100 --name node-exporter prom/node-exporter:latest

增加监控主机时，修改prometheus.yml，然后更新prometheus：

docker service update prom_prometheus

测试告警

docker stop node-exporter

docker psCONTAINER ID        IMAGE                                          COMMAND                  CREATED             STATUS              PORTS                    NAMES
daf3f972ceea        timonwong/prometheus-webhook-dingtalk:latest   "/bin/prometheus-web…"   17 minutes ago      Up 17 minutes       8060/tcp                 prom_dingtalk.1.76ick5qr2fquysl6noztepypa
bcd8f36c78dc        grafana/grafana:latest                         "/run.sh"                17 minutes ago      Up 17 minutes       3000/tcp                 prom_grafana.1.ybv3yqburoc6olwys0xh2pqlk
160b53a9f51e        prom/prometheus:latest                         "/bin/prometheus --c…"   17 minutes ago      Up 17 minutes       9090/tcp                 prom_prometheus.1.wo8gjnlqlup2nd0ejb88pca85
709ee8176696        prom/alertmanager:latest                       "/bin/alertmanager -…"   17 minutes ago      Up 17 minutes       9093/tcp                 prom_alertmanager.1.5beu8aeyt1towanyj9wixtggr

收到钉钉和邮件故障告警，

docker start node-exporter

docker psCONTAINER ID        IMAGE                                          COMMAND                  CREATED             STATUS              PORTS                    NAMES
daf3f972ceea        timonwong/prometheus-webhook-dingtalk:latest   "/bin/prometheus-web…"   17 minutes ago      Up 17 minutes       8060/tcp                 prom_dingtalk.1.76ick5qr2fquysl6noztepypa
bcd8f36c78dc        grafana/grafana:latest                         "/run.sh"                17 minutes ago      Up 17 minutes       3000/tcp                 prom_grafana.1.ybv3yqburoc6olwys0xh2pqlk
160b53a9f51e        prom/prometheus:latest                         "/bin/prometheus --c…"   17 minutes ago      Up 17 minutes       9090/tcp                 prom_prometheus.1.wo8gjnlqlup2nd0ejb88pca85
709ee8176696        prom/alertmanager:latest                       "/bin/alertmanager -…"   17 minutes ago      Up 17 minutes       9093/tcp                 prom_alertmanager.1.5beu8aeyt1towanyj9wixtggr
95252704e558        prom/node-exporter:latest                      "/bin/node_exporter"     24 hours ago        Up 7 minutes        0.0.0.0:9100->9100/tcp   node-exporter

收到钉钉和邮件恢复告警，

测试宕机完成，告警没有问题。

docker stack部署 prometheus + grafana 完成，整个部署过程在swarm集群中进行。与docker-compose类似，相比传统部署方式要简便很多。

---