ELK学习实验018：filebeat收集docker日志

2021-09-19 09:46:01

Filebeat收集Docker日志

1 安装docker

[root@node4 ~]# yum install -y yum-utils device-mapper-persistent-data lvm2

[root@node4 ~]# yum update

[root@node4 ~]# yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

[root@node4 ~]# yum makecache fast

[root@node4 ~]# yum -y install docker-ce

[root@node4 ~]# systemctl restart docker

[root@node4 ~]# systemctl enable docker

2 运行一个nginx容器

[root@node4 ~]# docker run --name nginx -p 8081:80 -d nginx

Unable to find image 'nginx:latest' locally

latest: Pulling from library/nginx

8ec398bc0356: Pull complete

dfb2a46f8c2c: Pull complete

b65031b6a2a5: Pull complete

Digest: sha256:8aa7f6a9585d908a63e5e418dc5d14ae7467d2e36e1ab4f0d8f9d059a3d071ce

Status: Downloaded newer image for nginx:latest

9c29964182697e55e7ca0fd793f1e243a9e404c84868bee814afdb700760ba5a

[root@node4 ~]# docker ps -a

CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                  NAMES

9c2996418269        nginx               "nginx -g 'daemon of…"    seconds ago      Up  seconds       0.0.0.0:->/tcp   nginx

访问http://192.168.132.134:8081/

[root@node4 ~]# docker exec -it 9c2996418269 /bin/bash

3 查看docker日志

[root@node4 ~]# docker logs -f nginx

192.168.132.1 - - [/Jan/::: +] "GET / HTTP/1.1"   "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" "-"

// :: [error] #: * open() "/usr/share/nginx/html/favicon.ico" failed (: No such file or directory), client: 192.168.132.1, server: localhost, request: "GET /favicon.ico HTTP/1.1", host: "192.168.132.134:8081", referrer: "http://192.168.132.134:8081/"

192.168.132.1 - - [/Jan/::: +] "GET /favicon.ico HTTP/1.1"   "http://192.168.132.134:8081/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" "-"

本地查看

[root@node4 ~]# tail -f /var/lib/docker/containers/9c29964182697e55e7ca0fd793f1e243a9e404c84868bee814afdb700760ba5a/9c29964182697e55e7ca0fd793f1e243a9e404c84868bee814afdb700760ba5a-json.log

是json格式日志

4 filebeat收集

docker的正确日志

错误日志

错误日志再stream显示的stdeer，正确的是stdout，根据这个规则配置filebeat

5 配置filebeat

filebeat.inputs:

#####################################################

## Nginx log

#####################################################

- type: log

  enabled: true

  paths:

    - /usr/local/nginx/logs/access.log

  json.key_under_root: true

  json.overwrite_keys: true

  tags: ["access"]

- type: log

  enabled: true

  paths:

    - /usr/local/nginx/logs/error.log

  tags: ["error"]

#####################################################

## tomcat  log

#####################################################

- type: log

  enabled: true

  paths:

    - /var/log/tomcat/localhost_access_log.*.txt

  json.key_under_root: true

  json.overwrite_keys: true

  tags: ["tomcat"]

#####################################################

## java  log

#####################################################

- type: log

  enabled: true

  paths:

    - /usr/local/elasticsearch/logs/my-elktest-cluster.log

  tags: ["es-java"]

  multiline.pattern: '^\['

  multiline.negate: true

  multiline.match: "after"

#####################################################

## docker  log

#####################################################

- type: docker

  containers.ids:

    - '9c29964182697e55e7ca0fd793f1e243a9e404c84868bee814afdb700760ba5a'

  json.key_under_root: true

  json.overwrite_keys: true

  tags: ["docker"]

#####################################################

## Output

#####################################################

setup.kibana:

  host: "192.168.132.131:5601"

output.elasticsearch:

  hosts: ["192.168.132.131:9200","192.168.132.132:9200","192.168.132.133:9200"]

  #index: "nginx-%{[agent.version]}-%{+yyyy.MM.dd}"

  indices:

    - index: "access-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

        tags: "access"

    - index: "error-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

        tags: "error"

    - index: "tomcat-access-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

        tags: "tomcat"

    - index: "javaes-access-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

        tags: "es-java"

    - index: "docker-access-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

         tags: "docker"

         stream: "stdout"

    - index: "docker-error-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

         tags: "docker"

         stream: "stderr"

setup.template.name: "nginx"

setup.template.pattern: "nginx-*"

setup.template.overwrite: true

setup.template.enabled: true

setup.ilm.enabled: false

查看索引

kibana查看

错误日志

源日志数据

@timestamp    Jan ,  @ ::11.016

    t_id    wXuZvW8BYiPduFlChbrm

    t_index    docker-error-7.4.-2020.01.

    #_score     -

    t_type    _doc

    tagent.ephemeral_id    66a6dffb-9e49--a6a0-ff1a073eea6a

    tagent.hostname    node4

    tagent.id    bb3818f9-66e2-4eb2-8f0c-3f35b543e025

    tagent.type    filebeat

    tagent.version    7.4.

    tecs.version    1.1.

    thost.name    node4

    tinput.type    docker

    tlog.file.path    /var/lib/docker/containers/9c29964182697e55e7ca0fd793f1e243a9e404c84868bee814afdb700760ba5a/9c29964182697e55e7ca0fd793f1e243a9e404c84868bee814afdb700760ba5a-json.log

    #log.offset    ,

    tmessage    // :: [error] #: * open() "/usr/share/nginx/html/tcp" failed (: No such file or directory), client: 192.168.132.1, server: localhost, request: "GET /tcp HTTP/1.1", host: "192.168.132.134:8081"

    tstream    stderr

    ttags    docker

正确日志

原日志数据

@timestamp    Jan ,  @ ::15.401

    t_id    hlGbvW8BOF7DoSFdbG5D

    t_index    docker-access-7.4.-2020.01.

    #_score     -

    t_type    _doc

    tagent.ephemeral_id    66a6dffb-9e49--a6a0-ff1a073eea6a

    tagent.hostname    node4

    tagent.id    bb3818f9-66e2-4eb2-8f0c-3f35b543e025

    tagent.type    filebeat

    tagent.version    7.4.

    tecs.version    1.1.

    thost.name    node4

    tinput.type    docker

    tlog.file.path    /var/lib/docker/containers/9c29964182697e55e7ca0fd793f1e243a9e404c84868bee814afdb700760ba5a/9c29964182697e55e7ca0fd793f1e243a9e404c84868bee814afdb700760ba5a-json.log

    #log.offset    ,

    tmessage    192.168.132.1 - - [/Jan/::: +] "GET / HTTP/1.1"   "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" "-"

    tstream    stdout

    ttags    docker

6 运行多个容器

[root@node4 ~]# docker run --name nginx-v2 -p 8082:80 -v /data:/usr/share/nginx/html -d nginx

[root@node4 ~]# cd /data/

[root@node4 data]# echo "this is second container" > index.html

[root@node4 data]# docker ps -a

CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                  NAMES

7778b091aa01        nginx               "nginx -g 'daemon of…"    seconds ago      Up  seconds       0.0.0.0:->/tcp   nginx-v2

9c2996418269        nginx               "nginx -g 'daemon of…"    minutes ago      Up  minutes       0.0.0.0:->/tcp   nginx

访问http://192.168.132.134:8082/

7 配置filebeat收集所有容器

想要收集所有的dokcer日志修改filebeat

filebeat.inputs:

#####################################################

## Nginx log

#####################################################

- type: log

  enabled: true

  paths:

    - /usr/local/nginx/logs/access.log

  json.key_under_root: true

  json.overwrite_keys: true

  tags: ["access"]

- type: log

  enabled: true

  paths:

    - /usr/local/nginx/logs/error.log

  tags: ["error"]

#####################################################

## tomcat  log

#####################################################

- type: log

  enabled: true

  paths:

    - /var/log/tomcat/localhost_access_log.*.txt

  json.key_under_root: true

  json.overwrite_keys: true

  tags: ["tomcat"]

#####################################################

## java  log

#####################################################

- type: log

  enabled: true

  paths:

    - /usr/local/elasticsearch/logs/my-elktest-cluster.log

  tags: ["es-java"]

  multiline.pattern: '^\['

  multiline.negate: true

  multiline.match: "after"

#####################################################

## docker  log

#####################################################

- type: docker

  containers.ids:

    - '*'

  json.key_under_root: true

  json.overwrite_keys: true

  tags: ["docker"]

#####################################################

## Output

#####################################################

setup.kibana:

  host: "192.168.132.131:5601"

output.elasticsearch:

  hosts: ["192.168.132.131:9200","192.168.132.132:9200","192.168.132.133:9200"]

  #index: "nginx-%{[agent.version]}-%{+yyyy.MM.dd}"

  indices:

    - index: "access-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

        tags: "access"

    - index: "error-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

        tags: "error"

    - index: "tomcat-access-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

        tags: "tomcat"

    - index: "javaes-access-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

        tags: "es-java"

    - index: "docker-access-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

         tags: "docker"

         stream: "stdout"

    - index: "docker-error-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

         tags: "docker"

         stream: "stderr"

setup.template.name: "nginx"

setup.template.pattern: "nginx-*"

setup.template.overwrite: true

setup.template.enabled: true

setup.ilm.enabled: false

随意访问nginx，查看索引

但是收集到日志以后，所有的容器日志集中在一起，无法分辨，则为每一个容器添加一个标签

使用docker-compose为容器添加新的标签

8 安装docker-compose

参考https://www.cnblogs.com/zyxnhr/p/12158816.html

[root@node4 src]# curl -L https://github.com/docker/compose/releases/download/1.25.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose

 % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current

                                 Dload  Upload   Total   Spent    Left  Speed

                              --:--:-- --:--:-- --:--:--

 16.2M   16.2M            529k        ::  :: --:--:--  551k

[root@node4 src]# chmod +x /usr/local/bin/docker-compose

[root@node4 src]# docker-compose --version

docker-compose version 1.25., build 0a186604

[root@node4 ~]# vim docker-compose.yaml

version: ''

services:

  nginx:

    image: nginx

    #设置labels

    labels:

      service: nginx

    #logging设置增加labels.service

    logging:

      options:

        labels: "service"

    ports:

      - "8083:80"

  httpd:

    image: httpd:2.4

    #设置labels

    labels:

      service: httpd

    #logging设置增加labels.service

    logging:

      options:

        labels: "service"

    ports:

      - "8084:80"

10 使用docker-compose发布容器

[root@node4 ~]# docker-compose up

Creating network "root_default" with the default driver

Pulling httpd (httpd:2.4)...

2.4: Pulling from library/httpd

8ec398bc0356: Already exists

354e6904d655: Pull complete

27298e4c749a: Pull complete

10e27104ba69: Pull complete

36412f6b2f6e: Pull complete

Digest: sha256:769018135ba22d3a7a2b91cb89b8de711562cdf51ad6621b2b9b13e95f3798de

Status: Downloaded newer image for httpd:2.4

Creating root_httpd_1 ... done

Creating root_nginx_1 ... done

[root@node4 ~]# docker ps -a

CONTAINER ID        IMAGE               COMMAND                  CREATED              STATUS              PORTS                  NAMES

0c68d79a9a73        nginx               "nginx -g 'daemon of…"   About a minute ago   Up About a minute   0.0.0.0:->/tcp   root_nginx_1

302d59b77fd9        httpd:2.4           "httpd-foreground"       About a minute ago   Up About a minute   0.0.0.0:->/tcp   root_httpd_1

7778b091aa01        nginx               "nginx -g 'daemon of…"    minutes ago       Up  minutes       0.0.0.0:->/tcp   nginx-v2

9c2996418269        nginx               "nginx -g 'daemon of…"   About an hour ago    Up About an hour    0.0.0.0:->/tcp   nginx

查看索引日志

另一个也有标记

kinban查看

@timestamp    Jan ,  @ ::49.919

    t_id    nFG_vW8BOF7DoSFdtm7C

    t_index    docker-access-7.4.-2020.01.

    #_score     -

    t_type    _doc

    tagent.ephemeral_id    22c670e2-26fe-459f--36cf36e6aa2f

    tagent.hostname    node4

    tagent.id    bb3818f9-66e2-4eb2-8f0c-3f35b543e025

    tagent.type    filebeat

    tagent.version    7.4.

    ?docker.attrs.service    httpd     #docker标记

    tecs.version    1.1.

    thost.name    node4

    tinput.type    docker

    tlog.file.path    /var/lib/docker/containers/302d59b77fd90a5fa664e5e44ff4c774fa66b0850d82a12f8d156463eba3a5dd/302d59b77fd90a5fa664e5e44ff4c774fa66b0850d82a12f8d156463eba3a5dd-json.log

    #log.offset    ,

    tmessage    192.168.132.1 - - [/Jan/::: +] "GET /tcp HTTP/1.1"

    tstream    stdout

    ttags    docker

11 根据容器类别自定义

filebeat.inputs:

#####################################################

## Nginx log

#####################################################

- type: log

  enabled: true

  paths:

    - /usr/local/nginx/logs/access.log

  json.key_under_root: true

  json.overwrite_keys: true

  tags: ["access"]

- type: log

  enabled: true

  paths:

    - /usr/local/nginx/logs/error.log

  tags: ["error"]

#####################################################

## tomcat  log

#####################################################

- type: log

  enabled: true

  paths:

    - /var/log/tomcat/localhost_access_log.*.txt

  json.key_under_root: true

  json.overwrite_keys: true

  tags: ["tomcat"]

#####################################################

## java  log

#####################################################

- type: log

  enabled: true

  paths:

    - /usr/local/elasticsearch/logs/my-elktest-cluster.log

  tags: ["es-java"]

  multiline.pattern: '^\['

  multiline.negate: true

  multiline.match: "after"

#####################################################

## docker  log

#####################################################

- type: docker

  containers.ids:

    - '*'

  json.key_under_root: true

  json.overwrite_keys: true

  tags: ["docker"]

#####################################################

## Output

#####################################################

setup.kibana:

  host: "192.168.132.131:5601"

output.elasticsearch:

  hosts: ["192.168.132.131:9200","192.168.132.132:9200","192.168.132.133:9200"]

  #index: "nginx-%{[agent.version]}-%{+yyyy.MM.dd}"

  indices:

    - index: "access-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

        tags: "access"

    - index: "error-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

        tags: "error"

    - index: "tomcat-access-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

        tags: "tomcat"

    - index: "javaes-access-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

        tags: "es-java"

    - index: "docker-nginx-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

         tags: "docker"

         docker.attrs.service: "nginx"

    - index: "docker-httpd-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

         tags: "docker"

         docker.attrs.service: "httpd"

setup.template.name: "nginx"

setup.template.pattern: "nginx-*"

setup.template.overwrite: true

setup.template.enabled: true

setup.ilm.enabled: false

访问后查看索引

12 修改filebeat再细致划分

filebeat.inputs:

#####################################################

## Nginx log

#####################################################

- type: log

  enabled: true

  paths:

    - /usr/local/nginx/logs/access.log

  json.key_under_root: true

  json.overwrite_keys: true

  tags: ["access"]

- type: log

  enabled: true

  paths:

    - /usr/local/nginx/logs/error.log

  tags: ["error"]

#####################################################

## tomcat  log

#####################################################

- type: log

  enabled: true

  paths:

    - /var/log/tomcat/localhost_access_log.*.txt

  json.key_under_root: true

  json.overwrite_keys: true

  tags: ["tomcat"]

#####################################################

## java  log

#####################################################

- type: log

  enabled: true

  paths:

    - /usr/local/elasticsearch/logs/my-elktest-cluster.log

  tags: ["es-java"]

  multiline.pattern: '^\['

  multiline.negate: true

  multiline.match: "after"

#####################################################

## docker  log

#####################################################

- type: docker

  containers.ids:

    - '*'

  json.key_under_root: true

  json.overwrite_keys: true

  tags: ["docker"]

#####################################################

## Output

#####################################################

setup.kibana:

  host: "192.168.132.131:5601"

output.elasticsearch:

  hosts: ["192.168.132.131:9200","192.168.132.132:9200","192.168.132.133:9200"]

  #index: "nginx-%{[agent.version]}-%{+yyyy.MM.dd}"

  indices:

    - index: "access-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

        tags: "access"

    - index: "error-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

        tags: "error"

    - index: "tomcat-access-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

        tags: "tomcat"

    - index: "javaes-access-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

        tags: "es-java"

    - index: "docker-access-%{[docker.attrs.service]}-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

         tags: "docker"

         stream: "stdout"

    - index: "docker-error-%{[docker.attrs.service]}-%{[agent.version]}-%{+yyyy.MM.dd}"

      when.contains:

         tags: "docker"

         stream: "stderr"

setup.template.name: "nginx"

setup.template.pattern: "nginx-*"

setup.template.overwrite: true

setup.template.enabled: true

setup.ilm.enabled: false

访问后

但是没有docker-error-httpd*

经过日志访问后，发现没有stderr的这个标记

关于Docker的日志收集介绍到这里

码农公寓