filebeat获取nginx的access日志配置

产生nginx日志的服务器即生产者服务器配置：

拿omp.chinasoft.com举例：

.nginx.conf主配置文件添加日志格式

log_format  log_json  '{ "remote_addr": "$remote_addr", '

                          '"remote_user": "$remote_user", '

                          '"time_local": "$time_local", '

                          '"request_url": "$scheme://$host$request_uri", '

                          '"status": "$status", '

                          '"body_bytes_sent": "$body_bytes_sent", '

                          '"http_referer": "$http_referer", '

                          '"http_user_agent": "$http_user_agent", '

                          '"http_x_forwarded_for": "$http_x_forwarded_for", '

                          '"request_time": "$request_time", '

                          '"timestamp": "$msec", '

                          '"request_method": "$request_method", '

                          '"root_id": "$root_id", '

                          '"trace_id": "$http_trace_id", '

                          '"span_id": "$http_span_id" '

                        '}';

.vhosts虚拟主机配置

a.server部分添加追踪参数

####### 添加链路追踪参数 #########

set $app_name "omp.chinasoft.com";

set $root_id $pid.$msec.$remote_addr.$connection.$connection_requests;

if ($http_root_id != "")

{

    set $root_id $http_root_id;

}

.location部分添加

####### 添加链路追踪参数 #########

fastcgi_param ROOT_ID $root_id;

fastcgi_param APP_NAME $app_name;

.调整日志的路径

error_log /data/www/logs/nginx_log/error/omp.chinasoft.com_error.log;

access_log /data/www/logs/nginx_log/access/omp.chinasoft.com_access.log log_json;

# 具体配置

# cat /usr/local/nginx/config/vhost.d/omp.chinasoft.com

server {

    listen ;

    server_name omp.chinasoft.com;

    return  https://$server_name$request_uri;

}

server {

    listen  ssl;

    ssl_certificate /usr/local/nginx/config/cert2016/chinasoft_com.crt;

    ssl_certificate_key /usr/local/nginx/config/cert2016/chinasoft_com.key;

    ssl_session_cache shared:SSL1:20m;

    ssl_session_timeout 60m;

    ssl_protocols TLSv1 TLSv1. TLSv1.;

    server_name omp.chinasoft.com;

    index index.html index.php;

    root /var/www/vhosts/cloud_oa/public;

    client_max_body_size 800M;

    add_header Ws-From 'http/1.1 2.1.1.1 stable';

    set $app_name "omp.chinasoft.com";

    set $root_id $pid.$msec.$remote_addr.$connection.$connection_requests;

    if ($http_root_id != "")

    {

        set $root_id $http_root_id;

    }

    location / {

        try_files $uri $uri/ /index.php?$query_string;

    }

    location ~ /*\.php {

        fastcgi_param ROOT_ID $root_id;

        fastcgi_param APP_NAME $app_name;

        include fastcgi_params;

        fastcgi_param SCRIPT_FILENAME  $document_root$fastcgi_script_name;

        fastcgi_param APP_ENV "production";

        fastcgi_pass unix:/tmp/php5.6-fpm.sock;

        fastcgi_index index.php;

    }

    location ~ .*\.(xml|gif|jpg|jpeg|png|bmp|swf|woff|woff2|ttf|js|css)$ {

        expires 30d;

    }

    #error_log /var/log/nginx/omp.chinasoft.com_error.log;

    #access_log /var/log/nginx/omp.chinasoft.com_access.log;

    error_log /data/www/logs/nginx_log/error/omp.chinasoft.com_error.log;

    access_log /data/www/logs/nginx_log/access/omp.chinasoft.com_access.log log_json;

}

3.filebeat的配置

# cat /usr/local/filebeat-7.3.1-linux-x86_64/modules.d/nginx.yml

# Module: nginx

# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.3/filebeat-module-nginx.html

filebeat.inputs:

# 采集nginx错误日志

- type: log

  enabled: true

  paths:

    - /data/www/logs/nginx_log/error/*_error.log

  fields:

    log_type: ngx_error

  fields_under_root: true

  tail_files: true

  multiline.pattern: '^\['

  multiline.negate: true

  multiline.match: after

# 采集nginx访问日志

- type: log

  enabled: true

  paths:

    - /data/www/logs/nginx_log/access/*_access.log

  fields:

    log_type: ngx_access

  fields_under_root: true

  tail_files: true

  exclude_lines: ['\.(xml|gif|jpg|jpeg|png|bmp|swf|woff|woff2|ttf|js|css|svg|ico)']

# 输出

output.kafka:

  hosts: ["1.1.1.1:19092"]

  topic: filebeats-log

# 启动filebeat命令

./filebeat -c modules.d/nginx.yml

/usr/local/filebeat-7.3.1-linux-x86_64/filebeat -c /usr/local/filebeat-7.3.1-linux-x86_64/modules.d/nginx.yml &

查看kafka日志具体内容的命令：

kafka配置路径，可以查看日志存放的路径：

config/server.config

/usr/local/elk/kafka/bin/kafka-run-class.sh kafka.tools.DumpLogSegments --files /data/kafka/logs/filebeats-log-omp-0/00000000000000000000.log --print-data-log

filebeat udp监听配置

filebeat.inputs:

# 监听udp

- type: udp

  enabled: true

  host: "0.0.0.0:9999"

output.kafka:

  hosts: ["10.10.18.242:9092"]

  topic: filebeats-log

# 采集者

processors:

- drop_fields:

    fields: ["ecs",  "input", "agent", "@metadata", "tags"]