一 Filebeat工作原理
Filebeat由两个主要组件组成: prospector和 harvester
1.1 harvester
- 负责读取单个文件的内容
- 如果文件在读取时被制除或重命名， Filebeat将继续读取文件。
1.2 prospector
- prospector负责管理 harvester并找到所有要读取的文件来源
- 如果输入类型为日志，则查找器将查找路径匹配的所有文件，并为每个文件启动一个 harvester
- Filebeat目前支持两种 prospector类型:log和 stdin
1.3 Filebeat如何保持文件的状态
- Filebeat保存每个文件的状
- 并经常将状态新到磁盘上的注册文件中
- 该状态用于记住 harvester正在读取的最后偏移量，并确保发送所有日志行。
- 如果输出（例如 Elasticsearch或 Logstash）无法访问， Filebeat会跟踪最后发送的行，并在输出再次可用时继续读取文件
- 在 Filebeat运行时，每个 prospect内存中也会保存的文件状态信息，当重新启动 Filebeat时，将使用注册文件的数据来重建文件状态， Filebeat将毎个 harvester在从保存的最后偏移量继续读取
- 文件状态记录在data/ registry文件中
1.4 启动命令
./filebeat -e -c darren-log.yml
./filebeat -e -c darren-log.yml -d "publish"
#参数说明
-e：输出到标准输出，默认输出到syslog和logs下
-c：指定配置文件
-d：输出debug信息
#测试
[root@node4 filebeat]# ./filebeat -e -c darren-log.yml -d "publish"
{
"_index": "filebeat-7.4.2-2019.11.24-000001",
"_type": "_doc",
"_id": "O4vOnG4BNbSd3xvSaBQk",
"_version": ,
"_score": ,
"_source": {
"@timestamp": "2019-11-24T09:46:40.787Z",
"log": {
"offset": ,
"file": {
"path": "/opt/logs/a.log"
}
},
"message": "",
"tags": [
"web"
,
"test"
],
"input": {
"type": "log"
},
"from": "test-web",
"ecs": {
"version": "1.1.0"
},
"host": {
"name": "node4"
},
"agent": {
"version": "7.4.2",
"type": "filebeat",
"ephemeral_id": "be331c63-1342-432b-b969-1c51955f184d",
"hostname": "node4",
"id": "2832793b-3bb6-4081-b05f-1955815440d0"
}
}
}
二 把filebeat整合到nginx
[root@node4 ~]# cd /usr/local/filebeat/
[root@node4 filebeat]# vi nginx-log.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /usr/local/nginx/logs/*.log
tags: ["nginx"]
setup.template.settings:
index.number_of_shards: 3
output.elasticsearch:
hosts: ["192.168.132.131","192.168.132.132","192.168.132.133"]
#output.console:
# pretty: true
# enable: true
2.1 删除filebeat*的索引
2.2 启用这个配置
[root@node4 filebeat]# ./filebeat -e -c nginx-log.yml
已经有数据写入
2.3 看原始数据
{
"_index": "filebeat-7.4.2-2019.11.24-000001",
"_type": "_doc",
"_id": "QYvPnW4BNbSd3xvSqhRe",
"_version": ,
"_score": ,
"_source": {
"@timestamp": "2019-11-24T14:27:41.402Z",
"host": {
"name": "node4"
},
"agent": {
"type": "filebeat",
"ephemeral_id": "622a2491-72d6-4c5a-936c-2d7b0d796d3b",
"hostname": "node4",
"id": "2832793b-3bb6-4081-b05f-1955815440d0",
"version": "7.4.2"
},
"log": {
"offset": ,
"file": {
"path": "/usr/local/nginx/logs/access.log"
}
},
"message": "192.168.132.1 - - [24/Nov/2019:03:15:12 -0500] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"", #请求的日志
"tags": [
"nginx"
],
"input": {
"type": "log"
},
"ecs": {
"version": "1.1.0"
}
}
}
可以看到，在message中已经获取到了你滚下日志，但是内容并没有经过处理，遂于后期的日志处理不利，可以使用Module解决
2.4 Module配置
前面想要实现日志数据的读取及处理都是使用手动配置的，在filebeat中，有大量的module，可以简化我们的配置直接使用就可以
[root@node4 filebeat]# ./filebeat modules list
Enabled: Disabled:
apache
auditd
aws
cef
cisco
coredns
elasticsearch
envoyproxy
googlecloud
haproxy
ibmmq
icinga
iis
iptables
kafka
kibana
logstash
mongodb
mssql
mysql
nats
netflow
nginx
osquery
panw
postgresql
rabbitmq
redis
santa
suricata
system
traefik
zeek
没有启用module，如果需要启用需要进行enable操作
2.5 启用nginx的module
[root@node4 filebeat]# ./filebeat modules enable nginx #启用
[root@node4 filebeat]# ./filebeat modules disbale nginx #禁用
[root@node4 filebeat]# ./filebeat modules list
Enabled:
nginx Disabled:
apache
auditd
aws
cef
cisco
coredns
elasticsearch
envoyproxy
googlecloud
haproxy
ibmmq
icinga
iis
iptables
kafka
kibana
logstash
mongodb
mssql
mysql
nats
netflow
osquery
panw
postgresql
rabbitmq
redis
santa
suricata
system
traefik
zeek
配置nginx module
[root@node4 filebeat]# cd modules.d/
[root@node4 modules.d]# vi nginx.yml
# Module: nginx
# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.4/filebeat-module-nginx.html - module: nginx
# Access logs
access:
enabled: true
var.paths: ["/usr/local/nginx/logs/access.log*"]
# Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
#var.paths: # Error logs
error:
enabled: true
var.paths: ["/usr/local/nginx/logs/error.log*"] # Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
#var.paths:
2.6 配置yml文件
[root@node4 modules.d]# cd ../
[root@node4 filebeat]# vi nginx-log.yml
filebeat.inputs:
#- type: log
# enabled: true
# paths:
# - /usr/local/nginx/logs/*.log
# tags: ["nginx"]
setup.template.settings:
index.number_of_shards: 3
output.elasticsearch:
hosts: ["192.168.132.131","192.168.132.132","192.168.132.133"]
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enable: false
#output.console:
# pretty: true
# enable: true
[root@node4 filebeat]# ./filebeat -e -c nginx-log.yml
--24T09::43.129- INFO crawler/crawler.go: Loading Inputs:
--24T09::43.131- INFO log/input.go: Configured paths: [/usr/local/nginx/logs/access.log*]
--24T09::43.132- INFO log/input.go: Configured paths: [/usr/local/nginx/logs/error.log*]
--24T09::43.132- INFO crawler/crawler.go: Loading and starting Inputs completed. Enabled inputs:
--24T09::43.132- INFO cfgfile/reload.go: Config reloader started
--24T09::43.134- INFO log/input.go: Configured paths: [/usr/local/nginx/logs/access.log*]
--24T09::43.136- INFO log/input.go: Configured paths: [/usr/local/nginx/logs/error.log*]
刷新nginx网页
2.7 查看原始数据
{
"_index": "filebeat-7.4.2-2019.11.24-000001",
"_type": "_doc",
"_id": "TIvvnW4BNbSd3xvSwhR5",
"_version": ,
"_score": ,
"_source": {
"agent": {
"hostname": "node4",
"id": "2832793b-3bb6-4081-b05f-1955815440d0",
"type": "filebeat",
"ephemeral_id": "1e036f81-346b-42a6-b5c6-f197e9ba149c",
"version": "7.4.2"
},
"nginx": {
"access": {
"remote_ip_list": [
"192.168.132.1"
]
}
},
"log": {
"file": {
"path": "/usr/local/nginx/logs/access.log"
},
"offset":
},
"source": {
"address": "192.168.132.1",
"ip": "192.168.132.1"
},
"fileset": {
"name": "access"
},
"url": {
"original": "/"
},
"input": {
"type": "log"
},
"@timestamp": "2019-11-24T15:02:38.000Z",
"ecs": {
"version": "1.1.0"
},
"service": {
"type": "nginx"
},
"host": {
"name": "node4"
},
"http": {
"request": {
"referrer": "-",
"method": "GET"
},
"response": {
"status_code": ,
"body": {
"bytes":
}
},
"version": "1.1"
},
"event": {
"timezone": "-05:00",
"created": "2019-11-24T15:02:44.599Z",
"module": "nginx",
"dataset": "nginx.access"
},
"user": {
"name": "-"
},
"user_agent": {
"original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",
"os": {
"name": "Windows 10"
},
"name": "Chrome",
"device": {
"name": "Other"
},
"version": "78.0.3904"
}
}
}
配置模块后，message的信息前后如下
模块配置后，信息更可读化
Nginx的filebeat的module配置结束