1 安装一个redis服务
[root@node4 ~]# yum -y install redis
直接启动
[root@node4 ~]# systemctl restart redis
[root@node4 ~]# systemctl status redis
[root@node4 ~]# redis-cli -h 127.0.0.1
2 配置filebeat，把数据传给redis
[root@node4 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
#####################################################
## Nginx log
#####################################################
- type: log
enabled: true
paths:
- /usr/local/nginx/logs/access.log
json.key_under_root: true
json.overwrite_keys: true
tags: ["access"] #- type: log
# enabled: true
# paths:
# - /usr/local/nginx/logs/error.log
# tags: ["error"] #####################################################
## tomcat log
#####################################################
- type: log
enabled: true
paths:
- /var/log/tomcat/localhost_access_log.*.txt
json.key_under_root: true
json.overwrite_keys: true
tags: ["tomcat"] #####################################################
## java log
#####################################################
- type: log
enabled: true
paths:
- /usr/local/elasticsearch/logs/my-elktest-cluster.log
tags: ["es-java"]
multiline.pattern: '^\['
multiline.negate: true
multiline.match: "after" #####################################################
## docker log
#####################################################
- type: docker
containers.ids:
- '*'
json.key_under_root: true
json.overwrite_keys: true
tags: ["docker"] #####################################################
## outout redis
#####################################################
output.redis:
hosts: ["127.0.0.1"]
key: "filebeat"
db: 0
timeout: 5
[root@node4 ~]# systemctl restart filebeat
访问产生日志
3 查看redis
127.0.0.1:6379> keys *
1) "filebeat"
127.0.0.1:6379>
127.0.0.1:6379> keys *
1) "filebeat"
127.0.0.1:6379> type filebeat #查看类型
list
127.0.0.1:6379> llen filebeat #查看长度
(integer) 22
127.0.0.1:6379> LRANGE filebeat 1 22
1) "{\"@timestamp\":\"2020-01-20T14:22:15.291Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"_doc\",\"version\":\"7.4.2\"},\"agent\":{\"hostname\":\"node4\",\"id\":\"bb3818f9-66e2-4eb2-8f0c-3f35b543e025\",\"version\":\"7.4.2\",\"type\":\"filebeat\",\"ephemeral_id\":\"663027a7-1bdc-4a9f-b9d3-1297ef06c0b0\"},\"log\":{\"offset\":21185,\"file\":{\"path\":\"/usr/local/nginx/logs/error.log\"}},\"message\":\"2020/01/20 09:22:08 [error] 2790#0: *32 open() \\\"/usr/local/nginx/html/favicon.ico\\\" failed (2: No such file or directory), client: 192.168.132.1, server: localhost, request: \\\"GET /favicon.ico HTTP/1.1\\\", host: \\\"192.168.132.134\\\", referrer: \\\"http://192.168.132.134/\\\"\",\"tags\":[\"error\"],\"input\":{\"type\":\"log\"},\"ecs\":{\"version\":\"1.1.0\"},\"host\":{\"name\":\"node4\"}}"
使用json解析
{
"@timestamp": "2020-01-20T14:22:15.293Z",
"@metadata": {
"beat": "filebeat",
"type": "_doc",
"version": "7.4.2"
},
"log": {
"offset": 21460,
"file": {
"path": "/usr/local/nginx/logs/access.log"
}
},
"json": {
"size": 612,
"xff": "-",
"upstreamhost": "-",
"url": "/index.html",
"domain": "192.168.132.134",
"upstreamtime": "-",
"@timestamp": "2020-01-20T09:22:08-05:00",
"clientip": "192.168.132.1",
"host": "192.168.132.134",
"status": "200",
"http_host": "192.168.132.134",
"Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36",
"responsetime": 0,
"referer": "-"
},
"tags": ["access"],
"input": {
"type": "log"
},
"ecs": {
"version": "1.1.0"
},
"host": {
"name": "node4"
},
"agent": {
"id": "bb3818f9-66e2-4eb2-8f0c-3f35b543e025",
"version": "7.4.2",
"type": "filebeat",
"ephemeral_id": "663027a7-1bdc-4a9f-b9d3-1297ef06c0b0",
"hostname": "node4"
}
}
4 使用logstash收集消费resdis的数据
再node4节点安装logstash
[root@node4 ~]# wget https://artifacts.elastic.co/downloads/logstash/logstash-7.5.1.rpm
[root@node4 ~]# rpm -ivh logstash-7.5.1.rpm
[root@node4 ~]# vim /etc/logstash/conf.d/logsatsh.conf
input {
redis {
host => "127.0.0.1"
port => "6379"
db => "0"
key => "filebeat"
data_type => "list"
}
}
filter{
mutate {
convert => ["upstream_time","float"]
convert => ["request_time","float"]
}
}
output{
stdout {}
elasticsearch {
hosts => "192.168.132.131:9200"
manage_template => false
index => "nginx_access-%{+yyyy.MM.dd}"
}
}
[root@node4 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logsatsh.conf
已经有索引
访问
[root@node5 ~]# ab -n 20000 -c 20 http://192.168.132.134
查看redis
127.0.0.1:6379> keys *
(empty list or set)
127.0.0.1:6379> keys *
1) "filebeat"
127.0.0.1:6379> LLEN filebeat
(integer) 16875
127.0.0.1:6379> LLEN filebeat
(integer) 16000
127.0.0.1:6379> LLEN filebeat
(integer) 15125
127.0.0.1:6379> LLEN filebeat
(integer) 14375
127.0.0.1:6379> LLEN filebeat
(integer) 13625
127.0.0.1:6379> LLEN filebeat
(integer) 13000
127.0.0.1:6379> LLEN filebeat
(integer) 12375
使用kibana查看
{
"_index": "nginx_access-2020.01.20",
"_type": "_doc",
"_id": "A1J-w28BOF7DoSFdyQr8",
"_version": 1,
"_score": null,
"_source": {
"host": {
"name": "node4"
},
"tags": [
"access"
],
"input": {
"type": "log"
},
"ecs": {
"version": "1.1.0"
},
"log": {
"file": {
"path": "/usr/local/nginx/logs/access.log"
},
"offset": 12386215
},
"json": {
"host": "192.168.132.134",
"upstreamtime": "-",
"xff": "-",
"status": "200",
"referer": "-",
"http_host": "192.168.132.134",
"Agent": "ApacheBench/2.3",
"url": "/index.html",
"responsetime": 0,
"domain": "192.168.132.134",
"size": 612,
"clientip": "192.168.132.135",
"upstreamhost": "-",
"@timestamp": "2020-01-20T10:07:11-05:00"
},
"agent": {
"hostname": "node4",
"id": "bb3818f9-66e2-4eb2-8f0c-3f35b543e025",
"type": "filebeat",
"ephemeral_id": "efddca40-1d19-4036-9724-410b1b6d4c8b",
"version": "7.4.2"
},
"@version": "1",
"@timestamp": "2020-01-20T15:07:16.439Z"
},
"fields": {
"json.@timestamp": [
"2020-01-20T15:07:11.000Z"
],
"@timestamp": [
"2020-01-20T15:07:16.439Z"
]
},
"sort": [
1579532836439
]
}
5 filebeat添加错误日志
- type: log
enabled: true
paths:
- /usr/local/nginx/logs/error.log
tags: ["error"]
output.redis:
hosts: ["127.0.0.1"]
keys:
- key: "nginx_access"
when.contains:
tags: "access"
- key: "nginx_error"
when.contains:
tags: "error"
访问的错误日志
[root@node5 ~]# ab -n 20000 -c 200 http://192.168.132.134/hehe
127.0.0.1:6379> keys *
1) "nginx_error"
2) "nginx_access"
6 有两个key，配置logstash
[root@node4 ~]# cat /etc/logstash/conf.d/logsatsh.conf
input {
redis {
host => "127.0.0.1"
port => "6379"
db => "0"
key => "nginx_access"
data_type => "list"
}
redis {
host => "127.0.0.1"
port => "6379"
db => "0"
key => "nginx_error"
data_type => "list"
}
}
filter{
mutate {
convert => ["upstream_time","float"]
convert => ["request_time","float"]
}
}
output{
stdout {}
if "access" in [tags]{
elasticsearch {
hosts => "192.168.132.131:9200"
manage_template => false
index => "nginx_access-%{+yyyy.MM.dd}"
}
}
if "error" in [tags]{
elasticsearch {
hosts => "192.168.132.131:9200"
manage_template => false
index => "nginx_error-%{+yyyy.MM.dd}"
}
}
}
[root@node4 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logsatsh.conf
7 启动后，redis被消费
127.0.0.1:6379> keys *
1) "nginx_error"
2) "nginx_access"
127.0.0.1:6379> LLEN nginx_error
(integer) 20000
127.0.0.1:6379> LLEN nginx_error
(integer) 20000
127.0.0.1:6379> LLEN nginx_error
(integer) 20000
127.0.0.1:6379> LLEN nginx_error
(integer) 20000
127.0.0.1:6379> LLEN nginx_error
(integer) 16125
127.0.0.1:6379> LLEN nginx_error
(integer) 15750
127.0.0.1:6379> LLEN nginx_access
(integer) 14625
127.0.0.1:6379> LLEN nginx_access
(integer) 14375
127.0.0.1:6379> LLEN nginx_error
(integer) 14000
127.0.0.1:6379> LLEN nginx_error
查看索引
8 优化配置
通过上面的传输，logstah重视通过tags来区分日志，所以再logstash的input中可以不配置两个key，只需要配置一个key即可
[root@node4 ~]# vim /etc/logstash/conf.d/logsatsh.conf
input {
redis {
host => "127.0.0.1"
port => "6379"
db => "0"
key => "nginx"
data_type => "list"
}
}
filter{
mutate {
convert => ["upstream_time","float"]
convert => ["request_time","float"]
}
}
output{
stdout {}
if "access" in [tags]{
elasticsearch {
hosts => "192.168.132.131:9200"
manage_template => false
index => "nginx_access-%{+yyyy.MM.dd}"
}
}
if "error" in [tags]{
elasticsearch {
hosts => "192.168.132.131:9200"
manage_template => false
index => "nginx_error-%{+yyyy.MM.dd}"
}
}
}
filebeat配置
#####################################################
## outout redis
#####################################################
output.redis:
hosts: ["127.0.0.1"]
keys: "nginx"
[root@node4 ~]# systemctl restart filebeat
[root@node4 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logsatsh.conf
访问生成日志
[root@node5 ~]# ab -n 20000 -c 200 http://192.168.132.134/hehe
查看redis
127.0.0.1:6379> keys *
1) "nginx"
127.0.0.1:6379> LLEN nginx
(integer) 20125
127.0.0.1:6379> LLEN nginx
(integer) 19625
127.0.0.1:6379> LLEN nginx
(integer) 18250
127.0.0.1:6379> LLEN nginx
(integer) 17500
127.0.0.1:6379> LLEN nginx
(integer) 16750
127.0.0.1:6379> LLEN nginx
(integer) 0
查看索引
索引生成，实验完成