最终架构确定为 logs--->blieb--->redis/kafka--->logstash--->es--->kibana
注意： geoip下载地址：
wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz
filebeat 配置文件
filebeat-nginx
filebeat.prospectors:
- type: log
paths:
- /usr/local/nginx/logs/access.log #nginx路径
# tags: ["access"]
fields:
app: www #所属于的项目
type: nginx-access
fields_under_root: true #目录制顶 - type: log
paths:
- /usr/local/nginx/logs/error.log
# ags: ["error"]
fields:
app: www
type: nginx-error
fields_under_root: true output.redis: #输入的redis
hosts: ["192.168.0.215"] #ip 密码 key 数据库 数据类型
password: ""
key: "filebeat"
db: 0
datatype: list
filebeat-tomcat
filebeat.prospectors:
- type: log
paths:
- /usr/local/tomcat/logs/catalina.out
# tags: ["tomcat"]
fields:
app: www
type: tomcat-catalina
fields_under_root: true
multiline:
pattern: '^\[' #自定义正则
negate: true
match: after output.redis:
hosts: ["192.168.0.215"]
password: ""
key: "filebeat"
db: 0
datatype: list
nginx-json格式化
log_format json '{ "@timestamp": "$time_iso8601", '
'"remote_addr": "$remote_addr", '
'"remote_user": "$remote_user", '
'"body_bytes_sent": "$body_bytes_sent", '
'"request_time": "$request_time", '
'"status": "$status", '
'"request_uri": "$request_uri", '
'"request_method": "$request_method", '
'"http_referrer": "$http_referer", '
'"http_x_forwarded_for": "$http_x_forwarded_for", '
'"http_user_agent": "$http_user_agent"}';
logstash配置文件
logstash-to-es-nginxjson.com （nginx json格式化日志）
input { #进入的类型
redis {
host => "192.168.0.215"
port => 6379
password => ""
db => ""
data_type => "list"
key => "filebeat"
}
}
filter {
if [app] == "www" { #判断项目
if [type] == "nginx-access" { #判断类型
json {
source => "message"
remove_field => ["message"]
}
geoip {
source => "remote_addr"
target => "geoip"
database => "/opt/GeoLite2-City.mmdb" #geoip 数据库地址 可以自定义
add_field => ["[geoip][coordinates]", "%{[geoip][longitude]}"]
add_field => ["[geoip][coordinates]", "%{[geoip][latitude]}"]
}
mutate {
convert => ["[geoip][coordinates]", "float"]
}
}
}
}
output {
elasticsearch {
hosts => ["http://192.168.0.212:9200","http://192.168.0.213:9200","http://192.168.0.214:9200"] #es集群
index => "logstash-%{type}-%{+YYYY.MM.dd}"
}
stdout{codec => rubydebug }
}
logstash-to-es-custom.conf 自定义格式化
input {
redis {
host => "192.168.0.215"
port => 6379
password => ""
db => ""
data_type => "list"
key => "filebeat"
}
}
filter {
if [app] == "www" {
if [type] == "nginx-access" {
grok {
match => {
"message" => "%{IPV4:remote_addr} - (%{USERNAME:remote_user}|-) \[%{HTTPDATE:time_local}\] \"%{WORD:request_method} %{URIPATHPARAM:request_uri} HTTP/%{NUMBER:http_protocol}\" %{NUMBER:http_status} %{NUMBER:body_bytes_sent} \"%{GREEDYDATA:http_referer}\" \"%{GREEDYDATA:http_user_agent}\" \"(%{IPV4:http_x_forwarded_for}|-)\""
} #自定义格式化
overwrite => ["message"]
}
geoip {
source => "remote_addr"
target => "geoip"
database => "/opt/GeoLite2-City.mmdb"
add_field => ["[geoip][coordinates]", "%{[geoip][longitude]}"]
add_field => ["[geoip][coordinates]", "%{[geoip][latitude]}"]
}
date {
locale => "en"
match => ["time_local", "dd/MMM/yyyy:HH:mm:ss Z"]
}
mutate {
convert => ["[geoip][coordinates]", "float"]
}
}
}
}
output {
elasticsearch {
hosts => ["http://192.168.0.212:9200","http://192.168.0.213:9200","http://192.168.0.214:9200"]
index => "logstash-%{type}-%{+YYYY.MM.dd}"
}
stdout{codec => rubydebug }
}