|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
|
日志格式：log_format usgateway '$http_clientip\t$http_ServiceName\t$http_uid\t$http_sid\t[$time_local]' '\t$request\t$status\t$body_bytes_sent\t$connection_requests'
'\t$remote_addr\t$http_referer\t$http_user_agent'
'\t$request_body\t$request_time\t$msec';
日志实例：10.10.45.152---[23/Jun/2017:17:37:42 +0800]POST /sg HTTP/1.14055765910.10.130.100-Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0){\x22mbrNetName\x22:\x22\xE9\x87\x91\xE6\x98\x9F\x22,\x22nameCn\x22:\x22\xE8\x8A\x92\xE6\x9E\x9C\xE7\xBD\x91\x22,\x22gender\x22:\x2211\x22,\x22birthday\x22:\x222010-10-01\x22,\x22mbrId\x22:\x2235954629\x22,\x22emailAddr\x22:\x22334740263@qq.com\x22,\x22mobileNo\x22:\x2215902074059\x22}0.0011498210662.427logstash配置文件：input { file {
type => "uSG_gateway_access"
path => ["/usr/local/elk/elklog/nginxlog/log0/uSG_gateway_elk.log"]
}
} filter {ruby {init => "@kname = ['http_clientip','http_ServiceName','http_uid','http_sid','time_local','request','status','body_bytes_sent','connection_requests','remote_addr','http_referer','http_user_agent','request_body','request_time','msec']"code => "new_event = LogStash::Event.new(Hash[@kname.zip(event.get('message').split(''))])new_event.remove('@timestamp')event.append(new_event)"}if [request] {ruby {init => "@kname = ['method','uri','verb']"code => "new_event = LogStash::Event.new(Hash[@kname.zip(event.get('request').split(' '))])new_event.remove('@timestamp')event.append(new_event)"}}mutate {convert => ["body_bytes_sent" , "integer", "content_length", "integer", "upstream_response_time", "float","request_time", "float"]} grok {
match => [ "message", "%{IPORHOST:clientip}%{USER}%{USER}%{USER}\[%{HTTPDATE:timestamp}\]"]}date {match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ]locale => "en"} geoip {
source => "clientip" }
useragent { source => "http_user_agent"
target => "useragent"
}
}output { elasticsearch {hosts => "10.10.45.200:8201" index => "logstash-gateway-frontend-%{+YYYY.MM.dd}"
}}注意：日志分隔符为table键。 本文转自1321385590 51CTO博客，原文链接http://blog.51cto.com/linux10000/1941411：，如需转载请自行联系原作者
|