使用正则表达式来处理Nginx 日志
一、
先对单行的日志进行分组正则匹配，返回匹配后的结果(字典格式)：
from datetime import datetime
import re #单行日志
logline = '''183.60.212.153 - - [19/Feb/2013:10:23:29 +0800] "GET /o2o/media.html?menu=3 HTTP/1.1" 200 16691 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"''' #对每行匹配正则，提取匹配后的字典
def extract(line):
pattern = '''(?P<remote_addr>[\d\.]{7,}) - - (?:\[(?P<datetime>[^\[\]]+)\]) "(?P<request>[^"]+)" (?P<status>\d+) (?P<size>\d+) "(?:[^"]+)" "(?P<user_agent>[^"]+)"'''
regex = re.compile(pattern)
matcher = regex.match(line)
return matcher.groupdict()
#日志格式key与对应的处理函数 #写入新字典，key,value print(extract(logline))
输出结果：
{'request': 'GET /o2o/media.html?menu=3 HTTP/1.1', 'size': '16691', 'remote_addr': '183.60.212.153', 'status': '200', 'datetime': '19/Feb/2013:10:23:29 +0800', 'user_agent': 'Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)'}
二、
上面返回结果中再对部分内容细分处理，比如以下四部分：
'request': 'GET /o2o/media.html?menu=3 HTTP/1.1'
'size': '16691'
'status': '200'
'datetime': '19/Feb/2013:10:23:29 +0800'
request可以再细分请求方式(method)，请求地址(url)，协议版本(protocol)
size可以直接转换成整数，而不是字符串
status也可以转换位整数
datetime可以转换成其它格式（2013-02-19 10:23:29+08:00）
时间格式化解析字符串
%a 星期几的英文缩写 Sun, Mon, ..., Sat
%A 星期几的英文全拼 Sunday, Monday, ..., Saturday
%w 星期几的数字表示格式，0是星期天,1是星期一...6是星期六
%d 天 01, 02, ..., 31
%b 月份的英文缩写 Jan, Feb, ..., Dec
%Y 年份的4位的十进制整数 Year 0001, 0002, ..., 2013, 2014, ..., 9998, 9999
%H 小时 Hour(24小时制) 00, 01, ..., 23
%I 小时 Hour(12小时制) 01, 02, ..., 12
%M 分钟的零填充的十进制整数 Minute(01,02,03...59)
%S 秒的零填充的十进制整数 Second(01,02,03...59)
%z 时区偏移 UTC时区偏移大小 (empty), +0000, -0400, +1030
from datetime import datetime
import re #单行日志
logline = '''183.60.212.153 - - [19/Feb/2013:10:23:29 +0800] "GET /o2o/media.html?menu=3 HTTP/1.1" 200 16691 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"''' #对每行匹配正则，提取匹配后的字典
def extract(line):
pattern = '''(?P<remote_addr>[\d\.]{7,}) - - (?:\[(?P<datetime>[^\[\]]+)\]) "(?P<request>[^"]+)" (?P<status>\d+) (?P<size>\d+) "(?:[^"]+)" "(?P<user_agent>[^"]+)"'''
regex = re.compile(pattern)
matcher = regex.match(line)
return matcher.groupdict() #对request分别切割成请求方式(method)，请求地址(url)，协议版本(protocol)
def convert_request(request):
return dict(zip(('method','url','protocol'),request.split())) def convert_time(timestr):
formatstr = '%d/%b/%Y:%H:%M:%S %z'
ts = datetime.strptime(timestr,formatstr)
return ts #日志格式key与对应的处理函数,进一步对日志格式化处理 'request': 'GET /o2o/media.html?menu=3 HTTP/1.1'
log_format_func = {
'request':convert_request,
'size':int,
'status':int,
'datetime':convert_time
} #写入新字典，key,value
d = {}
for k,v in extract(logline).items():
# print(k,v)
d[k] = log_format_func.get(k,lambda x:x)(v) print(d)
输出结果：
{'request': {'method': 'GET', 'protocol': 'HTTP/1.1', 'url': '/o2o/media.html?menu=3'}, 'remote_addr': '183.60.212.153', 'datetime': datetime.datetime(2013, 2, 19, 10, 23, 29, tzinfo=datetime.timezone(datetime.timedelta(0, 28800))), 'size': 16691, 'status': 200, 'user_agent': 'Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)'}
三、
request 和 datetime处理的函数再简写成lambda 表达式
from datetime import datetime
import re logline = '''183.60.212.153 - - [19/Feb/2013:10:23:29 +0800] "GET /o2o/media.html?menu=3 HTTP/1.1" 200 16691 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"''' def extract(line):
pattern = '''(?P<remote_addr>[\d\.]{7,}) - - (?:\[(?P<datetime>[^\[\]]+)\]) "(?P<request>[^"]+)" (?P<status>\d+) (?P<size>\d+) "[^"]+" "(?P<user_agent>[^"]+)"'''
regex = re.compile(pattern)
matcher = regex.match(line)
if matcher:
return {k: ops.get(k, lambda x: x)(v) for k, v in matcher.groupdict().items()}
else:
raise Exception('No match') ops = {
'datetime': lambda timestr: datetime.strptime(timestr, "%d/%b/%Y:%H:%M:%S %z"),
'request': lambda request: dict(zip(('method', 'url', 'protocol'), request.split())),
'status': int,
'size': int
} if __name__ == '__main__':
log_pro = extract(logline)
print(log_pro)
# for k, v in log_pro.items():
# print(k, v)
输出结果：
{'remote_addr': '183.60.212.153', 'request': {'url': '/o2o/media.html?menu=3', 'method': 'GET', 'protocol': 'HTTP/1.1'}, 'status': 200, 'size': 16691, 'datetime': datetime.datetime(2013, 2, 19, 10, 23, 29, tzinfo=datetime.timezone(datetime.timedelta(0, 28800))), 'user_agent': 'Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)'}
remote_addr: 183.60.212.153
request: {'url': '/o2o/media.html?menu=3', 'method': 'GET', 'protocol': 'HTTP/1.1'}
status: 200
size: 16691
datetime: 2013-02-19 10:23:29+08:00
user_agent: Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)