shell进行Nginx日志分析

• 日志格式

   log_format combined_realip '$remote_addr $http_x_forwarded_for [$time_local]'
   ' $host "$request_uri" $status'
   ' "$http_referer" "$http_user_agent"';
  

• 日志条目

    95.213.177.126 - - [07/Jul/2019:13:17:30 +0800] "POST http://check.proxyradar.com/azenv.php?auth=156247664979&a=PSCN&i=2018476175&p=80 HTTP/1.1" 404 27 "http://best-proxies.ru/" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)"
  

1. 统计日PV量

    [root@shuaiaoli ~]# grep "07/Jul/2019" /usr/local/nginx/logs/access.log |wc -l
   38
   

2. 统计访问量最多的10个ip

    [root@shuaiaoli ~]# awk  '{print $1}' /usr/local/nginx/logs/access.log|sort -n |uniq -c |sort -nr |head -n10
       615 219.144.130.208
       271 80.82.70.187
       120 120.132.3.65
       102 5.39.67.11
        82 139.162.88.63
        57 163.177.13.2
        43 95.213.177.126
        43 61.219.11.153
        41 95.213.177.124
        41 42.51.209.64
        
   [root@shuaiaoli ~]# awk '{ips[$1]++ } END{for( i in ips ){ print i,ips[i] }}' /usr/local/nginx/logs/access.log |sort -k2 -rn |head -n10 
   219.144.130.208 615
   80.82.70.187 271
   120.132.3.65 120
   5.39.67.11 102
   139.162.88.63 82
   163.177.13.2 57
   95.213.177.126 43
   61.219.11.153 43
   95.213.177.124 41
   42.51.209.64 41
   

3. 统计大于100次的IP

   [root@shuaiaoli ~]# awk  '{print $1}' /usr/local/nginx/logs/access.log|sort -n |uniq -c |awk '{if ($1>100) {print $0}}'
       102 5.39.67.11
       271 80.82.70.187
       120 120.132.3.65
       615 219.144.130.208
       
   [root@shuaiaoli ~]# awk '{ips[$1]++ } END{for( i in ips ){ if (ips[i]>100) {print i,ips[i] }}}' /usr/local/nginx/logs/access.log |sort -k2 -rn |head -n10 
   219.144.130.208 615
   80.82.70.187 271
   120.132.3.65 120
   5.39.67.11 102
   

4. 统计前一分钟的PV量

   [root@shuaiaoli ~]# data=$(date -d '1 minute' +%d/%b/%Y:%H:%M)
   [root@shuaiaoli ~]# awk -v a=$data '$0 ~ a {i++} END {print i}' /usr/local/nginx/logs/access.log