ELK获取nginx日志

2022-09-07 22:47:53

ELK下载地址

https://www.elastic.co/cn/downloads/a
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.12.1-linux-x86_64.tar.gz
wget https://artifacts.elastic.co/downloads/logstash/logstash-7.12.1-linux-x86_64.tar.gz
wget https://artifacts.elastic.co/downloads/kibana/kibana-7.12.1-linux-x86_64.tar.gz
wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.12.1-linux-x86_64.tar.gz

安装elaticSearch

hostnamectl set-hostname node-1  
yum -y install java*
tar -zxvf  elasticsearch-7.12.1-linux-x86_64.tar.gz
mv   elasticsearch-7.12.1/config/elasticsearch.yml elasticsearch-7.12.1/config/elasticsearch.yml.bak
vi elasticsearch.yml

cluster.initial_master_nodes: ["node-1"]    
cluster.name: es-application
node.name: node-1

network.host: 0.0.0.0
http.port: 9200

path.data: /home/elk/elasticsearch-7.12.1/data
path.logs: /home/elk/elasticsearch-7.12.1/logs

http.cors.enabled: true
http.cors.allow-origin: "*"

useradd elk
useradd elk
chown -R elk:elk /home/elk/elasticsearch-7.12.1
vi /etc/security/limits.conf
* soft nofile 65536
* hard nofile 65536
vi  /etc/sysctl.conf
vm.max_map_count=655360

务必执行:

reboot
systemctl stop firewalld
systemctl enable firewalld
su elk
./elasticsearch-7.12.1/bin/elasticsearch -d

查看是否启动了 9200 9300端口

netstat -nltp
访问  http://192.168.43.116:9200/

ELK获取nginx日志

安装Logstash
以nginx日志为例

tar -zxvf  logstash-7.12.1-linux-x86_64.tar.gz
sudo rpm -ivh http://nginx.org/packages/centos/7/noarch/RPMS/nginx-release-centos-7-0.el7.ngx.noarch.rpm
yum repolist
yum install nginx
systemctl enable nginx

附上一份nginx配置，因为yum之后配置文件貌似有点少，有其他需求自行更改
将图片放到/home/images/下进行测试

http://192.168.43.116:8088/TEST.png
vi /etc/nginx/nginx.conf

user  root;
worker_processes  auto;

error_log  /var/log/nginx/error.log notice;
pid        /var/run/nginx.pid;

events {
    worker_connections  1024;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/elk_access.log  main;

    sendfile        on;
    #tcp_nopush     on;
    keepalive_timeout  65;
    #gzip  on;

#日志获取的字段
    log_format main2 '$http_host $remote_addr - $remote_user [$time_local] "$request" '
                     '$status $body_bytes_sent "$http_referer" '
                     '"$http_user_agent" "$upstream_addr" $request_time';

    server {
        listen       8088;#写内网端口，访问时用外网端口进行映射访问
        server_name  localhost;

        #charset koi8-r;
        #access_log  logs/host.access.log  main;

        location ~ .*\.(gif|jpg|jpeg|png)$ {
            expires 24h;
            root /home/images/;#指定图片存放路径
            access_log /etc/nginx/logs/images.log;#图片 日志路径
            proxy_store on;
            proxy_store_access user:rw group:rw all:rw;
            proxy_temp_path         /home/images/;#代理临时路径
            proxy_redirect          off;
#日志路径
            access_log  /var/log/nginx/elk_access.log  main2;

            proxy_set_header        Host 127.0.0.1;
            proxy_set_header        X-Real-IP $remote_addr;
            proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
            client_max_body_size    10m;
            client_body_buffer_size 1280k;
            proxy_connect_timeout   900;
            proxy_send_timeout      900;
            proxy_read_timeout      900;
            proxy_buffer_size       40k;
            proxy_buffers           40 320k;
            proxy_busy_buffers_size 640k;
            proxy_temp_file_write_size 640k;
            if ( !-e $request_filename)
            {
                 proxy_pass  http://127.0.0.1:8088;#代理访问地址,和上面的端口一致
            }
        }

        location / {
            root   html;
            index  index.html index.htm;
            add_header X-Frame-Options SAMEORIGIN;
            proxy_set_header Host $host:$server_port;
            proxy_set_header   X-Real-IP        $remote_addr;
            proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
#           alias  /home/images/;
            try_files $uri $uri/ /index.html last;
        }

        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }

        # proxy the PHP scripts to Apache listening on 127.0.0.1:80
        #
        #location ~ \.php$ {
        #    proxy_pass   http://127.0.0.1;
        #}

        # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
        #
        #location ~ \.php$ {
        #    root           html;
        #    fastcgi_pass   127.0.0.1:9000;
        #    fastcgi_index  index.php;
        #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
        #    include        fastcgi_params;
        #}

        # deny access to .htaccess files, if Apache's document root
        # concurs with nginx's one
        #
        #location ~ /\.ht {
        #    deny  all;
        #}
    }

    # another virtual host using mix of IP-, name-, and port-based configuration
    #
    #server {
    #    listen       8000;
    #    listen       somename:8080;
    #    server_name  somename  alias  another.alias;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}
    # HTTPS server
    #
    #server {
    #    listen       443 ssl;
    #    server_name  localhost;

    #    ssl_certificate      cert.pem;
    #    ssl_certificate_key  cert.key;

    #    ssl_session_cache    shared:SSL:1m;
    #    ssl_session_timeout  5m;

    #    ssl_ciphers  HIGH:!aNULL:!MD5;
    #    ssl_prefer_server_ciphers  on;

    include /etc/nginx/conf.d/*.conf;
}
systemctl restart nginx

编辑logstash 配置

vi  logstash-7.12.1/config/nginx_access.conf

input {
  file {
    path => "/var/log/nginx/elk_access.log"                 #设置为nginx访问日志的路径
    start_position => "beginning"
    type => "nginx"
  }
}
filter {
    grok {
        match => { "message" => "%{IPORHOST:http_host} %{IPORHOST:clientip} - %{USERNAME:remote_user} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:http_verb} %{NOTSPACE:http_request}(?: HTTP/%{NUMBER:http_version})?|%{DATA:raw_http_request})\" %{NUMBER:response} (?:%{NUMBER:bytes_read}|-) %{QS:referrer} %{QS:agent} %{QS:xforwardedfor} %{NUMBER:request_time:float}"}
    }
    geoip {
        source => "clientip"
    }
}
output {
    stdout { codec => rubydebug }
    elasticsearch {
        hosts => ["192.168.43.116:9200"]                #也可以为集群内其它机器的地址
        index => "nginx-test-%{+YYYY.MM.dd}"
  }
}

两种启动方式，一种打印日志一种不打印，建议使用第一种，可以看到报错，当可以正常启动之后再用第二种

 ./logstash-7.12.1/bin/logstash -f  logstash-7.12.1/config/nginx_access.conf

nohup  ./logstash-7.12.1/bin/logstash -f  logstash-7.12.1/config/nginx_access.conf &

ELK获取nginx日志

注：logstash出现如下报错，是之前运行的instance有缓冲

--------------------------------------------------------------------------------------------------------------------------------
Logstash could not be started because there is already another instance using the configured data directory.  If you wish to run multiple instances, you must change the "path.data" setting.

需要进入/logstash-7.12.1/data  删除.lock 文件之后重新启动即可

cd /logstash-7.12.1/data && rm -rf .lock
--------------------------------------------------------------------------------------------------------------------------------

安装Kibana

tar -zxvf kibana-7.12.1-linux-x86_64.tar.gz
mv   kibana-7.12.1-linux-x86_64/config/kibana.yml  kibana-7.12.1-linux-x86_64/config/kibana.yml.bak
vi  kibana-7.12.1-linux-x86_64/config/kibana.yml

server.port: 5601
server.host: "192.168.43.116"
elasticsearch.hosts: ["http://192.168.43.116:9200"]
i18n.locale: "zh-CN"    #kibana设置中文模式
chown -R elk:elk   kibana-7.12.1-linux-x86_64
su - elk
nohup ./kibana-7.12.1-linux-x86_64/bin/kibana &
http://192.168.43.116:5601

ELK获取nginx日志

重启kinbana（需要的时候在执行）

netstat -anltp|grep 5601
kill -9  （LISTEN后面的端口）

创建索引收集日志

ELK获取nginx日志

ELK获取nginx日志

ELK获取nginx日志

ELK获取nginx日志

ELK获取nginx日志

ELK获取nginx日志

选择创建的索引

ELK获取nginx日志

ELK获取nginx日志

如果没有日志调整下时间

ELK获取nginx日志

因为开了护眼模式所以截图颜色有点变化

上一篇：ELk-学习之filebeat(摘抄）


下一篇：Tomcat AJP协议漏洞分析与利用