ELK日志系统：Elasticsearch+Logstash+Kibana+Filebeat搭建教程

2021-12-02 16:56:37

ELK日志系统：Elasticsearch + Logstash + Kibana 搭建教程

系统架构

安装配置JDK环境

JDK安装（不能安装JRE）

JDK下载地址：http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html

下载包：jdk-8u131-linux-x64.rpm

yum localinstall jdk-8u131-linux-x64.rpm

mvn 安装

cd /usr/local

wget http://www-eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz

tar xzf apache-maven-3.3.9-bin.tar.gz

mv apache-maven-3.3.9 maven

vi /etc/profile.d/maven.sh

export M2_HOME=/usr/local/maven

export PATH=${M2_HOME}/bin:${PATH}

source /etc/profile.d/maven.sh

mvn -version

安装ElasticSearch

yum install epel-release

yum install npm nodejs

# centos7 若安装nodejs失败，请执行如下命令再重试

rpm -ivh https://kojipkgs.fedoraproject.org//packages/http-parser/2.7.1/3.el7/x86_64/http-parser-2.7.1-3.el7.x86_64.rpm

wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.2.3.rpm

yum localinstall elasticsearch-6.2.3.rpm

# 修改network.host: 0.0.0.0

vim /etc/elasticsearch/elasticsearch.yml

systemctl start elasticsearch

systemctl enable elasticsearch

systemctl status elasticsearch

# elasticsearch工具目录

/usr/share/elasticsearch/bin/

# 系统要求

vim /etc/security/limits.conf

* soft nofile 65535

* hard nofile 65535

vim /etc/sysctl.conf

vm.max_map_count=262144

# 临时生效命令

sysctl -w vm.max_map_count=262144

安装elasticsearch-head

# 增加新的参数，这样head插件可以访问es

vim /etc/elasticsearch/elasticsearch.yml

http.cors.enabled: true

http.cors.allow-origin: "*"

cd /usr/share/elasticsearch

git clone git://github.com/mobz/elasticsearch-head.git

cd elasticsearch-head

npm install

npm run start

# elasticsearch-head访问地址

http://localhost:9100/

# 若head插件无法连接到es，编辑app.js查找9200修改参数localhost为本机ip

vim _site/app.js

安装filebeat

wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.2.3-x86_64.rpm

yum localinstall filebeat-6.2.3-x86_64.rpm

vim /etc/filebeat/filebeat.yml

# 修改paths配置路径

# 将enabled设置为true！！

# 将Filebeat和Logstash连接起来

# 将output.elasticsearch注释掉#

# 打开Logstash的注释

# 修改完成后的配置如下：

grep -vE "^$|#|;" /etc/filebeat/filebeat.yml

filebeat.prospectors:

- type: log

  enabled: true

  paths:

    - /var/log/*.log

  exclude_lines: ['^DBG', '^OK','^$'] #排查DBG、OK和空行

  include_lines: ['^ERR', '^WARN']

  exclude_files: ['.gz$', '*error.log']

filebeat.config.modules:

  path: ${path.config}/modules.d/*.yml

  reload.enabled: false

setup.template.settings:

  index.number_of_shards: 3

setup.kibana:

output.logstash:

  hosts: ["localhost:5044"]

# 启动filebeat

systemctl start filebeat

systemctl enable filebeat

systemctl status filebeat

安装logstash

wget https://artifacts.elastic.co/downloads/logstash/logstash-6.2.3.rpm

yum localinstall logstash-6.2.3.rpm

vim /etc/logstash/logstash.yml

# 修改path.config配置

path.config: /etc/logstash/conf.d

vim /etc/logstash/conf.d/logstash.conf

input {

    beats {

        port => 5044

    }

}

filter {

    grok {

        match => {

            "request" => "\s+(?<api_path>.+?)(\?.*)?\s+"

        }

    }

    grok {

        match => {

            "agent" => "(?<browser>Maxthon|QQBrowser|Chrome|Safari|Firefox|Opera|MSIE?)(/[0-9.]+)?"

        }

    }

    grok {

        match => {

            "agent" => "(?<os>Android|SymbianOS|Macintosh|iPad|iPhone|iPod|Linux|Windows?)"

        }

    }

    mutate {

        split => [ "upstreamtime", "," ]

    }

}

output {

    elasticsearch {

        hosts => ["192.168.1.216:9200"]

        index => "logstash-%{+YYYY.MM.dd}_log"

    }

    stdout { codec => rubydebug }

}

# 给logstash做软连接

ln -s /usr/share/logstash/bin/logstash /usr/bin/logstash

systemctl start logstash

systemctl enable logstash

systemctl status logstash

cd /usr/share/logstash/bin

# 解析配置文件并报告任何出现错误的错误

logstash -f logstash.conf --config.test_and_exit

# 窗口启动 (以下启动方式不推荐，服务启动即可)

logstash -f /etc/logstash/conf.d/logstash.conf

# 后台运行

nohup logstash -f /etc/logstash/conf.d &

nohup logstash -f /etc/logstash/conf.d > logstash.log 2>&1 &

安装kibana

wget https://artifacts.elastic.co/downloads/kibana/kibana-6.2.3-x86_64.rpm

yum localinstall kibana-6.2.3-x86_64.rpm

vim /etc/kibana/kibana.yml

# 修改elasticsearch.url参数

server.host: "0.0.0.0"

elasticsearch.url: "http://localhost:9200"

systemctl start kibana

systemctl enable kibana

systemctl status kibana

安装nginx

yum install nginx httpd-tools

htpasswd -c /etc/nginx/htpasswd.users XXX

vi /etc/nginx/conf.d/kibana.conf

server {

    listen 80;

    server_name 192.168.1.216;

    auth_basic "Restricted Access";

    auth_basic_user_file /etc/nginx/htpasswd.users;

    location / {

        proxy_pass http://localhost:5601;

        proxy_http_version 1.1;

        proxy_set_header Upgrade $http_upgrade;

        proxy_set_header Connection 'upgrade';

        proxy_set_header Host $host;

        proxy_cache_bypass $http_upgrade;

    }

}

systemctl enable nginx

systemctl start nginx

验证

echo "hello world" >/var/opt/log/a.log

curl http://localhost:9200/_search?pretty 查看输出

删除索引

curl -XDELETE http://localhost:9200/twitter

curl -XDELETE http://localhost:9200/_all

列出所有索引

curl -u elastic:changeme 'http://localhost:9200/_cat/indices?v'

查看节点个数

curl http://localhost:9200/_cluster/health?pretty

已知bug

Chrome浏览器插件可能导致kibana显示存在bug，可通过禁用浏览器插件浏览

码农公寓