#通过 rsyslog 收集 haproxy 日志

[root@haproxy-118 ~]# yum install haproxy

[root@haproxy-118 ~]# vim /etc/haproxy/haproxy.cfg

listen kibana
bind 10.0.0.118:80
mode http
server kibana1 10.0.0.151:5601 check inter 2s fall 3 rise 5

[root@haproxy-118 ~]# vim /etc/rsyslog.conf

$ModLoad imudp
$UDPServerRun 514

local2.* @@10.0.0.154:2556

[root@haproxy-118 ~]# systemctl restart rsyslog

[root@haproxy-118 ~]# systemctl restart haproxy

root@logstash1:/etc/logstash/conf.d# vim rsyslog.conf

input {
syslog {
host => "10.0.0.154"
port => "2556"
type => "rsyslog"
}
}

output {
if [type] == "rsyslog" {
elasticsearch {
hosts => ["10.0.0.151:9200"]
index => "songyk-rsyslog-%{+YYYY.MM.dd}"
}
}
}
root@logstash1:/etc/logstash/conf.d# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/rsyslog.conf  -t

root@logstash1:/etc/logstash/conf.d# systemctl restart logstash.service 

 

 

#logstash 收集日志并写入 redis 

将web端的日志存入redis，logsstash从redis取出数据，logstash将数据通过haproxy发送给elasticsearch

root@redis:~# apt install redis

root@redis:~# vim /etc/redis/redis.conf

requirepass 12345678

save ""

#save 900 1
#save 300 10
#save 60 10000

 root@logstash1:~# vim /etc/logstash/conf.d/songyk-redis-to-es.conf

input {
redis {
data_type => "list"
key => "nginx-acceslog"
host => "10.0.0.155"
port => "6379"
db => "1"
password => "12345678"
}

redis {
data_type => "list"
key => "nginx-errorlog"
host => "10.0.0.155"
port => "6379"
db => "1"
password => "12345678"

}

redis {
data_type => "list"
key => "tomcat-accesslog"
host => "10.0.0.155"
port => "6379"
db => "0"
password => "12345678"
}

redis {
data_type => "list"
key => "systemlog"
host => "10.0.0.155"
port => "6379"
db => "0"
password => "12345678"

}

}
output {
if [type] == "nginx-acceslog" {
elasticsearch {
hosts => ["10.0.0.118:9200"]
index => "songyk-logstash-nginx-accesslog-%{+YYY.MM.dd}"
}
}

if [type] == "nginx-errorlog" {
elasticsearch {
hosts => ["10.0.0.118:9200"]
index => "songyk-logstash-nginx-errorlog-%{+YYY.MM.dd}"
}
}

if [type] == "tomcat-acceslog" {
elasticsearch {
hosts => ["10.0.0.118:9200"]
index => "songyk-logstash-tomcat-accesslog-%{+YYY.MM.dd}"
}
}

if [type] == "systemlog" {
elasticsearch {
hosts => ["10.0.0.118:9200"]
index => "songyk-logstash-systemlog-%{+YYY.MM.dd}"
}
}

}

root@web-1:/apps/apache-tomcat-8.5.57# cat /etc/logstash/conf.d/log-to-es.conf
input {
file {
path => "/apps/apache-tomcat-8.5.57/logs/tomcat_access_log.*.log"
type => "tomcat-acceslog"
start_position => "beginning"
stat_interval => "3"
codec => json
}

file {
path => "/var/log/syslog"
type => "systemlog"
start_position => "beginning"
stat_interval => "3"
}

file {
path => "/apps/nginx/logs/access.log"
type => "nginx-acceslog"
start_position => "beginning"
stat_interval => "3"
codec => json
}

file {
path => "/apps/nginx/logs/error.log"
type => "nginx-errorlog"
start_position => "beginning"
stat_interval => "3"
}
}

output {
if [type] == "tomcat-acceslog" {
redis {
data_type => "list"
key => "tomcat-accesslog"
host => "10.0.0.155"
port => "6379"
db => "0"
password => "12345678"
}
}

if [type] == "systemlog" {
redis {
data_type => "list"
key => "systemlog"
host => "10.0.0.155"
port => "6379"
db => "0"
password => "12345678"

}
}
if [type] == "nginx-acceslog" {
redis {
data_type => "list"
key => "nginx-acceslog"
host => "10.0.0.155"
port => "6379"
db => "1"
password => "12345678"

}
}

if [type] == "nginx-errorlog" {
redis {
data_type => "list"
key => "nginx-errorlog"
host => "10.0.0.155"
port => "6379"
db => "1"
password => "12345678"
}
}

}

[root@haproxy-118 ~]# vim /etc/haproxy/haproxy.cfg 

listen elasticsearch
bind 10.0.0.118:9200
mode tcp
server es1 10.0.0.151:9200 check inter 2s fall 3 rise 5
server es2 10.0.0.152:9200 check inter 2s fall 3 rise 5
server es3 10.0.0.153:9200 check inter 2s fall 3 rise 5