ELK5.0搭建部署

###关闭防火墙
service iptables stop
###定义vi=vim
alias vi=vim
vi ~/.bashrc
alias vi='vim'
yum -y install lrzsz*
创建文件夹
mkdir -P /opt/soft
 
上传一下包到soft路径
172.25.0.77
jdk-8u101-linux-x64.rpm
kibana-5.0.0-x86_64.rpm
 
172.25.0.78、79
elasticsearch-5.0.0.rpm
jdk-8u101-linux-x64.rpm
logstash-5.0.0.rpm
 
#####安装jdk
rpm -ivh jdk-8u101-linux-x64.rpm
java -version
 
######安装kibana(172.25.0.77)
cd /opt/soft
rpm -ivh kibana-5.0.0-x86_64.rpm
 
######安装elasticsearch、redis、logstash(172.25.0.78、172.25.0.79)
cd /opt/soft
rpm -ivh elasticsearch-5.0.0.rpm
rpm -ivh logstash-5.0.0.rpm
logstash启动脚本:
/etc/init.d/logstash

#!/bin/sh
# Init script for logstash
# Maintained by Elasticsearch
# Generated by pleaserun.
# Implemented based on LSB Core 3.1:
# * Sections: 20.2, 20.3
#
### BEGIN INIT INFO
# Provides: logstash
# Required-Start: $remote_fs $syslog
# Required-Stop: $remote_fs $syslog
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description:
# Description: Starts Logstash as a daemon.
### END INIT INFO

PATH=/sbin:/usr/sbin:/bin:/usr/bin
export PATH

if [ `id -u` -ne 0 ]; then
echo "You need root privileges to run this script"
exit 1
fi

name=logstash
pidfile="/var/run/$name.pid"

LS_USER=root
LS_GROUP=root
LS_HOME=/var/lib/logstash
LS_HEAP_SIZE="1000m"
LS_LOG_DIR=/var/log/logstash
LS_LOG_FILE="${LS_LOG_DIR}/$name.log"
LS_CONF_DIR=/etc/logstash/conf.d/
LS_OPEN_FILES=16384
LS_NICE=19
LS_OPTS=""

[ -r /etc/default/$name ] && . /etc/default/$name
[ -r /etc/sysconfig/$name ] && . /etc/sysconfig/$name

program=/usr/share/logstash/bin/logstash
args="-f ${LS_CONF_DIR} -l ${LS_LOG_FILE} ${LS_OPTS}"

start() {

LS_JAVA_OPTS="${LS_JAVA_OPTS} -Djava.io.tmpdir=${LS_HOME}"
HOME=${LS_HOME}
export PATH HOME LS_HEAP_SIZE LS_JAVA_OPTS LS_USE_GC_LOGGING

# chown doesn't grab the suplimental groups when setting the user:group - so we have to do it for it.
# Boy, I hope we're root here.
SGROUPS=$(id -Gn "$LS_USER" | tr " " "," | sed 's/,$//'; echo '')

if [ ! -z $SGROUPS ]
then
EXTRA_GROUPS="--groups $SGROUPS"
fi

# set ulimit as (root, presumably) first, before we drop privileges
ulimit -n ${LS_OPEN_FILES}

# Run the program!
nice -n ${LS_NICE} chroot --userspec $LS_USER:$LS_GROUP $EXTRA_GROUPS / sh -c "
cd $LS_HOME
ulimit -n ${LS_OPEN_FILES}
exec \"$program\" $args
" > "${LS_LOG_DIR}/$name.stdout" 2> "${LS_LOG_DIR}/$name.err" &

# Generate the pidfile from here. If we instead made the forked process
# generate it there will be a race condition between the pidfile writing
# and a process possibly asking for status.
echo $! > $pidfile

echo "$name started."
return 0
}

stop() {
# Try a few times to kill TERM the program
if status ; then
pid=`cat "$pidfile"`
echo "Killing $name (pid $pid) with SIGTERM"
kill -TERM $pid
# Wait for it to exit.
for i in 1 2 3 4 5 ; do
echo "Waiting $name (pid $pid) to die..."
status || break
sleep 1
done
if status ; then
if [ "$KILL_ON_STOP_TIMEOUT" -eq 1 ] ; then
echo "Timeout reached. Killing $name (pid $pid) with SIGKILL. This may result in data loss."
kill -KILL $pid
echo "$name killed with SIGKILL."
else
echo "$name stop failed; still running."
fi
else
echo "$name stopped."
fi
fi
}

status() {
if [ -f "$pidfile" ] ; then
pid=`cat "$pidfile"`
if kill -0 $pid > /dev/null 2> /dev/null ; then
# process by this pid is running.
# It may not be our pid, but that's what you get with just pidfiles.
# TODO(sissel): Check if this process seems to be the same as the one we
# expect. It'd be nice to use flock here, but flock uses fork, not exec,
# so it makes it quite awkward to use in this case.
return 0
else
return 2 # program is dead but pid file exists
fi
else
return 3 # program is not running
fi
}

force_stop() {
if status ; then
stop
status && kill -KILL `cat "$pidfile"`
fi
}

case "$1" in
start)
status
code=$?
if [ $code -eq 0 ]; then
echo "$name is already running"
else
start
code=$?
fi
exit $code
;;
stop) stop ;;
force-stop) force_stop ;;
status)
status
code=$?
if [ $code -eq 0 ] ; then
echo "$name is running"
else
echo "$name is not running"
fi
exit $code
;;
restart)

stop && start
;;
*)
echo "Usage: $SCRIPTNAME {start|stop|force-stop|status|restart}" >&2
exit 3
;;
esac

exit $?

######安装redis
# yum –y install tcl gcc  //安装测试依赖工具
# tar xf redis-3.2.3.tar.gz  //解压至当前目录
# cd redis-3.2.3  //切换至redis源码目录
# make  //编译
# make test  //有可能会失败,只是看看,不用在意
# make install  //安装
# mkdir /opt/redis/{db,conf} -pv  //创建redis安装目录
# cp redis.conf /opt/redis/conf/  //复制配置文件至redis安装目录
# cd src
# cp redis-benchmark redis-check-aof redis-cli redis-server mkreleasehdr.sh /opt/redis/  //复制各文件至redis安装目录
# ln -sv /opt/redis/redis-cli /usr/bin/  //将redis-cli可执行文件链接至/usr/bin目录,方便在任何目录下都可直接执行`redis-cli`
# vim /opt/redis/conf/redis.conf  //修改redis.conf 中的 `daemonize`为`yes`,让server以守护进程在后台执行,这一步可以不做,因为后面要执行的脚本会自动创建这个文件,且这个值会设置为`yes`
daemonize yes
make install仅仅在你的系统上安装了二进制文件,不会替你默认配置init脚本和配置文件,为了把它用在生产环境而安装它,在源码目录的utils目录下Redis为系统提供了
这样的一个脚本install_server.sh
# cd /opt/soft/redis-3.2.3/utils
# ./install_server.sh  //执行sh格式的安装脚本
Welcome to the redis service installer
This script will help you easily set up a running redis server
 
Please select the redis port for this instance: [6379]
Selecting default: 6379
Please select the redis config file name [/etc/redis/6379.conf] /opt/redis/conf/redis.conf
Please select the redis log file name [/var/log/redis_6379.log]
Selected default - /var/log/redis_6379.log
Please select the data directory for this instance [/var/lib/redis/6379] /opt/redis/db/6379.db
Please select the redis executable path [/usr/local/bin/redis-server]          
Selected config:
Port           : 6379
Config file    : /opt/redis/conf/redis.conf
Log file       : /var/log/redis_6379.log
Data dir       : /opt/redis/db/6379.db
Executable     : /usr/local/bin/redis-server
Cli Executable : /usr/local/bin/redis-cli
Is this ok? Then press ENTER to go on or Ctrl-C to abort.
Copied /tmp/6379.conf => /etc/init.d/redis_6379
Installing service...
Successfully added to chkconfig!
Successfully added to runlevels 345!
Starting Redis server...
Installation successful!
 # chkconfig --add redis_6379  //将redis加入系统服务
# chkconfig redis_6379 on  //加入开机启动
# vim /opt/redis/conf/redis.conf
requirepass Carsing2582#    //设置密码【坑1】:这步先不做,如果做了后面的测试连接redis的时候就需要认证)
bind 0.0.0.0  //redis要打开远程访问允许,在 redis.conf 中 bind 这一行改成 0.0.0.0(运行任何机器连接,为了测试先这样)【此为坑2】
# /etc/init.d/redis_6379 restart
# ss -an| grep 6379          //端口已经被监听,证明服务已提供
LISTEN     0      128               127.0.0.1:6379                     *:*    
TIME-WAIT  0      0                 127.0.0.1:6379             127.0.0.1:34506
 
###修改 elasticsearch配置文件 存放所收集到的日志
cd /etc/elasticsearch/
vim elasticsearch.yml
cluster.name: ELK ###集群名称
node.name: node-1 ###节点编号
node.master: true
node.data: true
path.data: /ELK/data ###elasticsearch数据路径
path.logs: /ELK/logs ###elasticsearch日志路径
network.host: zmt-elk02 ###本机主机名
discovery.zen.ping.unicast.hosts: ["zmt-elk02", "zmt-elk03"] ###自动发现节点主机名
discovery.zen.minimum_master_nodes: 2
http.cors.enabled: true
http.cors.allow-origin: "*"
xpack.security.enabled: false
 
启动异常:ERROR: bootstrap checks failed
system call filters failed to install; check the logs and fix your configuration or disable system call filters at your own risk
问题原因:因为Centos6不支持SecComp,而ES5.2.1默认bootstrap.system_call_filter为true进行检测,所以导致检测失败,失败后直接导致ES不能启动。详见 :https://github.com/elastic/elasticsearch/issues/22899
解决方法:在elasticsearch.yml中配置bootstrap.system_call_filter为false,注意要在Memory下面:
bootstrap.memory_lock: false
bootstrap.system_call_filter: false
 
设置线程池
thread_pool:
bulk:
size: 5
queue_size: 1000
search:
size: 5
queue_size: 1000
index:
size: 5
queue_size: 1000
 
#xpack.monitoring.enabled: false
#xpack.monitoring.exporters.my_local:
# type: local
 
indices.cache.filter.expire: 30d ###设置索引30天过期
index.cache.filter: 30d
 
vim /etc/elasticsearch/jvm.options ###内存酌情分配
-Xms8g
-Xmx8g
 
vi /etc/security/limits.conf
elasticsearch soft memlock unlimited
elasticsearch hard memlock unlimited
elasticsearch soft nofile 65536
elasticsearch hard nofile 131072
elasticsearch soft nproc 2048
elasticsearch hard nproc 4096
* soft nofile 65536
* hard nofile 65536
* soft memlock unlimited
* hard memlock unlimited
 
 
 
 
创建ELK数据、日志存放路径
mkdir /ELK/data
mkdir /ELK/logs
chown -R elasticsearch:elasticsearch /ELK
启动服务
service elasticsearch start
启动服务之后查看/ELK/logs路径中有日志,说明服务正常
查看端口9200、9300
9200端口是API 用来连接服务的接口
9300是内部服务
 
PS:报错
max number of threads [1024] for user [lish] likely too low, increase to at least [2048]
解决:切换到root用户,进入limits.d目录下修改配置文件。
vi /etc/security/limits.d/90-nproc.conf 
修改如下内容:
* soft nproc 1024
#修改为
* soft nproc 2048
 
##修改主机名,定义dns
vim /etc/sysconfig/network
NETWORKING=yes
HOSTNAME=zmt-elk02
GATEWAY=172.25.0.254
 
vim /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
172.25.0.77 zmt-elk01
172.25.0.78 zmt-elk02
172.25.0.79 zmt-elk03
注:重启生效 reboot
 
###配置kibana
cd /etc/kibana/
vim kibana.yml
server.port: 5601 ##定义kibana端口
server.host: "172.25.0.77" ##服务主机IP
elasticsearch.url: "http://172.25.0.78:9200" ##url调取elasticsearch
elasticsearch.url: "http://172.25.0.79:9200" ##url调取elasticsearch
######启动kibana
service kibana start
查看端口是否存在
netstat -anp | grep 5601
tcp 0 0 172.25.0.77:5601 0.0.0.0:* LISTEN 8492/node
###web页面*问
 
####安装elasticsearch-analysis-ik-5.0.0.zip 分词器
 
####安装elasticsearch-head插件
由于elasticsearch5.0版本变化较大,目前elasticsearch5.0 暂时不支持直接安装,但是head作者提供了另一种安装方法
###安装elasticsearch-head
cd /usr/share/elasticsearch
git clone git://github.com/mobz/elasticsearch-head.git
chmod -R 777 elasticsearch-head
cd elasticsearch-head/
PS:先修改license
vim package.json
"license": "Apache-2.0",
 
npm install --force ###安装完node就会有npm命令
cd /usr/share/elasticsearch/elasticsearch-head/node_modules/grunt/bin
./grunt server > /dev/null 2>&1 &
 
###安装node
node-v6.10.1-linux-x64.tar
解压
tar -xvf node-v6.10.1-linux-x64.tar
mv node-v6.10.1-linux-x64 /usr/share/elasticsearch/
vim /etc/profile
export NODE_HOME=/usr/share/elasticsearch/node-v6.10.1-linux-x64
export PATH=$PATH:$NODE_HOME/bin
source /etc/profile
cd /usr/share/elasticsearch/node-v6.10.1-linux-x64/bin/
./node -v
v6.10.1
npm -v
3.10.10
###安装grunt
######cd /usr/share/elasticsearch/elasticsearch-head
######npm install grunt-cli
###### npm install -g grunt --registry=https://registry.npm.taobao.org
grunt --version
#### 修改head目录下的Gruntfile.js配置,head默认监听127.0.0.1
vim Gruntfile.js
hostname: '*',
connect: {
server: {
options: {
port: 9100,
hostname: '*', ####修改此处
base: '.',
keepalive: true
}
}
cd _site/
vim app.js
this.base_uri = this.config.base_uri || this.prefs.get("app-base_uri") || "http://172.25.0.78:9200";
 
####elasticsearch配置允许跨域访问
修改elasticsearch配置文件elasticsearch.yml
cd /etc/elasticsearch/
vim elasticsearch.yml
http.cors.enabled: true
http.cors.allow-origin: "*"
 
####重启elasticsearch,并启动node
service elasticsearch restart ###启动elasticsearch
cd /usr/share/elasticsearch/elasticsearch-head/ ###启动node
①#######node_modules/grunt/bin/grunt server > /dev/null 2>&1 &
 
②#######/usr/share/elasticsearch/elasticsearch-head/node_modules/grunt/bin/grunt server > /dev/null 2>&1 &
 
查看集群信息
curl '172.25.0.78:9200 '
ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
172.25.0.78 13 33 0 0.01 0.01 0.00 mdi - node-1
172.25.0.79 10 33 0 0.00 0.02 0.00 mdi * node-2
 
#####安装x-pack(marvel) 进行性能监控
配置dns
vim /etc/resolv.conf
nameserver 210.73.88.1
 
kibana上安装server端
cd /usr/share/kibana/
bin/kibana-plugin install x-pack
 
es上安装agent端
cd /usr/share/elasticsearch/
bin/elasticsearch-plugin install x-pack
 
卸载
bin/elasticsearch-plugin remove x-pack
 
PS:
安装完需要重启kibana和es 默认用户名密码elastic/changeme
 
查看license时间 默认一个月
curl -XGET -u elastic:1qaz2wsx 'http://172.25.0.78:9200/_xpack/license'
替换license 默认密码changeme
curl -XPUT -u elastic 'http://172.25.0.78:9200/_xpack/license?acknowledge=true' -H "Content-Type: application/json" -d @zhaishaomin-fa370f7b-6eb7-4730-9b73-89b3d93471e3-v5.json
 
由于是基本的license,所以需要禁用安全性
vim /etc/kibana/kibana.yml
vim /etc/elasticsearch/elasticsearch.yml
xpack.security.enabled: false ####false为禁用
 
####安装logstash
rpm -ivh logstash-5.0.0.rpm
############################################################################
PS:安装出现一下错误:
warning: rpmts_HdrFromFdno: Header V3 RSA/SHA256 Signature, key ID fd431d51: NOKEY 
解决:
rpm --import /etc/pki/rpm-gpg/RPM*
############################################################################
cd /usr/share/logstash/
mkdir config
cp /etc/logstash/logstash.yml /usr/share/logstash/config/
cp /etc/logstash/log4j2.properties /usr/share/logstash/config/
######cd /etc/logstash/conf.d#######
vim input.conf
input{
#stdin{}
file{
path => "/opt/soft/sub-test.cm-inv.com_access_80.log"
start_position => "beginning" ####end(从末尾读)beginning(从头开始读)
stat_interval => 1
type => "access_log"
tags=> ["acc"]
}
 
}
 
filter{
grok{
match => [ "message" , "\[%{HTTPDATE:time_local}\] \"%{IP:client_ip}\" \"%{WORD:verb} (%{NOTSPACE:request}|-) (HTTP/%{NUMBER:http_version}|-)\" \"(%{GREEDYDATA:http_cookie}|-)\" \"(%{WORD:http_x_forword_for}|-)\" (%{GREEDYDATA:nomean}|-) (?<user>[a-zA-Z._-]+|-)(?<status>[0-9._-]+|-) (?:%{NUMBER:body_bytes_sent:int}|-) (%{BASE16FLOAT:request_time:float}|-) \"(%{GREEDYDATA:http_did}|-)\" \"(%{GREEDYDATA:http_x_up_calling_line_id}|-)\" \"(%{NOTSPACE:http_referrer}|-)\" \"%{GREEDYDATA:http_user_agent}\" (%{BASE16FLOAT}|-)(%{NUMBER:content_length}|-)(%{GREEDYDATA:upstream_addr}|-)"]
remove_field => [ "message","host","@version" ]
}
date{
match=>["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]
target=>"@timestamp"
}
geoip{
source => "client_ip"
}
}
###########
output{
elasticsearch {
hosts => "192.168.125.129:9200"
index => "logstash-%{type}-%{+YYYY.MM.dd}"
}
stdout {codec=> rubydebug }
}
################
/usr/share/logstash/bin/logstash -f input.conf
 
 
bin/logstash  -e  ‘input { stdin {} } output { stdout {codec=>rubydebug} }’
 
 
删除一个月的索引脚本
#!/bin/bash
data=`date -d '-1 months' +%Y-%m-%d`
curl -XDELETE 'http://172.25.0.77:9200/*-'${data}''
 
 
ELK学习10_ELK系列--实时日志分析系统ELK 部署与运行中的问题汇总 - wang_zhenwei的博客 - CSDN博客 http://blog.csdn.net/wang_zhenwei/article/details/50372000
 
上一篇:PHP数组关于数字键名的问题


下一篇:CTF 和 PHP ,数据库