查看一下agent端的shipper的配置：
# cat logstash_test2.shipper.conf
input {
file {
path => ["/apps/logstash/conf/test/test2_log.txt"]
start_position => "beginning"
sincedb_path => "/dev/null"
}
}
output {
stdout {
#codec => rubydebug
codec => json
}
}
#这个测试主要是看输出的格式为json的
先简测一下刚配好的shipper：
# ./../bin/logstash -f logstash_test2.shipper.conf -t
Sending Logstash's logs to /apps/logstash/logs which is now configured via log4j2.properties
Configuration OK
[--08T18::,][INFO ][logstash.runner ] Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash
可以看到没有报错，接下来启动logstash并指定刚才配置好的配置文件：
# ./../bin/logstash -f logstash_test2.shipper.conf -t
Sending Logstash's logs to /apps/logstash/logs which is now configured via log4j2.properties
Configuration OK
[--08T18::,][INFO ][logstash.runner ] Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash
[root@Appsrv130 conf]# ./../bin/logstash -f logstash_test2.shipper.conf
Sending Logstash's logs to /apps/logstash/logs which is now configured via log4j2.properties
[--08T18::,][INFO ][logstash.pipeline ] Starting pipeline {"id"=>"main", "pipeline.workers"=>, "pipeline.batch.size"=>, "pipeline.batch.delay"=>, "pipeline.max_inflight"=>}
[--08T18::,][INFO ][logstash.pipeline ] Pipeline main started
[--08T18::,][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>}
{"path":"/apps/logstash/conf/test/test2_log.txt","@timestamp":"2016-12-08T10:19:13.102Z","@version":"","host":"ofs1","message":"haha------>","tags":[]}{"path":"/apps/logstash/conf/test/test2_log.txt","@timestamp":"2016-12-08T10:19:13.113Z","@version":"","host":"ofs1","message":"haha------>2","tags":[]}{"path":"/apps/logstash/conf/test/test2_log.txt","@timestamp":"2016-12-08T10:19:13.118Z","@version":"","host":"ofs1","message":"haha------>3","tags":[]}{"path":"/apps/logstash/conf/test/test2_log.txt","@timestamp":"2016-12-08T10:19:13.121Z","@version":"","host":"ofs1","message":"haha------>3","tags":[]}
再看看所监控的log日志的内容：
# cat test/test2_log.txt
haha------>
haha------>
haha------>
haha------>
发现 这个shipper启动的时候会从头到尾，把配置文件全读一边（这种效里也是从配置文件中配置好的）
再看一下这个配置文件：
# cat logstash_test2.shipper.conf
input {
file {
path => ["/apps/logstash/conf/test/test2_log.txt"]
start_position => "beginning"
sincedb_path => "/dev/null"
}
}
output {
stdout {
#codec => rubydebug
codec => json
}
}
#要点就是这行sincedb_path =>"/dev/null"了！该参数用来指定sincedb文件名，但是如果我们设置为/dev/null这个linux系统上特殊的空洞文件，
那么logstash每次重启进程的时候，尝试读取sincedb内容，都只会读到空洞，也就可以理解为前不有过运行记录，自然就从初始位置开始读取了！
下面往监控文件里写入内容时，会发生下面变化：
# echo "查看json格式是什么输出－－－－－－－》">>test/test2_log.txt
再看一下输出的内容：
# ./../bin/logstash -f logstash_test2.shipper.conf -t
Sending Logstash's logs to /apps/logstash/logs which is now configured via log4j2.properties
Configuration OK
[--08T18::,][INFO ][logstash.runner ] Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash
[root@Appsrv130 conf]# ./../bin/logstash -f logstash_test2.shipper.conf
Sending Logstash's logs to /apps/logstash/logs which is now configured via log4j2.properties
[--08T18::,][INFO ][logstash.pipeline ] Starting pipeline {"id"=>"main", "pipeline.workers"=>, "pipeline.batch.size"=>, "pipeline.batch.delay"=>, "pipeline.max_inflight"=>}
[--08T18::,][INFO ][logstash.pipeline ] Pipeline main started
[--08T18::,][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>}
{"path":"/apps/logstash/conf/test/test2_log.txt","@timestamp":"2016-12-08T10:19:13.102Z","@version":"","host":"ofs1","message":"haha------>","tags":[]}{"path":"/apps/logstash/conf/test/test2_log.txt","@timestamp":"2016-12-08T10:19:13.113Z","@version":"","host":"ofs1","message":"haha------>2","tags":[]}{"path":"/apps/logstash/conf/test/test2_log.txt","@timestamp":"2016-12-08T10:19:13.118Z","@version":"","host":"ofs1","message":"haha------>3","tags":[]}{"path":"/apps/logstash/conf/test/test2_log.txt","@timestamp":"2016-12-08T10:19:13.121Z","@version":"","host":"ofs1","message":"haha------>3","tags":[]}{"path":"/apps/logstash/conf/test/test2_log.txt","@timestamp":"2016-12-08T11:17:45.060Z","@version":"","host":"ofs1","message":"查看json格式是什么输出－－－－－－－》","tags":[]}
修改配置文件：
# cat logstash_test2.shipper.conf
input {
file {
path => ["/apps/logstash/conf/test/test2_log.txt"]
start_position => "beginning"
sincedb_path => "/dev/null"
}
}
output {
stdout {
codec => rubydebug #查看这种格式的日志输出
#codec => json
}
}
查看日志:
# echo "查看rubydebug格式是什么输出－－－－－－－》">>test/test2_log.txt
# ./../bin/logstash -f logstash_test2.shipper.conf
Sending Logstash's logs to /apps/logstash/logs which is now configured via log4j2.properties
[--08T19::,][INFO ][logstash.pipeline ] Starting pipeline {"id"=>"main", "pipeline.workers"=>, "pipeline.batch.size"=>, "pipeline.batch.delay"=>, "pipeline.max_inflight"=>}
[--08T19::,][INFO ][logstash.pipeline ] Pipeline main started
[--08T19::,][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>}
{
"path" => "/apps/logstash/conf/test/test2_log.txt",
"@timestamp" => --08T11::.290Z,
"@version" => "",
"host" => "ofs1",
"message" => "haha------>",
"tags" => []
}
{
"path" => "/apps/logstash/conf/test/test2_log.txt",
"@timestamp" => --08T11::.299Z,
"@version" => "",
"host" => "ofs1",
"message" => "haha------>2",
"tags" => []
}
{
"path" => "/apps/logstash/conf/test/test2_log.txt",
"@timestamp" => --08T11::.301Z,
"@version" => "",
"host" => "ofs1",
"message" => "haha------>3",
"tags" => []
}
{
"path" => "/apps/logstash/conf/test/test2_log.txt",
"@timestamp" => --08T11::.302Z,
"@version" => "",
"host" => "ofs1",
"message" => "haha------>3",
"tags" => []
}
{
"path" => "/apps/logstash/conf/test/test2_log.txt",
"@timestamp" => --08T11::.303Z,
"@version" => "",
"host" => "ofs1",
"message" => "查看json格式是什么输出－－－－－－－》",
"tags" => []
}
{
"path" => "/apps/logstash/conf/test/test2_log.txt",
"@timestamp" => --08T11::.415Z,
"@version" => "",
"host" => "ofs1",
"message" => "查看rubydebug格式是什么输出－－－－－－－》",
"tags" => []
}
如果去掉上面的两个参数，看一下效果:
# cat logstash_test2.shipper.conf
input {
file {
path => ["/apps/logstash/conf/test/test2_log.txt"]
#start_position => "beginning"
#sincedb_path => "/dev/null"
}
}
output {
stdout {
codec => rubydebug
#codec => json
}
}
从另一个shell可以看到效果：
# ./../bin/logstash -f logstash_test2.shipper.conf
Sending Logstash's logs to /apps/logstash/logs which is now configured via log4j2.properties
[--09T13::,][INFO ][logstash.pipeline ] Starting pipeline {"id"=>"main", "pipeline.workers"=>, "pipeline.batch.size"=>, "pipeline.batch.delay"=>, "pipeline.max_inflight"=>}
[--09T13::,][INFO ][logstash.pipeline ] Pipeline main started
[--09T13::,][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>}
先导入数据:
echo '去掉参数start_position => "beginning" sincedb_path => "/dev/null"' >>test/test2_log.txt
下面看一下效果:
# ./../bin/logstash -f logstash_test2.shipper.conf
Sending Logstash's logs to /apps/logstash/logs which is now configured via log4j2.properties
[--09T13::,][INFO ][logstash.pipeline ] Starting pipeline {"id"=>"main", "pipeline.workers"=>, "pipeline.batch.size"=>, "pipeline.batch.delay"=>, "pipeline.max_inflight"=>}
[--09T13::,][INFO ][logstash.pipeline ] Pipeline main started
[--09T13::,][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>}
{
"path" => "/apps/logstash/conf/test/test2_log.txt",
"@timestamp" => --09T05::.155Z,
"@version" => "",
"host" => "ofs1",
"message" => "去掉参数start_position => \"beginning\" sincedb_path => \"/dev/null\"",
"tags" => []
}