ELK - logstash 动态指定动态模板

基于 es: 7.10.x

写在前面

通过logstash写到es的数据，es 默认匹配logstash* 的模板， 并且 索引( index )名称也会自动加上 logstash-, 默认的 logstash 模板为:

GET /_template/logstash
{
  "logstash" : {
    "order" : 0,
    "version" : 60001,
    "index_patterns" : [
      "logstash-*"
    ],
    "settings" : {
      "index" : {
        "number_of_shards" : "1",
        "refresh_interval" : "5s"
      }
    },
    "mappings" : {
      "dynamic_templates" : [
        {
          "message_field" : {
            "path_match" : "message",
            "mapping" : {
              "norms" : false,
              "type" : "text"
            },
            "match_mapping_type" : "string"
          }
        },
        {
          "string_fields" : {
            "mapping" : {
              "norms" : false,
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "ignore_above" : 256,
                  "type" : "keyword"
                }
              }
            },
            "match_mapping_type" : "string",
            "match" : "*"
          }
        }
      ],
      "properties" : {
        "@timestamp" : {
          "type" : "date"
        },
        "geoip" : {
          "dynamic" : true,
          "properties" : {
            "ip" : {
              "type" : "ip"
            },
            "latitude" : {
              "type" : "half_float"
            },
            "location" : {
              "type" : "geo_point"
            },
            "longitude" : {
              "type" : "half_float"
            }
          }
        },
        "@version" : {
          "type" : "keyword"
        }
      }
    },
    "aliases" : { }
  }
}

自定义模板

配置模板内容

现在我们要匹配 boot开头的索引，使用 tpl-boot.json 模板

创建一个名称为 tpl-boot.json 的文件, 主要是为了区别 logstash 模板, 动态模板可以自己改， 确保这个模板可执行正确，不然logstash会创建失败:

xiao@z:/opt/soft/lib/dc/elk/data/logstash/tpl$ cat tpl-boot.json 
{
    "index_patterns" : [
      "boot-*"
    ],
    "order" : 1,
    "settings" : {
      "index" : {
        "number_of_shards" : "1",
        "refresh_interval" : "5s"
      }
    },
    "mappings" : {
      "dynamic_templates" : [
        {
          "message_field" : {
            "path_match" : "message",
            "mapping" : {
              "norms" : false,
              "type" : "text"
            },
            "match_mapping_type" : "string"
          }
        },
        {
          "string_fields" : {
            "mapping" : {
              "norms" : false,
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "ignore_above" : 256,
                  "type" : "keyword"
                }
              }
            },
            "match_mapping_type" : "string",
            "match" : "*"
          }
        }
      ],
      "properties" : {
        "@timestamp" : {
          "type" : "date"
        },
        "geoip" : {
          "dynamic" : true,
          "properties" : {
            "ip" : {
              "type" : "ip"
            },
            "latitude" : {
              "type" : "half_float"
            },
            "location" : {
              "type" : "geo_point"
            },
            "longitude" : {
              "type" : "half_float"
            }
          }
        },
        "@version" : {
          "type" : "keyword"
        }
      }
    }
}

配置 logstash.conf

xiao@z:/opt/soft/lib/dc/elk/data/logstash/config$ cat logstash.conf 
input {
    syslog {
        port => "514"
    }
    redis {
        data_type => "pattern_channel"
        key => "logstash-*"
        host => "192.168.1.123"
        port => 6379
        threads => 1
        password => "111111" #如果有安全认证，此项为密码
        type => redis
    }

	//监听
    tcp{
        port => 5044
        codec => plain{charset => "UTF-8"}
        type => boot
    }
}

output { 

    if [type] == "redis"{
        elasticsearch {
            action => "index"
            hosts => ["192.168.1.123:9200"]
            index => "logstash-%{type}-%{+YYYY.MM.dd}"
        }
    }
    else if [type] == "boot"{
        elasticsearch {
            //如果滑创建模板，则加上
            document_type => "_doc"
            action => "index"
            hosts => ["192.168.1.123:9200"]
            index => "boot-%{+YYYY.MM.dd}"
            //true,代表交给logstash管理模板，false,使用自定义
            manage_template => true
            template => "/usr/share/logstash/config/tpl/tpl-boot.json"
            template_name => "tpl-boot.json"
            template_overwrite => true
        }
    }
    //默认输出到控制台
    stdout { codec => rubydebug }
}

如果在控制台看见下面，就说明创建成功:

[logstash.outputs.elasticsearch][main] Attempting to install template 
{
	manage_template => {
		"index_patterns" => ["boot-*"], "order" => 1, "settings" => {
			"index" => {
				"number_of_shards" => "1", "refresh_interval" => "5s"
			}
		},
		"mappings" => {
			"dynamic_templates" => [{
				"message_field" => {
					"path_match" => "message",
					"mapping" => {
						"norms" => false, "type" => "text"},
					"match_mapping_type" => "string"
				}
			}, {
				"string_fields" => {
					"mapping" => {
						"norms" => false, "type" => "text",
						"fields" => {"keyword" => {
								"ignore_above" => 256, "type" => "keyword"}}},
					"match_mapping_type" => "string", "match" => "*"
				}
			}],
			"properties" => {
				"@timestamp" => {"type" => "date"}, "geoip" => {
					"dynamic" => true, "properties" => {
						"ip" => {"type" => "ip"},
						"latitude" => {
							"type" => "half_float"}, "location" => {
							"type" => "geo_point"
						}, "longitude" => {
							"type" => "half_float"}}},
				"@version" => {
					"type" => "keyword"}
}}}}