1 整体目录结构

root@shutang:/home/shutang/docker-scripts# tree log-scripts/
log-scripts/
|-- README.MD
|-- curator
|   |-- Dockerfile
|   |-- curator_prod.yml
|   |-- curator_qa.yml
|   |-- delete_indices.yml
|   `-- docker-compose.yml
|-- docker-compose.yml
|-- es
|   `-- elasticsearch.yml
|-- fluentd
|   |-- Dockerfile
|   `-- conf
|       `-- fluent.conf
`-- kibana
    |-- Dockerfile
    `-- kibana.yml

2 部署efk服务的docker-compose文件

version: '3'

services:
  fluentd:
    build: ./fluentd
    restart: always
    volumes:
      - /mnt/disks/append-disk/fluentd/log:/fluentd/log
      - ./fluentd/conf/fluent.conf:/fluentd/etc/fluent.conf
    ports:
      - "24224:24224"
      - "24224:24224/udp"
    environment:
      - "discovery.zen.ping.unicast.hosts=elasticsearch"
    networks:
      - esnet

  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:6.8.22
    container_name: elasticsearch
    restart: always
    environment:
      - node.name=master1
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms6g -Xmx6g"
      - network.publish_host=10.175.100.100
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - esdata1:/usr/share/elasticsearch/data
      - ./es/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
    expose:
      - 9200
      - 9300
    ports:
      - 9200:9200
      - 9300:9300
    networks:
      - esnet

  kibana:
    build: ./kibana
    restart: always
    environment:
      - "discovery.zen.ping.unicast.hosts=elasticsearch"
    ports:
      - "5601:5601"
    networks:
      - esnet
    volumes:
      - ./kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
volumes:
  esdata1:
    driver: local
networks:
  esnet:

2 es/elasticsearch.yml 文件 [两个es实例同时作为master]

cluster.name: "docker-cluster"
network.host: 0.0.0.0

# minimum_master_nodes need to be explicitly set when bound on a public IP
# set to 1 to allow single node clusters
# Details: https://github.com/elastic/elasticsearch/pull/17288
discovery.zen.minimum_master_nodes: 1
discovery.zen.ping.unicast.hosts: ["10.175.100.100", "10.175.100.101"]
xpack.security.enabled: false

3 fluentd/Dockerfile 文件

FROM fluent/fluentd:v1.5

USER root

RUN echo "source 'https://mirrors.tuna.tsinghua.edu.cn/rubygems/'" > Gemfile && gem install bundler

RUN gem install fluent-plugin-elasticsearch -v 4.0.3   --no-document
#RUN gem install fluent-plugin-elasticsearch  --no-document
RUN gem install fluent-plugin-concat

USER fluent

CMD ["fluentd", "-o", "/fluentd/log/fluentd.log", "--log-rotate-age", "daily", "--log-rotate-size", "102400"]

4 /fluentd/conf/fluent.conf

<source>
  @type forward
  port 24224
  bind 0.0.0.0
</source>

<filter *.log>
  @type concat
  @log_level trace
  key log
  multiline_start_regexp /^\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}|^- GET|^\d{4}-\d{1,2}-\d{1,2}.\d{1,2}:\d{1,2}:\d{1,2}/
  multiline_end_regexp /\n$/
</filter>

<match *.log>
  @type copy
  <store>
    @type elasticsearch
    host 10.175.100.100
    port 9200
    logstash_format true
    logstash_prefix mxlog
    logstash_dateformat %Y%m%d
    include_tag_key true
    tag_key @log_name
    flush_interval 1s
  </store>
  <store>
    @type stdout
  </store>
</match>

<match *.**>
  @type copy
  <store>
    @type elasticsearch
    host 10.175.100.100
    port 9200
    logstash_format true
    logstash_prefix log
    logstash_dateformat %Y%m%d
    include_tag_key true
    tag_key @log_name
    flush_interval 1s
  </store>
  <store>
    @type stdout
  </store>
</match>

5 10.175.100.101 部署master2

root@shutang:/home/shutang/docker-scripts/log-scripts# cat es2-dc.yml
version: '3'

services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:6.8.22
    container_name: elasticsearch-slave
    restart: always
    environment:
      - node.name=master2
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
      - network.publish_host=10.175.100.101
    expose:
      - 9200
      - 9300
    ports:
      - 9200:9200
      - 9300:9300
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - ./es/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /mnt/disks/append/es-data:/usr/share/elasticsearch/data

6 es/elasticsearch.yml

cluster.name: "docker-cluster"
network.host: 0.0.0.0

# minimum_master_nodes need to be explicitly set when bound on a public IP
# set to 1 to allow single node clusters
# Details: https://github.com/elastic/elasticsearch/pull/17288
discovery.zen.minimum_master_nodes: 1
discovery.zen.ping.unicast.hosts: ["10.175.100.100", "10.176.100.101"]