docker-compose部署EFK

1 整体目录结构

root@shutang:/home/shutang/docker-scripts# tree log-scripts/log-scripts/|-- README.MD|-- docker-compose.yml|-- es| `-- elasticsearch.yml|-- fluentd| |-- Dockerfile| `-- conf| `-- fluent.conf`-- kibana |-- Dockerfile `-- kibana.yml

2 部署efk服务的docker-compose文件

version: '3'services: fluentd: build: ./fluentd restart: always volumes: - /mnt/disks/append-disk/fluentd/log:/fluentd/log - ./fluentd/conf/fluent.conf:/fluentd/etc/fluent.conf ports: - "24224:24224" - "24224:24224/udp" environment: - "discovery.zen.ping.unicast.hosts=elasticsearch" networks: - esnet elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:6.8.22 container_name: elasticsearch restart: always environment: - node.name=master1 - bootstrap.memory_lock=true - "ES_JAVA_OPTS=-Xms6g -Xmx6g" - network.publish_host=10.175.100.100 ulimits: memlock: soft: -1 hard: -1 volumes: - esdata1:/usr/share/elasticsearch/data - ./es/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml expose: - 9200 - 9300 ports: - 9200:9200 - 9300:9300 networks: - esnet kibana: build: ./kibana restart: always environment: - "discovery.zen.ping.unicast.hosts=elasticsearch" ports: - "5601:5601" networks: - esnet volumes: - ./kibana/kibana.yml:/usr/share/kibana/config/kibana.ymlvolumes: esdata1: driver: localnetworks: esnet:

2 es/elasticsearch.yml 文件 [两个es实例同时作为master]

cluster.name: "docker-cluster"network.host: 0.0.0.0# minimum_master_nodes need to be explicitly set when bound on a public IP# set to 1 to allow single node clusters# Details: https://github.com/elastic/elasticsearch/pull/17288discovery.zen.minimum_master_nodes: 1discovery.zen.ping.unicast.hosts: ["10.175.100.100", "10.175.100.101"]xpack.security.enabled: false

3 fluentd/Dockerfile 文件

FROM fluent/fluentd:v1.5USER rootRUN echo "source 'https://mirrors.tuna.tsinghua.edu.cn/rubygems/'" > Gemfile && gem install bundlerRUN gem install fluent-plugin-elasticsearch -v 4.0.3 --no-document#RUN gem install fluent-plugin-elasticsearch --no-documentRUN gem install fluent-plugin-concatUSER fluentCMD ["fluentd", "-o", "/fluentd/log/fluentd.log", "--log-rotate-age", "daily", "--log-rotate-size", "102400"]

4 /fluentd/conf/fluent.conf

@type forward port 24224 bind 0.0.0.0 @type concat @log_level trace key log multiline_start_regexp /^d{1,3}.d{1,3}.d{1,3}.d{1,3}|^- GET|^d{4}-d{1,2}-d{1,2}.d{1,2}:d{1,2}:d{1,2}/ multiline_end_regexp /n$/ @type copy @type elasticsearch host 10.175.100.100 port 9200 logstash_format true logstash_prefix mxlog logstash_dateformat %Y%m%d include_tag_key true tag_key @log_name flush_interval 1s @type stdout @type copy @type elasticsearch host 10.175.100.100 port 9200 logstash_format true logstash_prefix log logstash_dateformat %Y%m%d include_tag_key true tag_key @log_name flush_interval 1s @type stdout

5 10.175.100.101 部署master2

root@shutang:/home/shutang/docker-scripts/log-scripts# cat es2-dc.ymlversion: '3'services: elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:6.8.22 container_name: elasticsearch-slave restart: always environment: - node.name=master2 - bootstrap.memory_lock=true - "ES_JAVA_OPTS=-Xms2g -Xmx2g" - network.publish_host=10.175.100.101 expose: - 9200 - 9300 ports: - 9200:9200 - 9300:9300 ulimits: memlock: soft: -1 hard: -1 volumes: - ./es/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml - /mnt/disks/append/es-data:/usr/share/elasticsearch/data

6 es/elasticsearch.yml

cluster.name: "docker-cluster"network.host: 0.0.0.0# minimum_master_nodes need to be explicitly set when bound on a public IP# set to 1 to allow single node clusters# Details: https://github.com/elastic/elasticsearch/pull/17288discovery.zen.minimum_master_nodes: 1discovery.zen.ping.unicast.hosts: ["10.175.100.100", "10.176.100.101"]