docker-compose部署ELK

Elastic Stack

elasticsearch集群

mkdir ~/elasticsearch/data{01...03}
mkdir ~/elasticsearch/logs{01...03}

制作数字证书

# 启动一个临时容器
sudo docker run --name elasticsearch -exec -it --rm elasticsearch:7.14.1 /bin/bash
./bin/elasticsearch-certutil ca
./bin/elasticsearch-certutil cert --ca elastic-stack-ca.p12 --out config/elastic-certificates.p12
chmod 644 config/elastic-certificates.p12
# 导出容器
sudo docker cp elasticsearch:/user/share/elasticsearch/config/elastic-certificates.p12 ~/elasticsearch/

docker-compose.yml

version: '3.8'

services:
  es01:
    image: elasticsearch:7.14.1
    container_name: es01
    privileged: true
    environment:
      - node.name=es01
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es02,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.transport.ssl.enabled=true
      - xpack.monitoring.collection.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=elastic-certificates.p12
      - "ES_JAVA_OPTS=-Xms32G -Xmx32G"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - ~/elasticsearch/data01:/usr/share/elasticsearch/data
      - ~/elasticsearches/log01:/usr/share/elasticsearch/logs
      - ~/elasticsearch/elastic-certificates.p12:/usr/share/elasticsearch/config/elastic-certificates.p12
    ports:
      - 9200:9200
    networks:
      - elastic
  es02:
    image: elasticsearch:7.14.1
    container_name: es02
    privileged: true
    environment:
      - node.name=es02
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.transport.ssl.enabled=true
      - xpack.monitoring.collection.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=elastic-certificates.p12
      - "ES_JAVA_OPTS=-Xms32G -Xmx32G"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - ~/elasticsearches/data02:/usr/share/elasticsearch/data
      - ~/elasticsearches/log02:/usr/share/elasticsearch/logs
      - ~/elasticsearches/elastic-certificates.p12:/usr/share/elasticsearch/config/elastic-certificates.p12
    networks:
      - elastic
  es03:
    image: elasticsearch:7.14.1
    container_name: es03
    privileged: true
    environment:
      - node.name=es03
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es02
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.transport.ssl.enabled=true
      - xpack.monitoring.collection.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.keystore.path=elastic-certificates.p12
      - xpack.security.transport.ssl.truststore.path=elastic-certificates.p12
      - "ES_JAVA_OPTS=-Xms32G -Xmx32G"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - ~/elasticsearches/data03:/usr/share/elasticsearch/data
      - ~/elasticsearches/log03:/usr/share/elasticsearch/logs
      - ~/elasticsearches/elastic-certificates.p12:/usr/share/elasticsearch/config/elastic-certificates.p12
    networks:
      - elastic

networks:
  elastic:
    driver: bridge

设置认证

./bin/elasticsearch-setup-passwords interactive			#设置密码 Unique#2021
Enter password for [elastic]: 
Reenter password for [elastic]: 
Enter password for [apm_system]: 
Reenter password for [apm_system]: 
Enter password for [kibana]: 
Reenter password for [kibana]: 
Enter password for [logstash_system]: 
Reenter password for [logstash_system]: 
Enter password for [beats_system]: 
Reenter password for [beats_system]: 
Enter password for [remote_monitoring_user]: 
Reenter password for [remote_monitoring_user]: 
Changed password for user [apm_system]
Changed password for user [kibana]
Changed password for user [logstash_system]
Changed password for user [beats_system]
Changed password for user [remote_monitoring_user]
Changed password for user [elastic]

kibana

kibana.yml

server.port: 5601
server.host: "0"
server.publicBaseUrl: "http://visitIP:5601"
elasticsearch.hosts: ["http://esIP:9200"]
elasticsearch.username: "elastic"
elasticsearch.password: "Unique#2021"
kibana.index: ".kibana"
i18n.locale: "zh-CN"

docker-compose.yml

version: '3.8'

services:
  kbana:
    container_name: kibana
    hostname: kibana
    image: kibana:7.14.1
    restart: always
    privileged: true
    volumes:
      - ~/kibana.yml:/usr/share/kibana/config/kibana.yml
    ports:
      - 5601:5601

logstash

mkdir -p ~/config/conf.d
touch ~/config/logstash.yml
touch ~/config/conf.d/logstash-filebeat.conf

logstash.yml

http.host: "0.0.0.0"
path.logs: /usr/share/logstash/logs

xpack.monitoring.enabled: true
xpack.monitoring.elasticsearch.hosts: ["http://esIP:9200"]
xpack.monitoring.elasticsearch.username: logstash_system
xpack.monitoring.elasticsearch.password: Unique#2021

pipelines.yml

- pipeline.id: wonotice_2
  path.config: "/usr/share/logstash/pipeline/wonotice_2.conf"
- pipeline.id: wonotice_3
  path.config: "/usr/share/logstash/pipeline/wonotice_3.conf"
- pipeline.id: wonotice_4
  path.config: "/usr/share/logstash/pipeline/wonotice_4.conf"
- pipeline.id: wonotice_5
  path.config: "/usr/share/logstash/pipeline/wonotice_5.conf"

application-1.conf

input {
  beats {
  port => 5017
  host => "0.0.0.0"
  }
}
filter {
  multiline {
  pattern => "^[\[202]"
  negate => true
  what => "previous"
  }
  grok {
  match => {"message" => "%{TIMESTAMP_ISO8601:date} .* %{LOGLEVEL:level} %{GREEDYDATA:information}"}
  }
  mutate {
    replace => {"tags" => "syncData"}
  }
  prune {
    whitelist_names => ["@timestamp","tags","date","level","information"]
  } 
} 
output {
  elasticsearch{
    hosts => ["http://10.236.14.127:9200"]
    user =>  "elastic"
    password => "Unique#2021"
    manage_template => false
    index => "syncdata-%{+YYYY.MM.dd}"
  }
}

application-2.conf

input {
  beats {
  port => 5017
  host => "0.0.0.0"
  }
}
filter {
  multiline {
  pattern => "^[\[202]"
  negate => true
  what => "previous"
  }
  grok {
  match => {"message" => "\[%{LOGLEVEL:level}.*\] %{TIMESTAMP_ISO8601:date} %{GREEDYDATA:information}"}
  }
  mutate {
    replace => {"tags" => "syncData"}
  }
  prune {
    whitelist_names => ["@timestamp","tags","date","level","information"]
  } 
} 
output {
  elasticsearch{
    hosts => ["http://10.236.14.127:9200"]
    user =>  "elastic"
    password => "Unique#2021"
    manage_template => false
    index => "syncdata-%{+YYYY.MM.dd}"
  }
}

docker-compose.yml

version: '3.8'
  
services:
  logstash:
    container_name: logstash
    hostname: logstash
    image: logstash:7.14.1
    privileged: true
    restart: always
    volumes:
      - /home/deployer/ELK/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml
      - /home/deployer/ELK/logstash/config/pipelines.yml:/usr/share/logstash/config/pipelines.yml
      - /home/deployer/ELK/logstash/config/pipeline/:/usr/share/logstash/pipeline/
      - /orabak/unique/logstash/data:/usr/share/logstash/data
    environment:
      - "LG_JAVA_OPTS=-Xms32G -Xmx32G"
    ports:
      - 5011:5011
      - 5012:5012
      - 5013:5013
      - 5014:5014
      - 5015:5015

docker-compose

sudo cp docker-compose /usr/bin/

启动

~/elasticsearch/:
	sudo docker-compose -f docker-compose.yml up -d
	sudo docker-compose down
~/kibana/:
	sudo docker-compose -f docker-compose.yml up -d
	sudo docker-compose down
~/logstah/:
	sudo docker-compose -f docker-compose.yml up -d
	sudo docker-compose down

filebeat

filebeat.yml

filebeat.inputs:

- type: log

  enabled: true
  encoding: gbk
  paths:
    - /home/unionmon/wonotice_2/logs/catalina.out
  tags: ["wonotice_2"]
  multiline.pattern: '^\['
  multiline.negate: true
  multiline.match: previous

filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false

output.logstash:
  hosts: ["logstashIP:5044"]

启动

./filebeat -e -c filebeat.yml > filebeat.log &