docker 部署 filebeat 采集日志导入到elasticsearch 设置pipeline

filebeat.yml

filebeat.inputs:
 - type: log #默认log，从日志文件读取每一行。stdin，从标准输入读取
   paths:
    - /root/logs/*/*.log
   multiline.pattern: '^\d{4}\-\d{2}\-\d{2}' #匹配的正则
   multiline.negate: true #多行匹配模式后配置的模式是否取反，默认false
   multiline.match: after #定义多行内容被添加到模式匹配行之后还是之前，默认无，可以被设置为after或者before
   fields:
    index: 'server_log'

setup.ilm.enabled: false
setup.template.name: "java_logback_service_index_template"
setup.template.pattern: "java_logback_service_index_template-*"
setup.template.overwrite: true
setup.template.settings:
  index.number_of_shards: 1

output.elasticsearch:
  hosts: ["127.0.0.1:9001"]
  indices:
    - index: "server_log-%{+yyyy.MM.dd}"
      when.contains:
        fields:
          index: "server_log"
  pipeline: "test_java_log_pipeline"
  document_type: log #该type会被添加到type字段，对于输出到ES来说，这个输入时的type字段会被存储，默认log
  max_retries: 3 #ES重试次数，默认3次，超过3次后，当前事件将被丢弃


processors: 
  - drop_fields:
      fields: ["log","host","input","agent","ecs","start_time"]

docker run

docker run -itd \
  --privileged=true \
  --user=root \
  --name=filebeat \
  --restart always \
  --network=my_network \
  -v /root/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml:rw \
  -v /root/filebeat/data/:/usr/share/filebeat/data/:rw  \
  -v /root/logs/:/root/logs/:rw  \
  -v /root/xiaoye_worker/logs/:/root/xiaoye_worker/logs/:rw  \
  docker.elastic.co/beats/filebeat:6.4.2

Elasticsearch 模版和pipeline 设置

GET _template/java_logback_service_index_template
DELETE _template/java_logback_service_index_template
# 创建模版索引
PUT _template/java_logback_service_index_template
{
   
  "order": 1,
  "index_patterns": [
    "java_log-*"
  ],
  "settings": {
   
    "number_of_shards": 1,
    "number_of_replicas": 1
  },
  "mappings": {
   
    "type_name":{
   
      "properties": {
   
      "app_name": {
   
        "type": "keyword"
      },
      "trance_id": {
   
        "type": "keyword"
      },
      "log_level": {
   
        "type": "keyword"
      },
      "thread": {
   
        "type": "keyword"
      },
      "class_line": {
   
        "type":