elasticsearch 慢日志收集，采用fluent

发布一个k8s部署视频：https://edu.youkuaiyun.com/course/detail/26967

课程内容：各种k8s部署方式。包括minikube部署，kubeadm部署，kubeasz部署，rancher部署，k3s部署。包括开发测试环境部署k8s，和生产环境部署k8s。

腾讯课堂连接地址https://ke.qq.com/course/478827?taid=4373109931462251&tuin=ba64518

第二个视频发布  https://edu.youkuaiyun.com/course/detail/27109

腾讯课堂连接地址https://ke.qq.com/course/484107?tuin=ba64518

介绍主要的k8s资源的使用配置和命令。包括configmap，pod，service，replicaset，namespace，deployment，daemonset，ingress，pv，pvc，sc，role，rolebinding，clusterrole，clusterrolebinding，secret，serviceaccount，statefulset，job，cronjob，podDisruptionbudget，podSecurityPolicy,networkPolicy,resourceQuota，limitrange，endpoint，event，conponentstatus，node，apiservice，controllerRevision等。

第三个视频发布：https://edu.youkuaiyun.com/course/detail/27574

详细介绍helm命令，学习helm chart语法，编写helm chart。深入分析各项目源码，学习编写helm插件
————————————————

慢日志配置：

elasticsearch.yml 

cluster.name: elasticsearch

node.data: ${NODE_DATA:true}
node.master: ${NODE_MASTER:true}
node.name: ${HOSTNAME}

network.host: 0.0.0.0

bootstrap.mlockall: ${BOOTSTRAP_MLOCKALL:false}

cloud:
  kubernetes:
    service: ${SERVICE}
    namespace: ${KUBERNETES_NAMESPACE}
    
discovery:
  type: kubernetes
  zen:
    minimum_master_nodes: ${MINIMUM_MASTER_NODES:2}

index.analysis.analyzer.default.type: starfish
index.similarity.default.type: ybSimilarity

index.number_of_shards: 6
index.number_of_replicas: 1
index.store.type: mmapfs
index.refresh_interval: 3600s
index.mapper.dynamic: false

path.conf: /etc/elasticsearch

index.search.slowlog.threshold.query.warn: 10s  
index.search.slowlog.threshold.query.info: 5s  
index.search.slowlog.threshold.query.debug: 2s  
index.search.slowlog.threshold.query.trace: 500ms  
  
index.search.slowlog.threshold.fetch.warn: 1s  
index.search.slowlog.threshold.fetch.info: 800ms  
index.search.slowlog.threshold.fetch.debug:500ms  
index.search.slowlog.threshold.fetch.trace: 200ms 

index.search.slowlog.threshold.index.warn: 10s
index.search.slowlog.threshold.index.info: 5s
index.search.slowlog.threshold.index.debug: 2s
index.search.slowlog.threshold.index.trace: 500ms
index.search.slowlog.level: trace
index.search.slowlog.source: 1000

monitor.jvm.gc.ParNew.info: 700ms
monitor.jvm.gc.ConcurrentMarkSweep.info: 5s

index:
  analysis:
    analyzer:
      starfish_small_query_syno:
          type: custom
          tokenizer: starfish_small_query
          filter: [my_synonym_small]
    filter:
      my_synonym_small:
          type: synonym
          ignore_case: true
          synonyms_path: analysis/synonym.txt

index:
  analysis:
    analyzer:
      starfish_syno:
          type: custom
          tokenizer: starfish_query
          filter: [my_synonym]
    filter:
      my_synonym:
          type: synonym
          ignore_case: true
          synonyms_path: analysis/synonym.txt

threadpool:
    index:
        type: fixed
        size: 100
        queue_size: 2000

threadpool:
    search:
        type: fixed
        size: 500
        queue_size: 1000

# see https://github.com/elastic/elasticsearch-definitive-guide/pull/679
processors: ${PROCESSORS:}

# avoid split-brain w/ a minimum consensus of two masters plus a data node
gateway.expected_master_nodes: ${EXPECTED_MASTER_NODES:2}
gateway.expected_data_nodes: ${EXPECTED_DATA_NODES:1}
gateway.recover_after_time: ${RECOVER_AFTER_TIME:5m}
gateway.recover_after_master_nodes: ${RECOVER_AFTER_MASTER_NODES:2}
gateway.recover_after_data_nodes: ${RECOVER_AFTER_DATA_NODES:1}

logging.yml(2.4版本)

# you can override this using by setting a system property, for example -Des.logger.level=DEBUG
es.logger.level: INFO
rootLogger: ${es.logger.level}, console, file
logger:
  # log action execution errors for easier debugging
  action: DEBUG

  # deprecation logging, turn to DEBUG to see them
  deprecation: INFO, deprecation_log_file

  # reduce the logging for aws, too much is logged under the default INFO
  com.amazonaws: WARN
  # aws will try to do some sketchy JMX stuff, but its not needed.
  com.amazonaws.jmx.SdkMBeanRegistrySupport: ERROR
  com.amazonaws.metrics.AwsSdkMetrics: ERROR

  org.apache.http: INFO

  # gateway
  #gateway: DEBUG
  #index.gateway: DEBUG

  # peer shard recovery
  #indices.recovery: DEBUG

  # discovery
  #discovery: TRACE

  index.search.slowlog: TRACE, index_search_slow_log_file
  index.indexing.slowlog: TRACE, index_indexing_slow_log_file

additivity:
  index.search.slowlog: true
  index.indexing.slowlog: true
  deprecation: true

appender:
  console:
    type: console
    layout:
      type: consolePattern
      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"

  file:
    type: dailyRollingFile
    file: ${path.logs}/${cluster.name}.log
    datePattern: "'.'yyyy-MM-dd"
    layout:
      type: pattern
      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %.10000m%n"

  # Use the following log4j-extras RollingFileAppender to enable gzip compression of log files. 
  # For more information see https://logging.apache.org/log4j/extras/apidocs/org/apache/log4j/rolling/RollingFileAppender.html
  #file:
    #type: extrasRollingFile
    #file: ${path.logs}/${cluster.name}.log
    #rollingPolicy: timeBased
    #rollingPolicy.FileNamePattern: ${path.logs}/${cluster.name}.log.%d{yyyy-MM-dd}.gz
    #layout:
      #type: pattern
      #conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"

  deprecation_log_file:
    type: dailyRollingFile
    file: ${path.logs}/${cluster.name}_deprecation.log
    datePattern: "'.'yyyy-MM-dd"
    layout:
      type: pattern
      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"

  index_search_slow_log_file:
    type: dailyRollingFile
    file: ${path.logs}/${cluster.name}_index_search_slowlog.log
    datePattern: "'.'yyyy-MM-dd"
    layout:
      type: pattern
      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"

  index_indexing_slow_log_file:
    type: dailyRollingFile
    file: ${path.logs}/${cluster.name}_index_indexing_slowlog.log
    datePattern: "'.'yyyy-MM-dd"
    layout:
      type: pattern
      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"

 

收集配置在es data节点上：

 

apiVersion: apps/v1
kind: StatefulSet
metadata:
  labels:
    app: elasticsearch
    chart: elasticsearch-0.4.9
    component: data
    heritage: Tiller
    release: es
  name: es-elasticsearch-data
  namespace: dev-es
spec:
  podManagementPolicy: OrderedReady
  replicas: 2
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: elasticsearch
      component: data
      release: es
  serviceName: es-elasticsearch-data
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: elasticsearch
        component: data
        release: es
    spec:
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - podAffinityTerm:
              labelSelector:
                matchLabels:
                  app: elasticsearch
                  component: data
                  release: es
              topologyKey: kubernetes.io/hostname
            weight: 1
      containers:
      - env:
        - name: SERVICE
          value: es-elasticsearch-master
        - name: KUBERNETES_MASTER
          value: kubernetes.default.svc.cluster.local
        - name: KUBERNETES_NAMESPACE
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
        - name: NODE_MASTER
          value: "false"
        - name: PROCESSORS
          valueFrom:
            resourceFieldRef:
              divisor: "0"
              resource: limits.cpu
        - name: ES_JAVA_OPTS
          value: -Djava.net.preferIPv4Stack=true -Xms1536m -Xmx1536m
        - name: MINIMUM_MASTER_NODES
          value: "2"
        image: 192.168.1.225:5000/elasticsearch:2.4-youben
        imagePullPolicy: Always
        lifecycle:
          preStop:
            exec:
              command:
              - /bin/bash
              - /pre-stop-hook.sh
        name: elasticsearch
        ports:
        - containerPort: 9300
          name: transport
          protocol: TCP
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /_cluster/health?local=true
            port: 9200
            scheme: HTTP
          initialDelaySeconds: 5
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 1
        resources:
          limits:
            cpu: "1"
          requests:
            cpu: 25m
            memory: 1536Mi
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /usr/share/elasticsearch/data
          name: data
        - mountPath: /usr/share/elasticsearch/config/logging.yml
          name: config
          subPath: logging.yml
        - mountPath: /pre-stop-hook.sh
          name: config
          subPath: pre-stop-hook.sh
        - name: shared-data
          mountPath: /usr/share/elasticsearch/logs
      - name: fluentd-es
        image: 192.168.1.225:5000/fluentd-elasticsearch:v2.0.4-youben
        imagePullPolicy: Always
        command: ["/bin/sh"]
        args: ["-c", "/run.sh $FLUENTD_ARGS"]
        env:
        - name: FLUENTD_ARGS
          value: --no-supervisor -q
        volumeMounts:
        - name: config-volume
          mountPath: /etc/fluent/config.d
        - name: shared-data
          mountPath: /usr/share/elasticsearch/logs
      dnsPolicy: ClusterFirst
      initContainers:
      - command:
        - sysctl
        - -w
        - vm.max_map_count=262144
        image: busybox
        imagePullPolicy: Always
        name: sysctl
        resources: {}
        securityContext:
          privileged: true
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
      - command:
        - /bin/bash
        - -c
        - chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/data && chown
          -R elasticsearch:elasticsearch /usr/share/elasticsearch/logs
        image: 192.168.1.225:5000/elasticsearch:2.4-youben
        imagePullPolicy: Always
        name: chown
        resources: {}
        securityContext:
          runAsUser: 0
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /usr/share/elasticsearch/data
          name: data
      nodeSelector:
        deploy: app
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      serviceAccount: es-elasticsearch
      serviceAccountName: es-elasticsearch
      terminationGracePeriodSeconds: 3600
      volumes:
      - configMap:
          defaultMode: 420
          name: es-elasticsearch
        name: config
      - name: config-volume
        configMap:
          name: fluentd-es-config-es-slow
      - name: shared-data
        emptyDir: {}
  updateStrategy:
    type: OnDelete
  volumeClaimTemplates:
  - metadata:
      creationTimestamp: null
      name: data
    spec:
      accessModes:
      - ReadWriteOnce
      resources:
        requests:
          storage: 30Gi
      storageClassName: glusterfs-storage-dv

、

 

fluent配置：

kind: ConfigMap
apiVersion: v1
metadata:
  name: fluentd-es-config-es-slow
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
data:
  system.conf: |-
    <system>
      root_dir /tmp/fluentd-buffers/
    </system>

  input.conf: |- 
    <source>
      @type tail
      path /usr/share/elasticsearch/logs/elasticsearch_index_search_slowlog.log
      tag elasticsearch.search_slowlog_query
      pos_file /usr/share/elasticsearch/logs/elasticsearch-search-slow.pos
      format /^\[(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\]\[(?<severity>[a-zA-Z]+\s*)\]\[(?<source>\S+)\] \[(?<node>\S+)\] \[(?<index>.+)\]\[(?<shard>\d+)\] took\[(?<took>.+)\], took_millis\[(?<took_millis>\d+)\], types\[(?<types>.*)\], stats\[(?<stats>.*)\], search_type\[(?<search_type>.*)\], total_shards\[(?<total_shards>\d+)\], source\[(?<source_body>.*)\], extra_source\[(?<extra_source>.*)\], /
    </source>

    <source>
      @type tail
      path /usr/share/elasticsearch/logs/elasticsearch_index_indexing_slowlog.log
      tag elasticsearch.indexing_slowlog_query
      pos_file /usr/share/elasticsearch/logs/elasticsearch-indexing-slow.pos
      format /^\[(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\]\[(?<severity>[a-zA-Z]+\s*)\]\[(?<source>\S+)\] \[(?<node>\S+)\] \[(?<index>.+)\]\[(?<shard>\d+)\] took\[(?<took>.+)\], took_millis\[(?<took_millis>\d+)\], type\[(?<type>.+)\], id\[(?<indexing_id>.*)\], routing\[(?<routing>.*)\], source\[(?<source_body>.*)\]/
    </source>
    
  output.conf: |-
    <match elasticsearch.*>
      @id elasticsearch.es
      @type elasticsearch
      @log_level info
      include_tag_key true
      host elasticsearch-logging.kube-system.svc.cluster.local
      port 9200
      logstash_format true
      logstash_prefix docker.es.slow
      logstash_dateformat %Y-%m-%d
      type_name docker_es_slow
      <buffer>
        @type file
        path /var/log/fluentd-buffers/kubernetes.system.buffer
        flush_mode interval
        retry_type exponential_backoff
        flush_thread_count 2
        flush_interval 5s
        retry_forever
        retry_max_interval 30
        chunk_limit_size 2M
        queue_limit_length 8
        overflow_action block
      </buffer>
    </match>