Kubernetes ELLK

Kubernetes ELLK

本次方案是按照 Elasticsearch + Logspout + Logstash + kibana 并且手机的日志可以被简单切分：

Elasticsearch-rc 配置文件：

apiVersion: v1 kind: ReplicationController metadata:   name: elasticsearch-logging-v1   labels:     k8s-app: elasticsearch-logging     version: v1     kubernetes.io/cluster-service: "true" spec:   replicas: 1   selector:     k8s-app: elasticsearch-logging     version: v1   template:     metadata:       labels:         k8s-app: elasticsearch-logging         version: v1         kubernetes.io/cluster-service: "true"     spec:       nodeSelector:         role: elk       containers:       - image: registry.aliyuncs.com/slzcc/elasticsearch         name: elasticsearch         resources:           limits:             cpu: 1000m           requests:             cpu: 100m         ports:         - containerPort: 9200           name: db           protocol: TCP         - containerPort: 9300           name: transport           protocol: TCP         volumeMounts:         - name: es-persistent-storage           mountPath: "/usr/share/elasticsearch/data"       volumes:       - name: es-persistent-storage         hostPath:           path: "/data/elasticsearch"

Elasticsearch-svc 配置文件：

apiVersion: v1 kind: Service metadata:   name: elasticsearch-logging   labels:     k8s-app: elasticsearch-logging     kubernetes.io/cluster-service: "true"     kubernetes.io/name: "Elasticsearch" spec:   ports:   - port: 9200     name: http     protocol: TCP     targetPort: db   selector:     k8s-app: elasticsearch-logging

Kibana-rc 配置文件：

apiVersion: extensions/v1beta1 kind: Deployment metadata:   name: kibana-logging   labels:     k8s-app: kibana-logging     kubernetes.io/cluster-service: "true" spec:   replicas: 1   selector:     matchLabels:       k8s-app: kibana-logging   template:     metadata:       labels:         k8s-app: kibana-logging     spec:       nodeSelector:         role: elk       containers:       - name: kibana-logging         image: registry.aliyuncs.com/slzcc/kibana         resources:           # keep request = limit to keep this container in guaranteed class           limits:             cpu: 100m           requests:             cpu: 100m         env:           - name: "ELASTICSEARCH_URL"             value: "http://elasticsearch-logging:9200"         ports:         - containerPort: 5601           name: ui           protocol: TCP

Kibana-svc 配置文件：

apiVersion: v1 kind: Service metadata:   name: kibana-logging   labels:     k8s-app: kibana-logging     kubernetes.io/cluster-service: "true"     kubernetes.io/name: "Kibana" spec:   ports:   - port: 5601     name: http     protocol: TCP     targetPort: ui   selector:     k8s-app: kibana-logging

Logstash-configmap 配置文件：

apiVersion: v1 kind: ConfigMap metadata:   name: logstash data:   logstash.conf: |-       input {         udp {         port => 514         type => syslog         codec => json         }         tcp {         port => 514         type => syslog         codec => json         }       }       filter {         if [type] == "syslog" {           grok {             match => { "message" => "%{SYSLOG5424PRI}%{NONNEGINT:ver} +(?:%{TIMESTAMP_ISO8601:ts}|-) +(?:%{HOSTNAME:containerid}|-) +(?:%    {NOTSPACE:containername}|-) +(?:%{NOTSPACE:proc}|-) +(?:%{WORD:msgid}|-) +(?:%{SYSLOG5424SD:sd}|-|) +%{GREEDYDATA:msg}" }           }           syslog_pri { }           date {               match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]           }           if !("_grokparsefailure" in [tags]) {               mutate {                 replace => [ "@source_host", "%{syslog_hostname}" ]                 replace => [ "@message", "%{syslog_message}" ]               }           }           mutate {             remove_field => [ "syslog_hostname", "syslog_message", "syslog_timestamp" ]           }         }       }       output {           elasticsearch {               hosts => ["elasticsearch-logging:9200"]               index => "k8s-%{type}-%{+YYYY.MM.dd}"               document_type => "%{type}"               workers => 1               flush_size => 20000               idle_flush_time => 10               template_overwrite => true               codec => json           }       }

Logstash-rc 配置文件：

apiVersion: v1 kind: ReplicationController metadata:   name: logstash   labels:     k8s-app: logstash     version: v1     kubernetes.io/cluster-service: "true" spec:   replicas: 1   selector:     k8s-app: logstash     version: v1   template:     metadata:       labels:         k8s-app: logstash         version: v1         kubernetes.io/cluster-service: "true"     spec:       nodeSelector:         role: elk       containers:       - image: registry.aliyuncs.com/slzcc/logstash-build         name: logstash         resources:           limits:             cpu: 1000m           requests:             cpu: 100m         ports:         - containerPort: 514           name: input           protocol: TCP         - containerPort: 514           name: output           protocol: UDP         command:         - '/logstash-5.1.1/bin/logstash'         - '-f'         - '/etc/logstash/logstash.conf'         - '-w 20'         volumeMounts:           - mountPath: "/etc/logstash/"             name: config-volume       volumes:         - name: config-volume           configMap:             name: logstash

Logstash-svc 配置文件：

apiVersion: v1 kind: Service metadata:   name: logstash   labels:     k8s-app: logstash     kubernetes.io/cluster-service: "true"     kubernetes.io/name: "logstash" spec:   ports:   - port: 514     name: input     protocol: TCP     targetPort: input #  - port: 514 #    name: output #    protocol: UDP #    targetPort: output   selector:     k8s-app: logstash   clusterIP: None

Lospout-daemon 配置文件：

apiVersion: extensions/v1beta1 kind: DaemonSet metadata:   name: logspout-elasticsearch   labels:     k8s-app: logspout-logging spec:   template:     metadata:       labels:         name: logspout-elasticsearch     spec:       containers:       nodeSelector:         role: elk       - name: logspout-elasticsearch         image: registry.aliyuncs.com/slzcc/logspout-logstash         resources:           limits:             memory: 200Mi           requests:             cpu: 100m             memory: 200Mi         env:           - name: "ROUTE_URIS"             value: "logstash+tcp://logstash:514"         volumeMounts:           - mountPath: "/var/run/docker.sock"             name: sock       volumes:         - hostPath:             path: "/var/run/docker.sock"           name: sock       terminationGracePeriodSeconds: 30