k8s部署elk+filebeat+logstash+kafka集群（二）kafka-zookeeper模式集群

zookeeper集群部署

zookeeper使用官方镜像，版本为3.5，上传到镜像仓库

rbac

apiVersion: v1
kind: ServiceAccount
metadata:
  name: kafka
  namespace: elk
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: kafka
  labels:
    app: kafka-clusterrole
rules:
- apiGroups:
  - ""
  resources:
  - nodes
  - events
  - namespaces
  - pods
  verbs:
  - get
  - watch
  - list
- apiGroups:
  - ""
  resourceNames:
  - kafka-prospectors
  resources:
  - configmaps
  verbs:
  - get
  - update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kafka
  labels:
    app: kafka-clusterrolebinding
roleRef:
  apiGroup: ""
  kind: ClusterRole
  name: kafka
subjects:
- apiGroup: ""
  kind: ServiceAccount
  name: kafka
  namespace: elk

zk集群

# cat zk-statefulset.yaml

apiVersion: v1
kind: Service
metadata:
  name: zk-svc
  namespace: elk
  labels:
    app: zk-cluster-svc
spec:
  selector:
    app: zookeeper-3.5
  type: ClusterIP
  clusterIP: None
  #publishNotReadyAddresses: true
  sessionAffinity: ClientIP
  ports:
  - name: clientport
    port: 2181
    protocol: TCP
    targetPort: 2181
  - name: select-port
    port: 3888
    protocol: TCP
    targetPort: 3888
  - name: leader-port
    port: 2888
    protocol: TCP
    targetPort: 2888
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: zk-statefulset
  namespace: elk
spec:
  podManagementPolicy: OrderedReady
  replicas: 3
  selector:
    matchLabels:
      app: zookeeper-3.5
  serviceName: zk-svc
  template:
    metadata:
      name: zk
      namespace: elk
      labels:
        app: zookeeper-3.5
    spec:
		  serviceAccountName: kafka
      containers:
      - name: kafka
        image: 3.127.33.174:8443/elk/zookeeper:3.5
        imagePullPolicy: IfNotPresent
        resources:
          limits:
            memory: "1Gi"
            cpu: "256m"
        env:
        - name: NODE_ID
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: ZOO_SERVERS
          value: server.0=zk-statefulset-0.zk-svc:2888:3888;2181 server.1=zk-statefulset-1.zk-svc:2888:3888;2181 server.2=zk-statefulset-2.zk-svc:2888:3888;2181
        - name: ZOO_TICK_TIME
          value: "10000"
        - name: ZOO_DATA_LOG_DIR
          value: /data/log
        args: ["/bin/sh","-c","echo 'clientPort=2181' >> /conf/zoo.cfg; echo ${HOSTNAME##*-} > /data/myid; ./bin/zkServer.sh start-foreground"]
        ports:
        - name: clientport
          containerPort: 2181
          protocol: TCP
        - name: leaderport
          containerPort: 2888
          protocol: TCP
        - name: select-leader
          containerPort: 3888
          protocol: TCP
 volumeMounts:
        - name: zk-cluster-claim
          mountPath: /data
        - name: timezone
          mountPath: /etc/localtime
        livenessProbe:
          periodSeconds: 30
          tcpSocket:
            port: 2181
      volumes:
      - name: timezone
        hostPath:
          path: /etc/localtime
      tolerations:
      - key: node-role.kubernetes.io
        effect: NoSchedule
        operator: Equal
        value: master
      - effect: NoSchedule
        operator: Exists
      affinity:
        nodeAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 1
            preference:
              matchExpressions:
              - key: node
                operator: In
                values:
                - master1
                - node1
  volumeClaimTemplates:
  - metadata:
      name: zk-cluster-claim
      namespace: elk
    spec:
      accessModes:
      - ReadWriteMany
      resources:
        limits:
          storage: 5Gi
      storageClassName: zk-managed-nfs-storage
  updateStrategy:
    type: RollingUpdate

zk集群部署完毕

kafka集群部署

镜像依旧使用之前kraft模式构建的kafka镜像

# cat kafka-zk-statefulset.yaml

apiVersion: v1
kind: Service
metadata:
  name: kafka-zk-svc
  namespace: elk
  labels:
    app: kafka-cluster-svc
spec:
  selector:
    app: kafka-zk-2.8.0
  type: ClusterIP
  clusterIP: None
  #publishNotReadyAddresses: true
  sessionAffinity: ClientIP
  ports:
  - name: outer-port
    port: 9092
    protocol: TCP
    targetPort: 9092
  - name: cluster-port
    port: 9093
    protocol: TCP
    targetPort: 9093
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: kafka-zk-statefulset
  namespace: elk
spec:
  podManagementPolicy: OrderedReady
  serviceName: kafka
  replicas: 3
  selector:
    matchLabels:
      app: kafka-zk-2.8.0
  serviceName: kafka-zk-svc
  template:
    metadata:
      name: kafka-zk
      namespace: elk
      labels:
        app: kafka-zk-2.8.0
    spec:
      serviceAccountName: kafka
      containers:
      - name: kafka-zk
        image: 3.127.33.174:8443/elk/kafka:2.8.0
        imagePullPolicy: IfNotPresent
        resources:
          limits:
            memory: "4Gi"
            cpu: "512m"
        env:
        - name: NODE_ID
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: KAFKA_HEAP_OPTS
          value: -Xmx1G -Xms1G
        - name: HOST_IP
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
        command: ["/bin/bash","-c"]
        args: ["/kafka/start-kafka.sh"]
        ports:
        - name: interactiveport
          containerPort: 9092
          protocol: TCP
        - name: controlport
          containerPort: 9093
          protocol: TCP
        volumeMounts:
        - name: kafka-cluster-claim
          mountPath: /kafka/kafka-zk-logs/
        - name: config
          mountPath: /kafka/start-kafka.sh
          subPath: start-kafka.sh
        - name: timezone
          mountPath: /etc/localtime
        livenessProbe:
          periodSeconds: 15
          tcpSocket:
            port: 9092
      volumes:
      - name: config
        configMap:
          defaultMode: 493
          name: kafka-zk-configmap
      - name: timezone
        hostPath:
          path: /etc/localtime
      tolerations:
      - key: node-role.kubernetes.io
        effect: NoSchedule
        operator: Equal
        value: master
      - effect: NoSchedule
        operator: Exists
      affinity:
        nodeAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 1
            preference:
              matchExpressions:
              - key: node
                operator: In
                values:
                - master1
                - node1
  volumeClaimTemplates:
  - metadata:
      name: kafka-cluster-claim
      namespace: elk
    spec:
      accessModes:
      - ReadWriteMany
      resources:
        limits:
          storage: 60Gi
        requests:
          storage: 40Gi
      storageClassName: kafka-zk-managed-nfs-storage
  updateStrategy:
    type: RollingUpdate

查看运行状态kubectl get pods -n elk | grep zk
进入kafka执行相关操作进行验证
kafka常用主题命令
查看操作主题命令参数

 --bootstrap-server <String: server toconnect to>
    
    连接kafka broker主机名称和端口号
    
 --topic <String: topic>
    
    操作的topic名称
    
 --create
    
    创建主题
    
 --delete
    
    删除主题
    
 --alter
    
    修改主题
    
 --list
    
    查看所有主题
    
 --describe
    
    查看主题详细描述
    
 --partitions <Integer: # of partitions>
    
    设置分区数
    
 --replication-factor <Integer: replication factor>
    
    设置分区副本
    
 --config <String: name=value>