k8s部署Kafka-Mysql-Flume-Flink详细讲解

准备部署环境

Kubernetes集群信息

NAME	VERSION
k8s-master01-123	v1.27.0
k8s-master02-124	v1.27.0
k8s-master03-125	v1.27.0
k8s-node01-126	v1.27.0

Kafka：2.8.1版本
Zookeeper：3.9.3版本

Mysql：5.7.36版本

Flume：1.8.0版本

准备StorageClass

[root@k8s-master01-123 ~]# kubectl get sc
NAME         PROVISIONER      RECLAIMPOLICY   VOLUMEBINDINGMODE      ALLOWVOLUMEEXPANSION   AGE
fc-storage   nfs-fc-storage   Delete          WaitForFirstConsumer   false                  7d19h

创建namespace

Kubectl create ns databases

部署zookeeper

编写zookeeper部署zk.yaml文件

apiVersion: v1
kind: Service
metadata:
  name: zk-cs
  namespace: databases
spec:
  selector:
    app: zk-cs
  ports:
  - name: client
    port: 2181
    protocol: TCP
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: zk-cs
  namespace: databases
spec:
  selector:
    matchLabels:
      app: zk-cs
  serviceName: zk-cs
  replicas: 1
  template:
    metadata:
      labels:
        app: zk-cs
    spec:
      #nodeName: k8s-node01-126
      containers:
      - name: zk-cs
        image: zookeeper
        imagePullPolicy: IfNotPresent
        ports:
        - name: client
          containerPort: 2181
        readinessProbe:
          tcpSocket:
            port: client
          initialDelaySeconds: 10
          periodSeconds: 5 

kubectl apply -f zk.yaml

 

部署kafka

编写kafka部署kafka.yaml文件

apiVersion: apps/v1
kind: Deployment
metadata:
  name: kafka
  namespace: databases
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kafka
  template:
    metadata:
      labels:
        app: kafka
    spec:
      containers:
      - name: kafkad
        image: wurstmeister/kafka
#       imagePullPolicy: IfNotPresent
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 9092
        command:
        - sh 
        - -c 
        - "exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties --override broker.id=0 \
          --override listeners=PLAINTEXT://:9092 \
          --override advertised.listeners=PLAINTEXT://192.168.33.124:31092 \
          --override zookeeper.connect=zk-cs-0.zk-cs.databases.svc.cluster.local:2181,zk-cs-1.zk-cs.databases.svc.cluster.local:2181,zk-cs-2.zk-cs.databases.svc.cluster.local:2181/kafka \
          --override auto.create.topics.enable=true \
          --override zookeeper.session.timeout.ms=6000 \
          --override zookeeper.set.acl=false \
          --override log.dirs=/opt/kafka/logs" 
        env:
        - name: MY_POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name  
        - name: ALLOW_PLAINTEXT_LISTENER
          value: "yes"
        - name: KAFKA_HEAP_OPTS
          value : "-Xms1g -Xmx1g" 
        #volumeMounts:
        #- mountPath: /opt/kafka
        #  name: kafka-data
      #volumes:
       # - name: kafka-data
        #  persistentVolumeClaim:
         #   claimName: kafka-dep
---
apiVersion: v1
kind: Service
metadata:
  name: kafka-service
  namespace: databases
spec:
  type: NodePort
  ports:
  - port: 31092
    targetPort: 9092
    nodePort: 31092
  selector:
    app: kafka

kubectl apply -f kafka.yaml

 

部署mysql

编写mysql部署mysql.yaml文件

apiVersion: apps/v1
kind: Deployment
metadata:
  name: devops-mysql   # deployment控制器名称
  namespace: databases
spec:
  replicas: 1
  revisionHistoryLimit: 5
  strategy:
    type: RollingUpdate
  selector:
    matchLabels:
      app: devops-mysql
  template:
    metadata:
      labels:
        app: devops-mysql
    spec:
      volumes:
        - name: devops-mysql
          nfs:
            server: 192.168.33.126  
            path: /home/nfs_data/mysql   
      containers:
        - name: devops-mysql
#docker.m.daocloud.io/library/mysql:5.7
          image: mysql:5.7
          env:
            - name: MYSQL_ROOT_PASSWORD
              value: iopYN!234     
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 3306
          volumeMounts:
            - name: devops-mysql
              mountPath: /var/lib/mysql
---
apiVersion: v1
kind: Service
metadata:
  name: devops-mysql    # 数据库服务的名称
  namespace: databases
spec:
  ports:
    - port: 3306
      protocol: TCP
      targetPort: 3306
      nodePort: 30001    # 对外访问的端口
  selector:
    app: devops-mysql
  type: NodePort
  sessionAffinity: ClientIP 

kubectl apply -f mysql.yaml

 

部署flume

提前创建flume-config.yaml配置

apiVersion: v1
data:
  flume.conf: |-
    # Name the components on this agent
    app.sources = r1
    app.channels = c1

    # Describe/configure the source
    app.sources.r1.type = exec
    app.sources.r1.command = tail -F /opt/flume/messages
    app.sources.r1.channels = c1

    #filter
    #app.sources.r1.interceptors=filt1 filt2 filt3
    app.sources.r1.interceptors=filt2
    #app.sources.r1.interceptors.filt1.type=regex_filter
    #app.sources.r1.interceptors.filt1.regex=*containerd*
    app.sources.r1.interceptors.filt2.type=regex_filter
    app.sources.r1.interceptors.filt2.regex=level
    #app.sources.r1.interceptors.filt2.type=regex_filter
    #app.sources.r1.interceptors.filt3.regex=*msg*


    # Use a channel which buffers events in KafkaChannel
    # 设置app.channels.c1的类型
    app.channels.c1.type = org.apache.flume.channel.kafka.KafkaChannel
    # 设置Kafka集群中的Broker 集群以逗号分割
    app.channels.c1.kafka.bootstrap.servers = 192.168.33.124:31092
    # 设置app.channels.c1使用的Kafka的Topic
    app.channels.c1.kafka.topic = mytopic1
    # 设置成不按照flume event格式解析数据,因为同一个Kafka topic可能有非flume Event类数据传入
    app.channels.c1.parseAsFlumeEvent = false
    # 设置消费者组,保证每次消费时能够获取上次对应的Offset
    app.channels.c1.kafka.consumer.group.id = test-consumer-group
    # 设置消费过程poll()超时时间(ms)
    app.channels.c1.pollTimeout = 1000
kind: ConfigMap
metadata:
  name: flume-conf
  namespace: databases
  resourceVersion: '247196'

kubectl apply -f flume-config.yaml

 

编写flume部署flume.yaml文件, 

---
apiVersion: apps/v1
kind: Deployment
metadata:
  annotations: {}
  labels:
    k8s.kuboard.cn/name: flume
  name: flume
  namespace: databases
  resourceVersion: '334556'
spec:
  progressDeadlineSeconds: 600
  replicas: 3
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s.kuboard.cn/name: flume
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      annotations:
        kubectl.kubernetes.io/restartedAt: '2025-03-19T10:34:20+08:00'
      creationTimestamp: null
      labels:
        k8s.kuboard.cn/name: flume
    spec:
      containers:
        - command:
            - sh
            - '-c'
            - >-
              bash /opt/flume/bin/flume-ng agent -c /opt/flume/conf -f
              /opt/flume-config/flume.conf -n app
              -Dflume.root.logger=INFO,console
          env:
            - name: FLUME_AGENT_NAME
              value: agent
          image: probablyfine/flume
          imagePullPolicy: IfNotPresent
          name: flume
          resources: {}
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /opt/flume-config
              name: volume-pptr2
            - mountPath: /opt/flume/messages
              name: volume-eh5jm
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 30
      volumes:
        - configMap:
            defaultMode: 420
            name: flume-conf
          name: volume-pptr2
        - hostPath:
            path: /var/log/messages
            type: File
          name: volume-eh5jm

kubectl apply -f flume.yaml

 

验证部署信息

[root@k8s-master01-123 ~]# kubectl get pod -n databases
NAME                            READY   STATUS    RESTARTS   AGE
devops-mysql-789bb8b475-vsfvr   1/1     Running   1          5h34m
flume-5d457c699b-t56pf          1/1     Running   0          172m
flume-5d457c699b-tg9rd          1/1     Running   0          172m
flume-5d457c699b-vx7q7          1/1     Running   0          172m
kafkad-75959f597f-qw5hc         1/1     Running   1          3h7m
zk-0                            0/1     Running   0          3h50m
zk-1                            1/1     Running   9          2d2h
zk-2                            1/1     Running   2          2d2h
zk-cs-0                         1/1     Running   0          3h50m

也可以k8s  web界面查看

如遇报错，网上有很多资料都可以解决