使用Docker-compose创建kafka集群_docker-compose kafka集群-优快云博客

本文链接：https://blog.youkuaiyun.com/m0_59388826/article/details/133989637

一.单台服务器集群

1.准备工作

单台机器：192.168.40.174

修改主机名称

vim /etc/hosts
192.168.40.174 kafka1
192.168.40.174 kafka2
192.168.40.174 kafka3

2. 编写docker-compose.yml文件

version: '3.7'
services:
  zookeeper:
    container_name: zookeeper
    hostname: zookeeper
    image: 'bitnami/zookeeper:3.7.0'
    restart: always
    ports:
      - 2181:2181
    user: root
    environment:
      - ALLOW_ANONYMOUS_LOGIN=yes
    #      ALLOW_ANONYMOUS_LOGIN: yes


  kafka1:
    image: 'wurstmeister/kafka:2.13-2.7.0'
    container_name: kafka1
    hostname: kafka1
    ports:
      - "9091:9092"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: "kafka1"
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'PLAINTEXT:PLAINTEXT'
      KAFKA_BROKER_ID: 0
      KAFKA_NUM_PARTITIONS: 3
      KAFKA_DEFAULT_REPLICATION_FACTOR: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENERS: "PLAINTEXT://kafka1:9092"
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka1:9091'
      ALLOW_PLAINTEXT_LISTENER: "yes"
    volumes:
      - ./kafka/broker1/logs:/opt/kafka/logs
      - /var/run/docker.sock:/var/run/docker.sock
    restart: always
    depends_on:
      - zookeeper


  kafka2:
    image: 'wurstmeister/kafka:2.13-2.7.0'
    container_name: kafka2
    hostname: kafka2
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: "kafka2"
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'PLAINTEXT:PLAINTEXT'
      KAFKA_BROKER_ID: 1
      KAFKA_NUM_PARTITIONS: 3
      KAFKA_DEFAULT_REPLICATION_FACTOR: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENERS: "PLAINTEXT://kafka2:9092"
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:9092'
      ALLOW_PLAINTEXT_LISTENER: "yes"
    volumes:
      - ./kafka/broker2/logs:/opt/kafka/logs
      - /var/run/docker.sock:/var/run/docker.sock
    restart: always
    depends_on:
      - zookeeper
  

  kafka3:
    image: 'wurstmeister/kafka:2.13-2.7.0'
    container_name: kafka3
    hostname: kafka3
    ports:
      - "9093:9092"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: "kafka3"
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'PLAINTEXT:PLAINTEXT'
      KAFKA_BROKER_ID: 2
      KAFKA_NUM_PARTITIONS: 3
      KAFKA_DEFAULT_REPLICATION_FACTOR: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENERS: "PLAINTEXT://kafka3:9092"
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:9093'
      ALLOW_PLAINTEXT_LISTENER: "yes"
    volumes:
      - ./kafka/broker3/logs:/opt/kafka/logs
      - /var/run/docker.sock:/var/run/docker.sock
    restart: always
    depends_on:
      - zookeeper

3.运行kafka集群

docker-compose up -d

二.3台服务器创建kafka集群

1.准备工作

三台服务器：192.168.40.174 192.168.40.175 192.168.40.176

修改主机名称：

vim /etc/hosts
192.168.40.174 kafka1
192.168.40.175 kafka2
192.168.40.176 kafka3

2.编写docker-compose.yml文件

192.168.40.174 机器

version: '3.7'
services:
  zookeeper:
    image: 'bitnami/zookeeper:3.7.0'
    container_name: 'docker_zookeeper'
    network_mode: host
    restart: always
    environment:
      ZOO_SERVER_ID: 1
      ZOO_PORT_NUMBER: 2181
      ALLOW_ANONYMOUS_LOGIN: 'true'
      ZOO_LISTEN_ALLIPS_ENABLED: 'true'
      ZOO_SERVERS: "kafka1:2888:3888,kafka2:2888:3888,kafka3:2888:3888"
    volumes:
      - ./zookeeper:/bitnami/zookeeper
      - /etc/localtime:/etc/localtime:ro
      - /etc/hosts:/etc/hosts:ro
    user: root

  kafka:
    image: 'wurstmeister/kafka:2.13-2.7.0'
    container_name: 'docker_kafka'
    network_mode: host
    restart: always
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_PORT: 9090
      KAFKA_ADVERTISED_HOST_NAME: "kafka1"
      ALLOW_PLAINTEXT_LISTENER: "yes"
      KAFKA_ZOOKEEPER_CONNECT: "kafka1:2181,kafka2:2181,kafka3:2181"
      KAFKA_LISTENERS: "PLAINTEXT://kafka1:9090"
      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka1:9090"
      KAFKA_SECURITY_INTER_BROKER_PROTOCOL: "PLAINTEXT"
      KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=127.0.0.1 -Dcom.sun.management.jmxremote.rmi.port=19095"
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
      KAFKA_DEFAULT_REPLICATION_FACTOR: 3
      KAFKA_NUM_PARTITIONS: 1
      JMX_PORT: 19095
    volumes:
      - ./kafka/kafa_data:/kafka
      - /etc/localtime:/etc/localtime:ro
      - /etc/hosts:/etc/hosts:ro

192.168.40.175 机器

version: '3.7'
services:
  zookeeper:
    image: 'bitnami/zookeeper:3.7.0'
    container_name: 'docker_zookeeper'
    network_mode: host
    restart: always
    environment:
      ZOO_SERVER_ID: 2
      ZOO_PORT_NUMBER: 2181
      ALLOW_ANONYMOUS_LOGIN: 'true'
      ZOO_LISTEN_ALLIPS_ENABLED: 'true'
      ZOO_SERVERS: "kafka1:2888:3888,kafka2:2888:3888,kafka3:2888:3888"
    volumes:
      - ./zookeeper:/bitnami/zookeeper
      - /etc/localtime:/etc/localtime:ro
      - /etc/hosts:/etc/hosts:ro
    user: root

  kafka:
    image: 'wurstmeister/kafka:2.13-2.7.0'
    container_name: 'docker_kafka'
    network_mode: host
    restart: always
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_PORT: 9090
      KAFKA_ADVERTISED_HOST_NAME: "kafka2"
      ALLOW_PLAINTEXT_LISTENER: "yes"
      KAFKA_ZOOKEEPER_CONNECT: "kafka1:2181,kafka2:2181,kafka3:2181"
      KAFKA_LISTENERS: "PLAINTEXT://kafka2:9090"
      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka2:9090"
      KAFKA_SECURITY_INTER_BROKER_PROTOCOL: "PLAINTEXT"
      KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=127.0.0.1 -Dcom.sun.management.jmxremote.rmi.port=19095"
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
      KAFKA_DEFAULT_REPLICATION_FACTOR: 3
      KAFKA_NUM_PARTITIONS: 1
      JMX_PORT: 19095
    volumes:
      - ./kafka/kafa_data:/kafka
      - /etc/localtime:/etc/localtime:ro
      - /etc/hosts:/etc/hosts:ro

192.168.40.176 机器

version: '3.7'
services:
  zookeeper:
    image: 'bitnami/zookeeper:3.7.0'
    container_name: 'docker_zookeeper'
    network_mode: host
    restart: always
    environment:
      ZOO_SERVER_ID: 3
      ZOO_PORT_NUMBER: 2181
      ALLOW_ANONYMOUS_LOGIN: 'true'
      ZOO_LISTEN_ALLIPS_ENABLED: 'true'
      ZOO_SERVERS: "-kafka1:2888:3888,kafka2:2888:3888,kafka3:2888:3888"
    volumes:
      - ./zookeeper:/bitnami/zookeeper
      - /etc/localtime:/etc/localtime:ro
      - /etc/hosts:/etc/hosts:ro
    user: root

  kafka:
    image: 'wurstmeister/kafka:2.13-2.7.0'
    container_name: 'docker_kafka'
    network_mode: host
    restart: always
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_PORT: 5302
      KAFKA_ADVERTISED_HOST_NAME: "kafka3"
      ALLOW_PLAINTEXT_LISTENER: "yes"
      KAFKA_ZOOKEEPER_CONNECT: "kafka1:2181,kafka2:2181,kafka3:2181"
      KAFKA_LISTENERS: "PLAINTEXT://kafka3:9090"
      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka3:9090"
      KAFKA_SECURITY_INTER_BROKER_PROTOCOL: "PLAINTEXT"
      KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=127.0.0.1 -Dcom.sun.management.jmxremote.rmi.port=19095"
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
      KAFKA_DEFAULT_REPLICATION_FACTOR: 3
      KAFKA_NUM_PARTITIONS: 1
      JMX_PORT: 19095
    volumes:
      - ./kafka/kafa_data:/kafka
      - /etc/localtime:/etc/localtime:ro
      - /etc/hosts:/etc/hosts:ro

3.创建与赋权zookeeper文件夹

如果不改变权限，会提示权限不够的报错，kafka不能正常运行，具体原因暂未解决，有知道原因的可以留言告诉作者。

mkdir zookeeper
chmod 777 -R zookeeper

4.运行kafka集群

三台机器

docker-compose up -d

三.单台服务器部署kafka集群

version: '3.8'
services:
  videocloud-zookeeper:
    container_name: videocloud-zookeeper
    hostname: videocloud-zookeeper
    image: 'bitnami/zookeeper:3.7.0'
    restart: always
    ports:
      - 2181:2181
    user: root
    environment:
      - ALLOW_ANONYMOUS_LOGIN=yes


  videocloud-kafka1:
    image: 'wurstmeister/kafka:2.13-2.7.0'
    container_name: videocloud-kafka1
    hostname: videocloud-kafka1
    ports:
      - "5301:9092"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: "videocloud-kafka1"
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'PLAINTEXT:PLAINTEXT'
      KAFKA_BROKER_ID: 0
      KAFKA_NUM_PARTITIONS: 3
      KAFKA_DEFAULT_REPLICATION_FACTOR: 2
      KAFKA_ZOOKEEPER_CONNECT: videocloud-zookeeper:2181
      KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:9092"
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://192.168.40.174:5301'
      ALLOW_PLAINTEXT_LISTENER: "yes"
    volumes:
      - ./kafka/broker1/logs:/opt/kafka/logs
      - /var/run/docker.sock:/var/run/docker.sock
    restart: always
    depends_on:
      - videocloud-zookeeper


  videocloud-kafka2:
    image: 'wurstmeister/kafka:2.13-2.7.0'
    container_name: videocloud-kafka2
    hostname: videocloud-kafka2
    ports:
      - "5302:9092"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: "videocloud-kafka2"
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'PLAINTEXT:PLAINTEXT'
      KAFKA_BROKER_ID: 1
      KAFKA_NUM_PARTITIONS: 3
      KAFKA_DEFAULT_REPLICATION_FACTOR: 2
      KAFKA_ZOOKEEPER_CONNECT: videocloud-zookeeper:2181
      KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:9092"
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://192.168.40.175:5302'
      ALLOW_PLAINTEXT_LISTENER: "yes"
    volumes:
      - ./kafka/broker2/logs:/opt/kafka/logs
      - /var/run/docker.sock:/var/run/docker.sock
    restart: always
    depends_on:
      - videocloud-zookeeper
  

  videocloud-kafka3:
    image: 'wurstmeister/kafka:2.13-2.7.0'
    container_name: videocloud-kafka3
    hostname: videocloud-kafka3
    ports:
      - "5303:9092"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: "videocloud-kafka3"
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'PLAINTEXT:PLAINTEXT'
      KAFKA_BROKER_ID: 2
      KAFKA_NUM_PARTITIONS: 3
      KAFKA_DEFAULT_REPLICATION_FACTOR: 2
      KAFKA_ZOOKEEPER_CONNECT: videocloud-zookeeper:2181
      KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:9092"
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://192.168.40.176:5303'
      ALLOW_PLAINTEXT_LISTENER: "yes"
    volumes:
      - ./kafka/broker3/logs:/opt/kafka/logs
      - /var/run/docker.sock:/var/run/docker.sock
    restart: always
    depends_on:
      - videocloud-zookeeper