Docker Kafka集群安装

coding4all

已于 2023-02-16 18:08:22 修改

阅读量194

点赞数

分类专栏：大数据运维文章标签： kafka docker

于 2023-02-16 16:30:41 首次发布

本文链接：https://blog.youkuaiyun.com/q1298252589/article/details/129064653

版权

运维同时被 2 个专栏收录

11 篇文章

订阅专栏

大数据

3 篇文章

订阅专栏

机器信息

IP	配置
10.192.192.8	2C4G
10.192.192.9	2C4G
10.192.192.10	2C4G

拉取镜像

docker pull bitnami/kafka:3.3.1

配置docker-compose

yum install -y docker-compose
mkdir /opt/kafka/

# 10.192.192.8执行
cat >/opt/kafka/docker-compose.yaml<<EOF
version: "3.3"
services:
  kafka1:
    container_name: kafka
    image: 'bitnami/kafka:3.3.1'
    user: root
    ports:
      - '9092:9092'
      - '9093:9093'
    environment:
      - KAFKA_ENABLE_KRAFT=yes
      - KAFKA_CFG_PROCESS_ROLES=broker,controller
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://10.192.192.8:9092
      - KAFKA_BROKER_ID=1
      - KAFKA_KRAFT_CLUSTER_ID=iZWRiSqjZAlYwlKEqHFQWI
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@10.192.192.8:9093,2@10.192.192.9:9093,3@10.192.192.10:9093
      - ALLOW_PLAINTEXT_LISTENER=yes
    volumes:
      - /opt/kafka:/bitnami/kafka:rw
    network_mode: host
EOF

# 10.192.192.9执行
cat >/opt/kafka/docker-compose.yaml<<EOF
version: "3.3"
services:
  kafka1:
    container_name: kafka
    image: 'bitnami/kafka:3.3.1'
    user: root
    ports:
      - '9092:9092'
      - '9093:9093'
    environment:
      - KAFKA_ENABLE_KRAFT=yes
      - KAFKA_CFG_PROCESS_ROLES=broker,controller
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://10.192.192.9:9092
      - KAFKA_BROKER_ID=2
      - KAFKA_KRAFT_CLUSTER_ID=iZWRiSqjZAlYwlKEqHFQWI
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@10.192.192.8:9093,2@10.192.192.9:9093,3@10.192.192.10:9093
      - ALLOW_PLAINTEXT_LISTENER=yes
    volumes:
      - /opt/kafka:/bitnami/kafka:rw
    network_mode: host
EOF

# 10.192.192.10执行
cat >/opt/kafka/docker-compose.yaml<<EOF
version: "3.6"
services:
  kafka1:
    container_name: kafka
    image: 'bitnami/kafka:3.3.1'
    user: root
    ports:
      - '9092:9092'
      - '9093:9093'
    environment:
      - KAFKA_ENABLE_KRAFT=yes
      - KAFKA_CFG_PROCESS_ROLES=broker,controller
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://10.192.192.10:9092
      - KAFKA_BROKER_ID=3
      - KAFKA_KRAFT_CLUSTER_ID=iZWRiSqjZAlYwlKEqHFQWI
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@10.192.192.8:9093,2@10.192.192.9:9093,3@10.192.192.10:9093
      - ALLOW_PLAINTEXT_LISTENER=yes
    volumes:
      - /opt/kafka:/bitnami/kafka:rw
    network_mode: host
EOF

启动Kafka集群

# 三台机器依次执行
cd /opt/kafka/ && docker-compose up -d

验证集群

# 任意机器执行
docker exec -it kafka /bin/bash
# 创建topic
kafka-topics.sh --create --topic galaxy-future --partitions 5 --replication-factor 3 --bootstrap-server 10.192.192.8:9092,10.192.192.9:9092,10.192.192.10:9092
# 查看topic
kafka-topics.sh --describe --topic galaxy-future --bootstrap-server 10.192.192.8:9092,10.192.192.9:9092,10.192.192.10:9092
# 测试生产消息（10.192.192.8执行）
kafka-console-producer.sh --broker-list 10.192.192.8:9092,10.192.192.9:9092,10.192.192.10:9092 --topic galaxy-future
# 测试消费消息（10.192.192.9/10执行）
kafka-console-consumer.sh --bootstrap-server 10.192.192.8:9092,10.192.192.9:9092,10.192.192.10:9092 --topic galaxy-future
# 删除topic
kafka-topics.sh --delete --topic galaxy-future --bootstrap-server 10.192.192.8:9092,10.192.192.9:9092,10.192.192.10:9092