【Kafka】集群部署

Zookeeper集群安装及Zookeeper启动脚本

Kafka安装配置

下载Kafka

解压

tar -zxvf kafka_2.11-2.2.0.tgz -C /opt/modules

重命名

 mv kafka_2.11-2.2.0   kafka_2.11

在kafka目录下创建logs文件夹

root@master:/opt/modules# cd kafka_2.11/
root@master:/opt/modules/kafka_2.11# ll
total 60
drwxr-xr-x  6 root root  4096 3月  10 03:47 ./
drwxr-xr-x 10 root root  4096 6月  12 18:36 ../
drwxr-xr-x  3 root root  4096 3月  10 03:47 bin/
drwxr-xr-x  2 root root  4096 3月  10 03:47 config/
drwxr-xr-x  2 root root  4096 6月  12 18:35 libs/
-rw-r--r--  1 root root 32216 3月  10 03:44 LICENSE
-rw-r--r--  1 root root   336 3月  10 03:44 NOTICE
drwxr-xr-x  2 root root  4096 3月  10 03:47 site-docs/
root@master:/opt/modules/kafka_2.11# mkdir logs
root@master:/opt/modules/kafka_2.11# ls
bin  config  libs  LICENSE  logs  NOTICE  site-docs

修改配置文件

root@master:/opt/modules/kafka_2.11# cd config/
root@master:/opt/modules/kafka_2.11/config# ll
total 76
drwxr-xr-x 2 root root 4096 3月  10 03:47 ./
drwxr-xr-x 7 root root 4096 6月  12 18:38 ../
-rw-r--r-- 1 root root  906 3月  10 03:44 connect-console-sink.properties
-rw-r--r-- 1 root root  909 3月  10 03:44 connect-console-source.properties
-rw-r--r-- 1 root root 5321 3月  10 03:44 connect-distributed.properties
-rw-r--r-- 1 root root  883 3月  10 03:44 connect-file-sink.properties
-rw-r--r-- 1 root root  881 3月  10 03:44 connect-file-source.properties
-rw-r--r-- 1 root root 1111 3月  10 03:44 connect-log4j.properties
-rw-r--r-- 1 root root 2262 3月  10 03:44 connect-standalone.properties
-rw-r--r-- 1 root root 1221 3月  10 03:44 consumer.properties
-rw-r--r-- 1 root root 4727 3月  10 03:44 log4j.properties
-rw-r--r-- 1 root root 1925 3月  10 03:44 producer.properties
-rw-r--r-- 1 root root 6851 3月  10 03:44 server.properties
-rw-r--r-- 1 root root 1032 3月  10 03:44 tools-log4j.properties
-rw-r--r-- 1 root root 1169 3月  10 03:44 trogdor.conf
-rw-r--r-- 1 root root 1023 3月  10 03:44 zookeeper.properties
root@master:/opt/modules/kafka_2.11/config# vim server.properties

修改如下

#broker的全局唯一编号，不能重复
broker.id=0
#开启删除主题
delete.topic.enable=true

开启监听本机9092端口
listeners=PLAINTEXT://:9092

#处理网络请求的线程数量
num.network.threads=3

#用来处理磁盘IO的现成数量
num.io.threads=8
#发送套接字的缓冲区大小
socket.send.buffer.bytes=102400
#接收套接字的缓冲区大小
socket.receive.buffer.bytes=102400
#请求套接字的缓冲区大小
socket.request.max.bytes=104857600
#kafka运行日志存放的路径
log.dirs=/opt/modules/kafka_2.11/logs

#topic在当前broker上的分区个数
num.partitions=1

#用来恢复和清理data下数据的线程数量
num.recovery.threads.per.data.dir=1

#segment文件保留的最长时间，超时将被删除
log.retention.hours=168

#配置连接Zookeeper集群地址
zookeeper.connect=master:2181,slave01:2181,slave02:2181,slave03:2181,slave04:2181,slave05:2181,slave06:2181

配置环境变量

vim ~/.bashrc

添加如下

刷新生效

source ~/.bashrc

分发

scp -r kafka_2.11 root@slave01:/opt/modules/
scp -r kafka_2.11 root@slave02:/opt/modules/
scp -r kafka_2.11 root@slave03:/opt/modules/
scp -r kafka_2.11 root@slave04:/opt/modules/
scp -r kafka_2.11 root@slave05:/opt/modules/
scp -r kafka_2.11 root@slave06:/opt/modules/

分别在slave01…slave06修改server.properties中的broker.id，分别改为1…6，这里只截图slave01,broker.id不可相同

启动Zookeeper

root@master:/opt/server# ./zk.sh start
master
ZooKeeper JMX enabled by default
Using config: /opt/modules/zookeeper-3.4.14/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
slave01
ZooKeeper JMX enabled by default
Using config: /opt/modules/zookeeper-3.4.14/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
slave02
ZooKeeper JMX enabled by default
Using config: /opt/modules/zookeeper-3.4.14/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
slave03
ZooKeeper JMX enabled by default
Using config: /opt/modules/zookeeper-3.4.14/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
slave04
ZooKeeper JMX enabled by default
Using config: /opt/modules/zookeeper-3.4.14/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
slave05
ZooKeeper JMX enabled by default
Using config: /opt/modules/zookeeper-3.4.14/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
slave06
ZooKeeper JMX enabled by default
Using config: /opt/modules/zookeeper-3.4.14/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED

启动Kafka

分别在各个节点/opt/modules/kafka_2.11目录下执行

bin/kafka-server-start.sh config/server.properties &

关闭集群

bin/kafka-server-stop.sh config/server.properties

参考：
https://blog.youkuaiyun.com/qq_28652401/article/details/83509796