kafka重分配分区

基础操作

#检查topic分区情况
sh kafka-topics.sh --zookeeper ${kafkaZk} --describe --topic topicName
#客户端消费
sh kafka-console-consumer.sh --zookeeper ${kafkaZk} --topic topicName  --max-messages 5
#查看topic列表
sh kafka-topics.sh --zookeeper ${kafkaZk} --list
#修改topic分区数
./kafka-topics.sh --alter --zookeeper ${kafkaZk} --topic ltopicName --partitions 16

充分配分区

1、编辑需要重分配的tipicjson文件
vim topics-topicName.json
{
  "topics": [{
     "topic": "topicName"
   }],
   "version": 1
}
2、生成充分区的计划
sh kafka-reassign-partitions.sh --zookeeper ${kafkaZk} \
    --topics-to-move-json-file topics-topicName.json \
    --broker-list "0,1,2,3,4,5,6,7" \
    --generate


3、编辑计划文件
vi reassignment.json 

{"version":1,"partitions":[{"topic":"topicName","partition":5,"replicas":[5,0,1],"log_dirs":["any","any","any"]},{"topic":"topicName","partition":10,"replicas":[2,6,7],"log_dirs":["any","any","any"]},{"topic":"topicName","partition":2,"replicas":[2,5,6],"log_dirs":["any","any","any"]},{"topic":"topicName","partition":13,"replicas":[5,1,2],"log_dirs":["any","any","any"]},{"topic":"topicName","partition":7,"replicas":[7,2,3],"log_dirs":["any","any","any"]},{"topic":"topicName","partition":15,"replicas":[7,3,4],"log_dirs":["any","any","any"]},{"topic":"topicName","partition":4,"replicas":[4,7,0],"log_dirs":["any","any","any"]},{"topic":"topicName","partition":1,"replicas":[1,4,5],"log_dirs":["any","any","any"]},{"topic":"topicName","partition":12,"replicas":[4,0,1],"log_dirs":["any","any","any"]},{"topic":"topicName","partition":9,"replicas":[1,5,6],"log_dirs":["any","any","any"]},{"topic":"topicName","partition":14,"replicas":[6,2,3],"log_dirs":["any","any","any"]},{"topic":"topicName","partition":3,"replicas":[3,6,7],"log_dirs":["any","any","any"]},{"topic":"topicName","partition":6,"replicas":[6,1,2],"log_dirs":["any","any","any"]},{"topic":"topicName","partition":11,"replicas":[3,7,0],"log_dirs":["any","any","any"]},{"topic":"topicName","partition":0,"replicas":[0,3,4],"log_dirs":["any","any","any"]},{"topic":"topicName","partition":8,"replicas":[0,4,5],"log_dirs":["any","any","any"]}]}

4、开始执行重分配
sh kafka-reassign-partitions.sh \
    --zookeeper ${kafkaZk} \
    --reassignment-json-file reassignment.json \
    --execute

5、查看重分配执行情况
sh kafka-reassign-partitions.sh  \
    --zookeeper ${kafkaZk} \
    --reassignment-json-file reassignment.json \
    --verify