kafka集群部署

安装

官网地址：
http://kafka.apache.org/downloads
https://www.oracle.com/java/technologies/javase/javase-jdk8-downloads.html

下载：举例

【JDK】
Binary downloads:
   jdk-8u251-linux-x64.tar.gz
下载后解压即可，配置环境变量即可。

【KAFKA】
Binary downloads:
   Scala 2.11  - kafka_2.11-2.2.0.tgz (asc, sha512)
下载后解压即可，master配置完毕（多 server.properties），slave节点直接scp同步即可。

配置

环境变量

# vim ~/.bashrc
#jdk#
export JAVA_HOME=/home/alpha/java
export JRE_HOME=$JAVA_HOME/jre
export CLASS_PATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export PATH=$JAVA_HOME/bin:$PATH

#kafka#
export KAFKA_HOME=/home/alpha/kafka
export LOG_DIR=/data01/alpha/kafka/logs

# source ~/.bashrc

配置consumer

# vim consumer.properties

# Zookeeper connection string
# comma separated host:port pairs, each corresponding to a zk
# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002"
zookeeper.connect=172.17.1.6:3181,172.17.1.7:3181,172.17.1.8:3181

# timeout in ms for connecting to zookeeper
zookeeper.connection.timeout.ms=1000000

#consumer group id
group.id=test-consumer-group

num.consumer.fetchers=3
fetch.min.bytes=1
fetch.wait.max.ms=1000

#consumer timeout
#consumer.timeout.ms=5000
consumer.timeout.ms=-1
zookeeper.connection.timeout.ms=60000
zookeeper.session.timeout.ms=1200000
rebalance.backoff.ms=3000
rebalance.max.retries=10
auto.create.topics.enable=true

配置producer

# vim producer.properties

############################# Producer Basics #############################

# list of brokers used for bootstrapping knowledge about the rest of the cluster
# format: host1:port1,host2:port2 ...
bootstrap.servers=172.17.1.6:9092,172.17.1.7:9092,172.17.1.8:9092

# specify the compression codec for all data generated: none, gzip, snappy, lz4
compression.type=none

# name of the partitioner class for partitioning events; default partition spreads data randomly
#partitioner.class=

# the maximum amount of time the client will wait for the response of a request
#request.timeout.ms=

# how long `KafkaProducer.send` and `KafkaProducer.partitionsFor` will block for
#max.block.ms=

# the producer will wait for up to the given delay to allow other records to be sent so that the sends can be batched together
#linger.ms=

# the maximum size of a request in bytes
#max.request.size=

# the default batch size in bytes when batching multiple records sent to a partition
batch.size=616448

# the total bytes of memory the producer can use to buffer records waiting to be sent to the server
buffer.memory=1024M

#auto create topics
auto.create.topics.enable=true

配置server

# vim server6.properties

############################# Server Basics #############################
# The id of the broker. This must be set to a unique integer for each broker.
broker.id=0

# Switch to enable topic deletion or not, default value is false
delete.topic.enable=true

############################# Socket Server Settings #############################
# The address the socket server listens on. It will get the value returned from
# java.net.InetAddress.getCanonicalHostName() if not configured.
#   FORMAT:
#     listeners = security_protocol://host_name:port
#   EXAMPLE:
#     listeners = PLAINTEXT://your.host.name:9092
listeners=PLAINTEXT://172.17.1.6:9092

# Hostname and port the broker will advertise to producers and consumers. If not set,
# it uses the value for "listeners" if configured.  Otherwise, it will use the value
# returned from java.net.InetAddress.getCanonicalHostName().
#advertised.listeners=PLAINTEXT://your.host.name:9092

# The number of threads handling network requests
num.network.threads=6

# The number of threads doing disk I/O
num.io.threads=10

# The send buffer (SO_SNDBUF) used by the socket server
socket.send.buffer.bytes=1048576

# The receive buffer (SO_RCVBUF) used by the socket server
socket.receive.buffer.bytes=1048576

# The maximum size of a request that the socket server will accept (protection against OOM)过大会导致缓冲区溢出，nmap扫描时出现OOM故障
socket.request.max.bytes = 104857600


############################# Log Basics #############################
# A comma seperated list of directories under which to store log files
log.dirs=/data01/alpha/kafka/kafka-logs,/data02/alpha/kafka/kafka-logs

# The default number of log partitions per topic. More partitions allow greater
# parallelism for consumption, but this will also result in more files across
# the brokers.
num.partitions=1

# default replication factors for automatically created topics
default.replication.factor=2

# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
# This value is recommended to be increased for installations with data dirs located in RAID array.
num.recovery.threads.per.data.dir=2

############################# Log Flush Policy #############################
# Messages are immediately written to the filesystem but by default we only fsync() to sync
# the OS cache lazily. The following configurations control the flush of data to disk.
# There are a few important trade-offs here:
#    1. Durability: Unflushed data may be lost if you are not using replication.
#    2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.
#    3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks.
# The settings below allow one to configure the flush policy to flush data after a period of time or
# every N messages (or both). This can be done globally and overridden on a per-topic basis.

# The number of messages to accept before forcing a flush of data to disk
#log.flush.interval.messages=10000

# The maximum amount of time a message can sit in a log before we force a flush
#log.flush.interval.ms=1000

############################# Log Retention Policy #############################
# The following configurations control the disposal of log segments. The policy can
# be set to delete segments after a period of time, or after a given size has accumulated.
# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
# from the end of the log.

# The minimum age of a log file to be eligible for deletion
#log.retention.hours=1
log.retention.minutes=20

# The policy for clean logs.
log.cleanup.policy=delete

# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining
# segments don't drop below log.retention.bytes.
#log.retention.bytes=32212254720
log.retention.bytes=-1

# The maximum size of a log segment file. When this size is reached a new log segment will be created.
log.segment.bytes=1073741824

# Time in hours to roll a new segment file. Default value is 7*24
# This setting will force Kafka to roll a new log segment even if the log.segment.bytes size has not been reached.
#log.roll.hours=168

# The interval at which log segments are checked to see if they can be deleted according
# to the retention policies
log.retention.check.interval.ms=300000

############################# Replica #############################
num.replica.fetchers=4
# the number of byes of messages to attempt to fetch
# default value is 1024*1024(1M)
replica.fetch.max.bytes=2048000
replica.fetch.wait.max.ms=500
replica.high.watermark.checkpoint.interval.ms=5000
replica.socket.timeout.ms=300000
replica.socket.receive.buffer.bytes=65536
replica.lag.time.max.ms=300000

############################# Controller #############################
controller.socket.timeout.ms=300000
controller.message.queue.size=50
# enable controlled shutdown of the server
controlled.shutdown.enable =false
fetch.purgatory.purge.interval.requests=1000
producer.purgatory.purge.interval.requests=1000

# enable auto creation of topic on the server
auto.create.topics.enable=true

# the maximum number of connections we allow from each ip address
#max.connections.per.ip=99999999

# the maximum size of message that the server can receive
#message.max.bytes=1000012

############################# Zookeeper #############################
# Zookeeper connection string (see zookeeper docs for details).
# This is a comma separated host:port pairs, each corresponding to a zk
# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
# You can also append an optional chroot string to the urls to specify the
# root directory for all kafka znodes.
zookeeper.connect=172.17.1.6:3181,172.17.1.7:3181,172.17.1.8:3181

# Timeout in ms for connecting to zookeeper
zookeeper.connection.timeout.ms=60000
# zookeeper session timeout
zookeeper.session.timeout.ms=120000
zookeeper.sync.time.ms=5000

脚本

【conf.ini】
# vim /home/alpha/config/conf.ini
#USER#
EXECUSER=alpha
#JAVA#
JDKNODES=(172.17.1.{6..8})
#ZOOKEEPER#
ZKNODES=(172.17.1.{6..8})
#KAFKA#
KAFKANODES=(172.17.1.{6..8})

【start-all.sh】
# vim startall.sh
#!/bin/sh
DIR=$(cd `dirname $0`;pwd)
# Loading Configuration
. /home/alpha/config/conf.ini
for Node in ${KAFKANODES[@]};do
        echo "* Start Kafka Node : ${Node}"
    ssh ${Node} "${DIR%/*}/bin/kafka-server-start.sh ${DIR%/*}/config/server${Node##*.}.properties >/dev/null 2>&1 &"
done

【stop-all.sh】
# vim stopall.sh
#!/bin/sh
DIR=$(cd `dirname $0`;pwd)
# Loading Configuration
. /home/alpha/config/conf.ini
for Node in ${KAFKANODES[@]};do
        echo "* Stop Kafka Node : ${Node}"
    ssh ${Node} "bash ${DIR%/*}/bin/kafka-server-stop.sh"
done

【status-all.sh】
# vim statusall.sh
#!/bin/sh
# Loading Configuration
. /home/alpha/config/conf.ini
for Node in ${KAFKANODES[@]};do
    echo -e "* Check Kafka Node Status : ${Node} :\t $(ssh ${Node} "jps|grep Kafka" 2>/dev/null)"
done

生产kafka调优方案

【== kafka-server-start.sh ==】
# 调整堆外内存，默认-Xms1G -Xmx1g
export KAFKA_HEAP_OPTS="-Xms4g -Xmx4g"

【== producer.propertis ==】
# 增加batch到512kb,默认16kb
batch.size=616448
# 增加buffer到1024m，默认32m
buffer.memory=1073741824

【== server.properties ==】
#socket发送缓冲区调优
socket.send.buffer.bytes=1048576   
#接收缓冲区调优
socket.receive.buffer.bytes=1048576

# broker处理消息的最大线程数 
num.network.threads=16 
# broker处理磁盘IO的线程数 
num.io.threads=16
# 日志仅保留1小时 
log.retention.hours=1

#leader 进行复制的线程数。增大这个数值会增加follow的io（默认：1）
num.replica.fetchers=4
# replicas 每次获取数据的最大大小
replica.fetch.max.bytes=2048000    
# replicas 同leader之间的通信的最大等待时间，失败了会重试。
replica.fetch.wait.max.ms=500        
#每个replica检查是否将最高水平进行固化的频率
replica.high.watermark.checkpoint.interval.ms=5000   
#follower与leader之间的socket超时时间（默认：30000）
replica.socket.timeout.ms=300000
#leader复制时间的socket缓存大小
replica.socket.receive.buffer.bytes=65536  
#如果一个follower在有一个时间窗口内没有发送任意fetch请求，leader就会把这个follower从ISR(in-sync replicas)移除，并认为它已挂掉。
replica.lag.time.max.ms=300000

#partition leader与replicas之间通讯时,socket的超时时间（默认：30000）
controller.socket.timeout.ms=300000
#partition leader与replicas数据同步时,消息的队列尺寸（默认：10）
controller.message.queue.size=50
#是否允许控制器关闭broker ,若是设置为true,会关闭所有在这个broker上的leader，并转移到其他broker
controlled.shutdown.enable =false

# 防止oom 的参数  用于request 状态转变为complete后从purgatory中移除。
fetch.purgatory.purge.interval.requests=1000
producer.purgatory.purge.interval.requests=1000

# 默认：6000
zookeeper.connection.timeout.ms=60000
zookeeper.session.timeout.ms=120000
# 默认：2000
zookeeper.sync.time.ms=5000 

【consumer.properties】
# 默认：6000
zookeeper.connection.timeout.ms=60000
zookeeper.session.timeout.ms=120000