【大数据平台】——基于Confluent的Kafka Rest API探索（一）

• Confluent安装

本文主要对（时至2019/12/3）最新的 Confluent-5.3.1的单节点安装做说明，并使用 Confluent-5.3.1集成的Kafka-2.3。

• 下载并解压

官网下载2.confluent-5.3.1-2.12.tar.gz，上传服务器并解压，解压后的目录作为< CONLUENT_HOME >。

• 添加到环境变量

添加CONLUENT_HOME到环境变量，并添加CONFLUENT_HOME/bin 到 PATH。

• 在CONFLUENT_HOME新建数据文件夹

在CONFLUENT_HOME 新建zookeeper-data、kafka-log、confluent-data。

• 修改 CONFLUENT_HOME/etc/kafka/zookeeper.properties

tickTime=2000
dataDir=<CONFLUENT_HOME>/zookeeper-data
clientPort=2181
initLimit=5
syncLimit=2

• 新建myid 文件

在<CONFLUENT_HOME>/zookeeper-data新建文件 myid 并写入 1

• 修改 CONFLUENT_HOME/etc/kafka/server.properties

zookeeper.connect=<host>:2181
broker.id=1
listeners=PLAINTEXT://<host>:9092
log.dirs=<CONFLUENT_HOME>/kafka-log
metric.reporters=io.confluent.metrics.reporter.ConfluentMetricsReporter
confluent.metrics.reporter.bootstrap.servers=<host>:9092
confluent.support.metrics.enable=true
confluent.license.topic.replication.factor=1
confluent.metadata.topic.replication.factor=1
default.replication.factor=1
log.cleaner.enable=false
delete.topic.enable=true

• 修改 CONFLUENT_HOME/etc/confluent-control-center/control-center-production.properties

bootstrap.servers=<host>:9092
confluent.controlcenter.data.dir=<CONFLUENT_HOME>/confluent-data
confluent.license=XyZ
zookeeper.connect=<host>:2181

• 修改 CONFLUENT_HOME/etc/kafka/connect-distributed.properties

consumer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor
producer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor

• 修改 CONFLUENT_HOME/etc/schema-registry/schema-registry.properties

listeners=http://0.0.0.0:8081
kafkastore.bootstrap.servers=PLAINTEXT://<host>:9092

• 修改 CONFLUENT_HOME/etc/kafka-rest/kafka-rest.properties

zookeeper.connect=<host>:2181
bootstrap.servers=PLAINTEXT://<host>:9092
port=8082
consumer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor
producer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor

• 修改 CONFLUENT_HOME/etc/schema-registry/connect-avro-distributed.properties

bootstrap.servers=<host>:9092
key.converter.schema.registry.url=http://<host>:8081
value.converter.schema.registry.url=http://<host>:8081
rest.host.name=0.0.0.0
rest.port=8083

• 依次启动

zookeeper-server-start CONFLUENT_HOME/etc/kafka/zookeeper.properties

kafka-server-start CONFLUENT_HOME/etc/kafka/server.properties

schema-registry-start <CONFLUENT_HOME/etc/schema-registry/schema-registry.properties

control-center-start CONFLUENT_HOME/etc/confluent-control-center/control-center-production.properties

connect-distributed CONFLUENT_HOME/etc/schema-registry/connect-avro-distributed.properties

kafka-rest-start CONFLUENT_HOME/etc/kafka-rest/kafka-rest.properties

这提供一个脚本可以参考一下

# start-all
CONFLUENT_HOME="/opt/confluent-5.3.1"


echo ">>>>>> Starting Confluent Zookeeper ... ..."
$CONFLUENT_HOME/bin/zookeeper-server-start $CONFLUENT_HOME/etc/kafka/zookeeper.properties >> $CONFLUENT_HOME/logs/zookeeper.log 2>&1 &
echo $! > $CONFLUENT_HOME/logs/zookeeper.pid
pid=`cat "$CONFLUENT_HOME/logs/zookeeper.pid"`
echo ">>>>>> Confluent Zookeeper Started at PID:$pid"
echo


echo ">>>>>> Starting Confluent Kafka ... ..."
sleep 10
$CONFLUENT_HOME/bin/kafka-server-start $CONFLUENT_HOME/etc/kafka/server.properties >>$CONFLUENT_HOME/logs/kafka.log 2>&1 &
echo $! > $CONFLUENT_HOME/logs/kafka.pid
pid=`cat "$CONFLUENT_HOME/logs/kafka.pid"`
echo ">>>>>> Confluent Kafka Started at PID:$pid"
echo


echo ">>>>>> Wait for Kafka load log, it will cost about 30s"
sleep 30
echo ">>>>>> Starting Confluent Schema Registry ... ..."
$CONFLUENT_HOME/bin/schema-registry-start $CONFLUENT_HOME/etc/schema-registry/schema-registry.properties >>$CONFLUENT_HOME/logs/schema.log 2>&1 &
echo $! > $CONFLUENT_HOME/logs/schema.pid
pid=`cat "$CONFLUENT_HOME/logs/schema.pid"`
echo ">>>>>> Confluent Schema Registry Started at PID:$pid"
echo


echo ">>>>>> Starting Confluent Control Center ... ..."
sleep 10
#CONTROL_CENTER_OPTS="-Djava.security.auth.login.config=/tmp/confluent/propertyfile.jaas" \
$CONFLUENT_HOME/bin/control-center-start $CONFLUENT_HOME/etc/confluent-control-center/control-center-production.properties >>$CONFLUENT_HOME/logs/control.log 2>&1 &
echo $! > $CONFLUENT_HOME/logs/control.pid
pid=`cat "$CONFLUENT_HOME/logs/control.pid"`
echo ">>>>>> Confluent Control Center Started at PID:$pid"
echo


echo ">>>>>> Starting Confluent Kafka Connector ... ..."
sleep 5
$CONFLUENT_HOME/bin/connect-distributed $CONFLUENT_HOME/etc/schema-registry/connect-avro-distributed.properties >>$CONFLUENT_HOME/logs/connect.log 2>&1 &
echo $! > $CONFLUENT_HOME/logs/connect.pid
pid=`cat "$CONFLUENT_HOME/logs/connect.pid"`
echo ">>>>>> Confluent Kafka Connector Started at PID:$pid"
echo


echo ">>>>>> Starting Confluent Kafka Rest Proxy ... ..."
sleep 5
$CONFLUENT_HOME/bin/kafka-rest-start $CONFLUENT_HOME/etc/kafka-rest/kafka-rest.properties >>$CONFLUENT_HOME/logs/kafka-rest.log 2>&1 &
echo $! > $CONFLUENT_HOME/logs/kafka-rest.pid
pid=`cat "$CONFLUENT_HOME/logs/kafka-rest.pid"`
echo ">>>>>> Confluent Kafka Rest Proxy Connector Started at PID:$pid"
echo

echo ">>>>>> All Done !!!"

# stop-all
pid=`cat "$CONFLUENT_HOME/logs/kafka-rest.pid"`
kill -9 $pid
pid=`cat "$CONFLUENT_HOME/logs/connect.pid"`
kill -9 $pid
pid=`cat "$CONFLUENT_HOME/logs/control.pid"`
kill -9 $pid
pid=`cat "$CONFLUENT_HOME/logs/schema.pid"`
kill -9 $pid
pid=`cat "$CONFLUENT_HOME/logs/kafka.pid"`
kill -9 $pid
pid=`cat "$CONFLUENT_HOME/logs/zookeeper.pid"`
kill -9 $pid

 

• 进入172.16.32.41:9021/clusters查看监控界面