pykafka is ok, kafkapython is not ok

最新推荐文章于 2025-10-10 11:24:20 发布

原创最新推荐文章于 2025-10-10 11:24:20 发布 · 231 阅读

0 ·

CC 4.0 BY-SA版权

本文介绍如何使用PyKafka库连接到Kafka服务器，列出所有topics，并从特定topic中消费数据。代码示例展示了如何设置客户端，获取topic信息，以及如何创建消费者并消费消息。

from pykafka import KafkaClient
from pykafka.utils import serialize_utf8, deserialize_utf8

# 设置客户端的连接信息
client = KafkaClient(hosts="b-1.a205588-cloud-ens-msk.9t99m4.c5.kafka.us-east-1.amazonaws.com:9092")
# 打印所有的topic
print(client.topics)

# 设置循环次数

for n in client.brokers:
    # 取host
    host = client.brokers[n].host
    # 取端口号
    port = client.brokers[n].port
    # 取broker.id
    id = client.brokers[n].id
    # 打印所有信息
    print("host=%s | port=%s | broker.id=%s " % (host, port, id))

topic = client.topics['Equity-All-Topic']
topic_key_list = client.topics.keys()
print(topic_key_list)
partitions = topic.partitions
consumer = topic.get_simple_consumer(
    # 设置消费者组，没有可不写
    consumer_group="simple_consumer_group",
    partitions=[partitions[2]],
    # 设置从头开始消费
    reset_offset_on_start=True
)

for message in consumer:
    if message is not None:
        #打印消费信息
        print(message.offset)
        print ("key:")
        print(message.partition_key.decode('utf-8','ignore'))
        print("value:", message.value)
    break

#consumer = topic.get_simple_consumer(b"simple_consumer_group", partitions=[partitions[2]])