python-kafka pykafka

最新推荐文章于 2025-09-29 09:38:58 发布

原创最新推荐文章于 2025-09-29 09:38:58 发布 · 320 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#python #pykafka

kafka 同时被 2 个专栏收录

4 篇文章

订阅专栏

python

1 篇文章

订阅专栏

该博客展示了如何使用kafka-python和pykafka库来操作Kafka消费者。内容包括设置消费者组、获取分区的最早和最新offset、查看消费者当前的offset，以及消费消息。还展示了如何设置和提交offset。

kafka-python

from kafka import KafkaConsumer,TopicPartition,OffsetAndMetadata
consumer = KafkaConsumer("wd-prd-data-test",
                         group_id="wd_prd_info_group",
                         auto_offset_reset='latest',
                         enable_auto_commit=True,
                         bootstrap_servers=['10.7.128.7:9092','10.7.128.8:9092','10.7.128.9:9092'])


tp = TopicPartition("wd-prd-data-test",0)
pp = [tp]

#partition的最旧（最小）有效offset
ef = consumer.beginning_offsets(pp)
#partition的最新（最大）offset
lf = consumer.end_offsets(pp)
#获取consumer group 对应的当前offset 对应partition消费过的最大offset
consumer.assignment()
cf = consumer.position(tp)

print(ef)
print(cf)
print(lf)

#dd = OffsetAndMetadata(418400,None)
#consumer.commit({tp:dd})

n=0
for message in consumer:
     n=n+1
     print (n,"  %s:%d:%d: key=%s value=%s" % (message.topic, message.partition,
                                         message.offset, message.key,
                                        message.value))

pykafka

from pykafka import KafkaClient
client = KafkaClient(hosts = '10.7.128.7:9092,10.7.128.8:9092,10.7.128.9:9092')
#client = KafkaClient(zookeeper_hosts = 'test1:2181,test5:2181,test6:2181/chroot/finai')

offset=0
topic = client.topics["wd-prd-data-test".encode()]
partitions = topic.partitions
last_offset = topic.latest_available_offsets()
print("最新最大offset {}".format(sorted(last_offset.items(),key=lambda item:item[0])))  # 查看所有分区

earliest_offset = topic.earliest_available_offsets()
print("最小（最旧）可用offset {}".format(sorted(earliest_offset.items(),key=lambda item:item[0])))  # 查看所有分区

"""
consumer = topic.get_balanced_consumer(b"simple_consumer_group",auto_commit_enable=True,
                                     auto_commit_interval_ms=1,
                                     zookeeper_connect = 'test1:2181,test5:2181,test6:2181/chroot/finai'
                                     )  # 选择一个分区进行消费

"""
consumer = topic.get_simple_consumer(b"simple_consumer_group",
                                       auto_commit_enable=True,
                                       auto_commit_interval_ms=1,
                                       auto_offset_reset='latest'   #必须参数，group一次消费时起作用
                                       )  # 选择一个分区进行消费

offset_list = consumer.held_offsets
# 消费者拥有的分区offset的情况,从0开始计数，对应partition已经消费过的最大offset-1
print("当前消费者分区offset情况{}".format(sorted(offset_list.items(),key=lambda item:item[0])))

#consumer.reset_offsets([(partitions[0], offset)])  # 设置offset

#msg = consumer.consume()
#print("消费 :{}".format(msg.value.decode()))
#offset = consumer.held_offsets
#print("当前消费者分区offset情况{}".format(offset)) # 3
n=0
for message in consumer:
    if message is not None:
        n=n+1
        print(n,"  ::  ",message.offset, message.value)