kafka：python获取kafka的值

需求：获取通过python查看kafka中的值

#!/user/local/python2.6.6/bin/python
# -*- coding: utf-8 -*-
# __project__ = src
# __author__ = kassien@163.com
# __date__ = 2016-09-21 
# __time__ = 12:49

#kafka的节点
kafka_list = ["slave01:9092", "slave02:9092", "slave03:9092"]

from kafka import KafkaConsumer
from kafka import TopicPartition
import sys
import time

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print('Usage: python %s <topic_name>' % sys.argv[0])
        print('       e.g: python %s test_data' % sys.argv[0])
        sys.exit(1)
    topic_name = sys.argv[1]
    with open(topic_name, "w") as wfh:
        client_id = "zcx_%s" % time.time()
        consumer = KafkaConsumer(topic_name, bootstrap_servers=kafka_list, client_id=client_id, group_id=client_id, auto_offset_reset="earliest")

        #for i in range(0,12):
        #    consumer.seek_to_beginning(TopicPartition(topic_name, i))
        #consumer.seek_to_beginning(tp)
        for msg in consumer:
            data = msg.value+"\n"
            wfh.write(data)
            print "partition=%02d, offset=%s" % (msg.partition, msg.offset)

消费出来的日志

kafka中的topicName和partition

北京小辉微信公众号

大数据资料分享请关注