sparkstreaming之kafka--KafkaConsumer

最新推荐文章于 2025-03-31 12:55:45 发布

羽翼天宇

最新推荐文章于 2025-03-31 12:55:45 发布

阅读量568

点赞数

CC 4.0 BY-SA版权

文章标签： kakfa

本文链接：https://blog.youkuaiyun.com/qq_36764089/article/details/79333638

spark streaming实时流通用平台专栏收录该内容

20 篇文章

订阅专栏

本文介绍了一个简单的Kafka消费者实现方式，通过Java代码演示了如何创建Kafka消费者，并订阅特定的主题来接收消息。代码中详细展示了从配置属性到迭代消费消息的过程。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

package com.yys.spark.kafka;

import kafka.consumer.Consumer;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;

/**
* Kafka消费者
*/
public class KafkaConsumer extends Thread{

    private String topic;

    public KafkaConsumer(String topic) {
        this.topic = topic;
    }

    private ConsumerConnector createConnector(){
        Properties properties = new Properties();
        properties.put("zookeeper.connect", KafkaProperties.ZK);
        properties.put("group.id",KafkaProperties.GROUP_ID);
        return Consumer.createJavaConsumerConnector(new ConsumerConfig(properties));
    }

    @Override
    public void run() {
        ConsumerConnector consumer = createConnector();

        Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
        topicCountMap.put(topic, 1);
//        topicCountMap.put(topic2, 1);
//        topicCountMap.put(topic3, 1);

        // String: topic
        // List<KafkaStream<byte[], byte[]>> 对应的数据流
        Map<String, List<KafkaStream<byte[], byte[]>>> messageStream = consumer.createMessageStreams(topicCountMap);

        KafkaStream<byte[], byte[]> stream = messageStream.get(topic).get(0);   //获取我们每次接收到的暑假

        ConsumerIterator<byte[], byte[]> iterator = stream.iterator();

        while (iterator.hasNext()) {
            String message = new String(iterator.next().message());
            System.out.println("rec: " + message);
        }
    }
}