kafka Java编程

最新推荐文章于 2021-02-27 23:24:30 发布

转载最新推荐文章于 2021-02-27 23:24:30 发布 · 426 阅读

kafka 专栏收录该内容

3 篇文章

订阅专栏

本文介绍了一个使用 Java 实现的 Kafka 消费者与生产者的示例程序，展示了如何配置并运行消费者来消费指定主题的消息，以及如何配置并运行生产者向指定主题发送消息。

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import kafka.consumer.Consumer;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import kafka.message.MessageAndMetadata;

public class ConsumerDemo {
private static final String topic = "mysons";
private static final Integer threads = 1;

public static void main(String[] args) {

Properties props = new Properties();
props.put("zookeeper.connect", "weekend01:2181,weekend02:2181,weekend03:2181");

props.put("group.id", "1111");

//设置kafka消费偏移量，不设置就是从上次结束的地方开始读，smallest是这个topic的最小偏移量

props.put("auto.offset.reset", "smallest");

ConsumerConfig config = new ConsumerConfig(props);
ConsumerConnector consumer =Consumer.createJavaConsumerConnector(config);

Map<String, Integer> topicCountMap = new HashMap<String, Integer>();

//topicCountMap.put(topic, thread);thread是线程数，启动多少个线程去消费这个topic

topicCountMap.put(topic, 1);
topicCountMap.put("mygirls", 1);
topicCountMap.put("myboys", 1);

Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer.createMessageStreams(topicCountMap);

//List<KafkaStream<byte[], byte[]>> streams 包含数据和元数据

List<KafkaStream<byte[], byte[]>> streams = consumerMap.get("mygirls");

for(final KafkaStream<byte[], byte[]> kafkaStream : streams){
new Thread(new Runnable() {
@Override
public void run() {
for(MessageAndMetadata<byte[], byte[]> mm : kafkaStream){
String msg = new String(mm.message());
System.out.println(msg);
}
}

}).start();

}
}

}

import java.util.Properties;

import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;

public class ProducerDemo {
public static void main(String[] args) throws Exception {
Properties props = new Properties();
props.put("zk.connect", "weekend01:2181,weekend02:2181,weekend03:2181");
props.put("metadata.broker.list","weekend01:9092,weekend02:9092,weekend03:9092");
props.put("serializer.class", "kafka.serializer.StringEncoder");
ProducerConfig config = new ProducerConfig(props);
Producer<String, String> producer = new Producer<String, String>(config);

// 发送业务消息
// 读取文件读取内存数据库读socket端口
for (int i = 1; i <= 100; i++) {
Thread.sleep(500);
producer.send(new KeyedMessage<String, String>("wordcount",
"i said i love you baby for" + i + "times,will you have a nice day with me tomorrow"));
}

}
}