kafka API 使用

1. kafka使用

老版API 所有版本都适用 ，新版API只支持0.9+。而且，新版API对应的集群，必须配置

listeners=PLAINTEXT://xx.xx.xx.xx:9092，xx.xx.xx.xx必须配置，如果是IP地址则可直接使用，如果配置的是用户名，则需要在非本机使用kafKA API，则需要在本机的hosts中添加映射。如果不配置 xx.xx.xx.xx，则消费者无法识别消费

 

2. 老版 API代码

Producer：

package com.ruzz.kafka;

 

import java.text.SimpleDateFormat;

import java.util.Date;

import java.util.Properties;

import kafka.javaapi.producer.Producer;

import kafka.producer.KeyedMessage;

import kafka.producer.ProducerConfig;

 

public class KafkaProducerConfig {

public static void main(String[] args) {

Properties props = new Properties();

props.put("zk.connect", "172.16.3.21:2181,172.16.3.22:2181,172.16.3.23:2181/kafka/test-12345");

// serializer.class为消息的序列化类

props.put("serializer.class", "kafka.serializer.StringEncoder");

// 配置metadata.broker.list, 为了高可用, 最好配两个broker实例

props.put("metadata.broker.list", "172.16.3.22:32194");

// ACK机制, 消息发送需要kafka服务端确认

props.put("request.required.acks", "1");

 

// props.put("num.partitions", "3");

ProducerConfig config = new ProducerConfig(props);

Producer<String, String> producer = new Producer<String, String>(config);

for (int i = 0; i < 10; i++) {

SimpleDateFormat formatter = new SimpleDateFormat("yyyy年MM月dd日 HH:mm:ssSSS");

Date curDate = new Date(System.currentTimeMillis());

String str = formatter.format(curDate);

 

String msg = "test" + i + "=" + str;

String key = i + "";

 

/**

* KeyedMessage<K, V>,K对应Partition Key的类型,V对应消息本身的类型 topic: "test",

* key: "key", message: "message"

*/

producer.send(new KeyedMessage<String, String>("test-tip", key, msg));

}

}

}

Consumer：

package com.ruzz.kafka;

 

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import java.util.Properties;

import kafka.consumer.ConsumerConfig;

import kafka.consumer.ConsumerIterator;

import kafka.consumer.KafkaStream;

import kafka.javaapi.consumer.ConsumerConnector;

import kafka.serializer.StringDecoder;

import kafka.utils.VerifiableProperties;

 

public class ConsumerTest {

private final ConsumerConnector consumer;

 

public ConsumerTest() {

Properties props = new Properties();

// zookeeper 配置

// props.put("zookeeper.connect", "192.168.91.231:2181");

props.put("zookeeper.connect", "172.16.3.21:2181,172.16.3.22:2181,172.16.3.23:2181/kafka/test-12345");

// group 代表一个消费组

props.put("group.id", "jd-group");

 

// zk连接超时

props.put("zookeeper.session.timeout.ms", "4000");

props.put("zookeeper.sync.time.ms", "200");

props.put("auto.commit.interval.ms", "1000");

props.put("auto.offset.reset", "smallest");

// 序列化类

props.put("serializer.class", "kafka.serializer.StringEncoder");

ConsumerConfig config = new ConsumerConfig(props);

consumer = kafka.consumer.Consumer.createJavaConsumerConnector(config);

}

 

public void consume() {

Map<String, Integer> topicCountMap = new HashMap<String, Integer>();

topicCountMap.put("test-tip", new Integer(1));

StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());

StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());

Map<String, List<KafkaStream<String, String>>> consumerMap = consumer.createMessageStreams(topicCountMap,

keyDecoder, valueDecoder);

KafkaStream<String, String> stream = consumerMap.get("test-tip").get(0);

ConsumerIterator<String, String> it = stream.iterator();

while (it.hasNext()) {

System.out.println(it.next().message());

}

}

 

public void stop() {

try {

consumer.shutdown();

} catch (Exception e) {

// TODO: handle exception

e.printStackTrace();

}

}

 

public static void main(String[] args) {

new ConsumerTest().consume();

}

}

 

3. 新版代码

Producer：

package com.ruzz.kafka;

 

import java.util.Properties;

import java.util.concurrent.Future;

import org.apache.kafka.clients.producer.KafkaProducer;

import org.apache.kafka.clients.producer.Producer;

import org.apache.kafka.clients.producer.ProducerRecord;

import org.apache.kafka.clients.producer.RecordMetadata;

 

public class KafkaProducerConfig {

public static void main(String[] args) {

Properties props = new Properties();

props.put("bootstrap.servers", "192.168.187.120:9092,192.168.187.121:9092,192.168.187.122:9092");

props.put("acks", "1");

props.put("retries", 0);

props.put("batch.size", 16384);

props.put("linger.ms", 1);

props.put("buffer.memory", 33554432);

props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");

props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");

 

Producer<String, String> producer = new KafkaProducer<>(props);

for (int i = 0; i < 100; i++) {

Future<RecordMetadata> ll = producer

.send(new ProducerRecord<String, String>("test-12", Integer.toString(i), Integer.toString(i)));

System.out.printf("key = %s, value = %s", Integer.toString(i), Integer.toString(i));

}

 

producer.close();

}

}

Consumer：

package com.ruzz.kafka;

 

import java.util.Arrays;

import java.util.Properties;

import org.apache.kafka.clients.consumer.ConsumerRecord;

import org.apache.kafka.clients.consumer.ConsumerRecords;

import org.apache.kafka.clients.consumer.KafkaConsumer;

 

public class ConsumerTest {

 

public static void main(String[] args) {

Properties props = new Properties();

props.put("bootstrap.servers", "192.168.187.120:9092,192.168.187.121:9092,192.168.187.122:9092");

props.put("group.id", "test");

props.put("enable.auto.commit", "true");

props.put("auto.commit.interval.ms", "1000");

props.put("session.timeout.ms", "30000");

props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");

props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");

KafkaConsumer<String, String> consumer = new KafkaConsumer<String, String>(props);

consumer.subscribe(Arrays.asList("test-12"));

while (true) {

ConsumerRecords<String, String> records = consumer.poll(100);

for (ConsumerRecord<String, String> record : records) {

System.out.printf("offset = %d, key = %s, value = %s", record.offset(), record.key(), record.value());

}

}

}

}

如果有问题，请大伙指出，万分感谢！