kafka 指定partition生产，消费

kafka指定partition生产消费

在实际的业务中，特别是涉及到指定任务是否结束，任务对应消息是否消费完毕时，单纯指定topic消费，由kafka自动分配partition已经无法满足我们的实际需求了，这时我们就需要指定partition进行生产与消费。闲话少说，下面我们通过代码来详细描述生产者与消费者的配置。
producer代码
注意：producer代码中我们需要两个类，一个时指定partitioner的类，一个为真正的producer类.
指定partitioner的类

package com.wshare.common;

import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;

import java.util.Properties;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingDeque;

/**
 * Created by flyfy on 2017/11/15.
 */
public class KafkaPartitionerProducerUtil {
    private final Producer<String, String> producer;
    private final String topic = "test";
    private final String kafkaQueueHost = "1.1.1.1";
    private final int kafkaQueuePort = 9092;
    private Properties props = new Properties();

    private BlockingQueue<KeyedMessage<String, String>> blockingQueue = new LinkedBlockingDeque<KeyedMessage<String, String>>();

    public KafkaPartitionerProducerUtil() {
        props.put("serializer.class", "kafka.serializer.StringEncoder");
        props.put("metadata.broker.list", kafkaQueueHost + ":" + kafkaQueuePort);
        props.put("partitioner.class","com.wshare.common.CidPartitioner");
        this.producer = new Producer<String, String>(new ProducerConfig(props));
    }

    public void send(String key,String msg) {
        producer.send(new KeyedMessage<String, String>(topic,key,msg));
    }
}

producer类

package com.wshare.common;

import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;

import java.util.Properties;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingDeque;

/**
 * Created by flyfy on 2017/11/15.
 */
public class KafkaPartitionerProducerUtil {
    private final Producer<String, String> producer;
    private final String topic = "test";
    private final String kafkaQueueHost = "1.1.1.1";
    private final int kafkaQueuePort = 9092;
    private Properties props = new Properties();

    private BlockingQueue<KeyedMessage<String, String>> blockingQueue = new LinkedBlockingDeque<KeyedMessage<String, String>>();

    public KafkaPartitionerProducerUtil() {
        props.put("serializer.class", "kafka.serializer.StringEncoder");
        props.put("metadata.broker.list", kafkaQueueHost + ":" + kafkaQueuePort);
        props.put("partitioner.class","com.wshare.common.CidPartitioner");
        this.producer = new Producer<String, String>(new ProducerConfig(props));
    }

    public void send(String key,String msg) {
        producer.send(new KeyedMessage<String, String>(topic,key,msg));
    }
}

注意props.put(“partitioner.class”,”com.wshare.common.CidPartitioner”)即指定我们定义好的partitioner类

consumer代码

package com.lyf.scandatatofile.Utils;

import kafka.cluster.Partition;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.TopicPartition;

import java.io.File;
import java.io.FileWriter;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.BlockingQueue;

/**
 * Created by flyfy on 2017/10/7.
 */
public class KafkaConsumerUtil {
    final String topic = "test";
    final Properties props = new Properties();
    public void execute(String groupid, BlockingQueue<String> bq) {
        props.put("bootstrap.servers", "1.1.1.1:9092");
        props.put("group.id", "test");
        props.put("auto.offset.reset", "earliest");
        props.put("enable.auto.commit", "true");
        props.put("auto.commit.interval.ms", "1000");
        props.put("session.timeout.ms", "30000");
        props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
        //consumer.subscribe(Arrays.asList("IP_REGISTER_yangkai"));
        consumer.assign(Arrays.asList(new TopicPartition("kafka.wshare.match_meta_data.topic",0)));
        while (true) {
            ConsumerRecords<String, String> records = consumer.poll(100);
            for (ConsumerRecord<String, String> record : records)
                 System.out.println("offset"+record.offset()+"key"
                        +record.key()+"value"+record.value());
        }
    }
}

消费者有一处需要特别注意consumer.subscribe(Arrays.asList(“IP_REGISTER_yangkai”))与consumer.assign(Arrays.asList(new TopicPartition(“kafka.wshare.match_meta_data.topic”,0)))对kafka而言是互斥的，二者只能指定其一，前者指定消费模式为指定topic,由kafka指定partition分配策略，后者指定消费模式为指定partition,由用户自定义partition分配策略。