kafka学习五:kafka之Consumer API

目录

  • 熟练掌握kafka之Consumer客户端
  • 了解kafka Consumer客户端配置
  • 掌握kafka Consumer高级特性

Consumer客户端

Consumer之API

/**
* 自动提交
*/
public static void consumPractic(){
        Properties props = new Properties();
        props.setProperty("bootstrap.servers", "192.163.124.3:9092");
        props.setProperty("group.id", "test");
        //自动提交配置
        props.setProperty("enable.auto.commit", "true");
        props.setProperty("auto.commit.interval.ms", "1000");
        props.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        KafkaConsumer<String,String> consumer = new KafkaConsumer<String, String>(props);
        //消费订阅哪一个topic或者几个topic
        consumer.subscribe(Arrays.asList(TOPIC_NAME));
        while (true) {
            ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(10000));
            for (ConsumerRecord<String, String> record : records)
                System.out.printf("partition = %d,offset = %d, key = %s, value = %s%n", record.partition(),record.offset(), record.key(), record.value());
        }
    }
	/**
     * 手动提交,如果失败则不手动提交再次消费,如果成功则手动进行提交
     */
    public static void commitOffset(){
        Properties props = new Properties();
        props.setProperty("bootstrap.servers", "192.163.124.3:9092");
        props.setProperty("group.id", "test");
        //设置手动提交
        props.setProperty("enable.auto.commit", "false");
        props.setProperty("auto.commit.interval.ms", "1000");
        props.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        KafkaConsumer<String,String> consumer = new KafkaConsumer<String, String>(props);
        //消费订阅哪一个topic或者几个topic
        consumer.subscribe(Arrays.asList(TOPIC_NAME));
        while (true) {
            ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(10000));
            for (ConsumerRecord<String, String> record : records){
               //把数据保存到数据库,成功就成功,失败、、、、、
                //如果提交则回滚,不要提交offset
                System.out.printf("partition = %d,offset = %d, key = %s, value = %s%n", record.partition(),record.offset(), record.key(), record.value());
            }
            //如果成功手动通知offset提交
            consumer.commitAsync();
        }
    }
	/**
     * 手动提交,针对每个partition进行提交,多线程处理每个Partition进行处理,可能会成功或失败,失败	的就不提交了
     */
    public static void commitOffsetWithPartition(){
        Properties props = new Properties();
        props.setProperty("bootstrap.servers", "192.163.124.3:9092");
        props.setProperty("group.id", "test");
        //设置手动提交
        props.setProperty("enable.auto.commit", "false");
        props.setProperty("auto.commit.interval.ms", "1000");
        props.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        KafkaConsumer<String,String> consumer = new KafkaConsumer<String, String>(props);
        //消费订阅哪一个topic或者几个topic
        consumer.subscribe(Arrays.asList(TOPIC_NAME));
        while (true) {
            ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(10000));
            for (TopicPartition partition:records.partitions()){
                List<ConsumerRecord<String, String>> pRecord = records.records(partition);
                for (ConsumerRecord record : pRecord){
                    System.out.printf("partition = %d,offset = %d, key = %s, value = %s%n",
                        record.partition(),record.offset(), record.key(), record.value());
                }
                //针对单独的Partition进行提交offset
                Map<TopicPartition, OffsetAndMetadata> offset = new HashMap<>();
                long lastOffset = pRecord.get(pRecord.size() - 1).offset();
                //服务器上记录的offset是下一次消费的起点,每次从下一个位置查
                offset.put(partition,new OffsetAndMetadata(lastOffset+1));
                consumer.commitSync(offset);
            }

        }
    }
	/**
     * 订阅某个topic下的某个Partition
     */
    public static void commitTopicWithPartition(){
        Properties props = new Properties();
        props.setProperty("bootstrap.servers", "192.163.124.3:9092");
        props.setProperty("group.id", "test");
        //设置手动提交
        props.setProperty("enable.auto.commit", "false");
        props.setProperty("auto.commit.interval.ms", "1000");
        props.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        KafkaConsumer<String,String> consumer = new KafkaConsumer<String, String>(props);

        //kafka_study-topic下有两个Partition 0 1
        TopicPartition p0 = new TopicPartition(TOPIC_NAME,0);
        TopicPartition p1 = new TopicPartition(TOPIC_NAME,1);
        //消费订阅哪一个partittion 这里消费分区0
        consumer.assign(Arrays.asList(p0));

        while (true) {
            ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(10000));
            for (TopicPartition partition:records.partitions()){
                List<ConsumerRecord<String, String>> pRecord = records.records(partition);
                for (ConsumerRecord record : pRecord){
                    System.out.printf("partition = %d,offset = %d, key = %s, value = %s%n",
                            record.partition(),record.offset(), record.key(), record.value());
                }
                //针对单独的Partition进行提交offset
                Map<TopicPartition, OffsetAndMetadata> offset = new HashMap<>();
                long lastOffset = pRecord.get(pRecord.size() - 1).offset();
                //服务器上记录的offset是下一次消费的起点,每次从下一个位置查
                offset.put(partition,new OffsetAndMetadata(lastOffset+1));
                consumer.commitSync(offset);
            }

        }
    }
package com.sun.kafka.consumer;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.errors.WakeupException;

import java.time.Duration;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicBoolean;

public class ConsumerThreadSample {
    public final static String  TOPIC_NAME = "kafka_study-topic";

    /**
     * Consumer不是线程安全的,这种类型是经典模式,每一个线程单独创建一个KafkaConsumer,用于保证线程安全
     * 创建多个Consumer分别消费不同的partition分区的数据
     * @param args
     * @throws InterruptedException
     */
    public static void main(String[] args) throws InterruptedException {
        KafkaConsumerRunner r1 = new KafkaConsumerRunner();
        Thread t1 = new Thread(r1);

        t1.start();

        Thread.sleep(15000);

        r1.shutdown();
    }

    public static class KafkaConsumerRunner implements Runnable{
        private final AtomicBoolean closed = new AtomicBoolean(false);
        private final KafkaConsumer consumer;

        public KafkaConsumerRunner() {
            Properties props = new Properties();
            props.put("bootstrap.servers", "192.163.124.3:9092");
            props.put("group.id", "test");
            props.put("enable.auto.commit", "false");
            props.put("auto.commit.interval.ms", "1000");
            props.put("session.timeout.ms", "30000");
            props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
            props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");

            consumer = new KafkaConsumer<>(props);

            TopicPartition p0 = new TopicPartition(TOPIC_NAME, 0);
            TopicPartition p1 = new TopicPartition(TOPIC_NAME, 1);
            TopicPartition p2 = new TopicPartition(TOPIC_NAME, 2);

            consumer.assign(Arrays.asList(p0,p1,p2));
        }


        public void run() {
            try {
                while(!closed.get()) {
                    //处理消息
                    ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(10000));

                    for (TopicPartition partition : records.partitions()) {
                        List<ConsumerRecord<String, String>> pRecord = records.records(partition);
                        // 处理每个分区的消息
                        for (ConsumerRecord<String, String> record : pRecord) {
                            System.out.printf("patition = %d , offset = %d, key = %s, value = %s%n",
                                    record.partition(),record.offset(), record.key(), record.value());
                        }

                        // 返回去告诉kafka新的offset
                        long lastOffset = pRecord.get(pRecord.size() - 1).offset();
                        // 注意加1
                        consumer.commitSync(Collections.singletonMap(partition, new OffsetAndMetadata(lastOffset + 1)));
                    }

                }
            }catch(WakeupException e) {
                if(!closed.get()) {
                    throw e;
                }
            }finally {
                consumer.close();
            }
        }

        public void shutdown() {
            closed.set(true);
            consumer.wakeup();
        }
    }

}

package com.sun.kafka.consumer;

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;

import java.util.Arrays;
import java.util.Properties;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

public class ConsumerRecordThreadSample {
    public final static String  TOPIC_NAME = "kafka_study-topic";

    public static void main(String[] args) throws InterruptedException {
        String brokerList = "192.163.124.3:9092";
        String groupId = "test";
        int workerNum = 5;

        CunsumerExecutor consumers = new CunsumerExecutor(brokerList, groupId, TOPIC_NAME);
        consumers.execute(workerNum);

        Thread.sleep(1000000);

        consumers.shutdown();

    }

    // Consumer处理
    public static class CunsumerExecutor{
        private final KafkaConsumer<String, String> consumer;
        private ExecutorService executors;

        public CunsumerExecutor(String brokerList, String groupId, String topic) {
            Properties props = new Properties();
            props.put("bootstrap.servers", brokerList);
            props.put("group.id", groupId);
            props.put("enable.auto.commit", "true");
            props.put("auto.commit.interval.ms", "1000");
            props.put("session.timeout.ms", "30000");
            props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
            props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
            consumer = new KafkaConsumer<>(props);
            consumer.subscribe(Arrays.asList(topic));
        }

        public void execute(int workerNum) {
            executors = new ThreadPoolExecutor(workerNum, workerNum, 0L, TimeUnit.MILLISECONDS,
                    new ArrayBlockingQueue<>(1000), new ThreadPoolExecutor.CallerRunsPolicy());

            while (true) {
                ConsumerRecords<String, String> records = consumer.poll(200);
                for (final ConsumerRecord record : records) {
                    executors.submit(new ConsumerRecordWorker(record));
                }
            }
        }

        public void shutdown() {
            if (consumer != null) {
                consumer.close();
            }
            if (executors != null) {
                executors.shutdown();
            }
            try {
                if (!executors.awaitTermination(10, TimeUnit.SECONDS)) {
                    System.out.println("Timeout.... Ignore for this case");
                }
            } catch (InterruptedException ignored) {
                System.out.println("Other thread interrupted this shutdown, ignore for this case.");
                Thread.currentThread().interrupt();
            }
        }


    }

    // 记录处理
    public static class ConsumerRecordWorker implements Runnable {

        private ConsumerRecord<String, String> record;

        public ConsumerRecordWorker(ConsumerRecord record) {
            this.record = record;
        }

        @Override
        public void run() {
            System.out.println("Thread - "+ Thread.currentThread().getName());
            System.err.printf("patition = %d , offset = %d, key = %s, value = %s%n",
                    record.partition(), record.offset(), record.key(), record.value());
        }

    }
}

package com.sun.kafka.consumer;

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;

import java.util.Arrays;
import java.util.Properties;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

public class ConsumerRecordThreadSample {
    public final static String  TOPIC_NAME = "kafka_study-topic";

    /**
     * 创建多个Consumer会导致每个线程中对象过重
     * 一个Consumer监听多个分区,针对每一条数据进行多线程处理
     * @param args
     * @throws InterruptedException
     */
    public static void main(String[] args) throws InterruptedException {
        String brokerList = "192.163.124.3:9092";
        String groupId = "test";
        int workerNum = 5;

        CunsumerExecutor consumers = new CunsumerExecutor(brokerList, groupId, TOPIC_NAME);
        consumers.execute(workerNum);

        Thread.sleep(1000000);

        consumers.shutdown();

    }

    // Consumer处理
    public static class CunsumerExecutor{
        private final KafkaConsumer<String, String> consumer;
        private ExecutorService executors;

        public CunsumerExecutor(String brokerList, String groupId, String topic) {
            Properties props = new Properties();
            props.put("bootstrap.servers", brokerList);
            props.put("group.id", groupId);
            props.put("enable.auto.commit", "true");
            props.put("auto.commit.interval.ms", "1000");
            props.put("session.timeout.ms", "30000");
            props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
            props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
            consumer = new KafkaConsumer<>(props);
            consumer.subscribe(Arrays.asList(topic));
        }

        public void execute(int workerNum) {
            executors = new ThreadPoolExecutor(workerNum, workerNum, 0L, TimeUnit.MILLISECONDS,
                    new ArrayBlockingQueue<>(1000), new ThreadPoolExecutor.CallerRunsPolicy());

            while (true) {
                ConsumerRecords<String, String> records = consumer.poll(200);
                for (final ConsumerRecord record : records) {
                    executors.submit(new ConsumerRecordWorker(record));
                }
            }
        }

        public void shutdown() {
            if (consumer != null) {
                consumer.close();
            }
            if (executors != null) {
                executors.shutdown();
            }
            try {
                if (!executors.awaitTermination(10, TimeUnit.SECONDS)) {
                    System.out.println("Timeout.... Ignore for this case");
                }
            } catch (InterruptedException ignored) {
                System.out.println("Other thread interrupted this shutdown, ignore for this case.");
                Thread.currentThread().interrupt();
            }
        }


    }

    // 记录处理
    public static class ConsumerRecordWorker implements Runnable {

        private ConsumerRecord<String, String> record;

        public ConsumerRecordWorker(ConsumerRecord record) {
            this.record = record;
        }

        @Override
        public void run() {
            System.out.println("Thread - "+ Thread.currentThread().getName());
            System.err.printf("patition = %d , offset = %d, key = %s, value = %s%n",
                    record.partition(), record.offset(), record.key(), record.value());
        }

    }
}

	/**
     * 手动指定offset的起始位置,及手动提交offset
     */
    public static void controlOffset(){
        Properties props = new Properties();
        props.setProperty("bootstrap.servers", "192.163.124.3:9092");
        props.setProperty("group.id", "test");
        //设置手动提交
        props.setProperty("enable.auto.commit", "false");
        props.setProperty("auto.commit.interval.ms", "1000");
        props.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        KafkaConsumer<String,String> consumer = new KafkaConsumer<String, String>(props);

        //kafka_study-topic下有两个Partition 0 1
        TopicPartition p0 = new TopicPartition(TOPIC_NAME,0);
        //消费订阅哪一个partittion 这里消费分区0
        consumer.assign(Arrays.asList(p0));

        while (true) {
            //手动指定offset起始位置
            /**
             * 1.第一次从0消费
             * 2.比如第一次消费了100条,offset置为101并且存入redis
             * 3.每次poll之前,从redis中获取最新offset位置
             * 4.每次从这个位置开始消费
             */
            consumer.seek(p0,200);
            ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(10000));
            for (TopicPartition partition:records.partitions()){
                List<ConsumerRecord<String, String>> pRecord = records.records(partition);
                for (ConsumerRecord record : pRecord){
                    System.out.printf("partition = %d,offset = %d, key = %s, value = %s%n",
                            record.partition(),record.offset(), record.key(), record.value());
                }
                //针对单独的Partition进行提交offset
                Map<TopicPartition, OffsetAndMetadata> offset = new HashMap<>();
                long lastOffset = pRecord.get(pRecord.size() - 1).offset();
                //服务器上记录的offset是下一次消费的起点,每次从下一个位置查
                offset.put(partition,new OffsetAndMetadata(lastOffset+1));
                consumer.commitSync(offset);
            }

        }
    }
	/**
     * kafka消费限流
     */
    public static void controllPause(){
        Properties props = new Properties();
        props.setProperty("bootstrap.servers", "192.163.124.3:9092");
        props.setProperty("group.id", "test");
        //设置手动提交
        props.setProperty("enable.auto.commit", "false");
        props.setProperty("auto.commit.interval.ms", "1000");
        props.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        KafkaConsumer<String,String> consumer = new KafkaConsumer<String, String>(props);
        //kafka_study-topic下有两个Partition 0 1
        TopicPartition p0 = new TopicPartition(TOPIC_NAME,0);
        //消费订阅哪一个topic或者几个topic
        consumer.subscribe(Arrays.asList(TOPIC_NAME));
        long totalNum = 40;
        while (true) {
            ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(10000));
            long num =0;
            for (TopicPartition partition:records.partitions()){
                List<ConsumerRecord<String, String>> pRecord = records.records(partition);
                for (ConsumerRecord record : pRecord){
                    System.out.printf("partition = %d,offset = %d, key = %s, value = %s%n",
                            record.partition(),record.offset(), record.key(), record.value());
                    /**
                     * 1.接受到record信息以后,去令牌桶中拿去令牌
                     * 2.若获取到令牌,则继续业务处理
                     * 3.若获取不到令牌,则pause等待令牌
                     * 4.当令牌桶中令牌足够,则将Consumer置为resume状态
                     */
                    num++;
                    if (record.partition()==0){
                        if (num >= totalNum)
                            consumer.pause(Arrays.asList(p0));
                    }
                    if (record.partition()==1){
                        if (num == 50)
                            consumer.resume(Arrays.asList(p0));
                    }
                }
                //针对单独的Partition进行提交offset
                Map<TopicPartition, OffsetAndMetadata> offset = new HashMap<>();
                long lastOffset = pRecord.get(pRecord.size() - 1).offset();
                //服务器上记录的offset是下一次消费的起点,每次从下一个位置查
                offset.put(partition,new OffsetAndMetadata(lastOffset+1));
                consumer.commitSync(offset);
            }

        }
    }

Consumer注意事项

  • 单个分区的消息只能由ConsumerGroup中的某个Consumer消费(在同一个ConsumerGroup中Consumer和Partition可以是一对一或者其他的ConsumerGroup中的consumer依然可以消费这个partition,但是一个Partition不能对应多个Consumer)
  • Consumer从Partition中消费消息是顺序,默认从头开始消费
  • 单个ConsumerGroup会消费所有Partition中的消息
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值