kafka通过timestamp找到offset消费数据

kafka通过timestamp找到offset消费数据

package com.offsetandtimestamp;

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.OffsetAndTimestamp;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;

import java.text.SimpleDateFormat;
import java.util.*;

public class Demo {
    public static void main(String[] args) {
        Properties props = new Properties();
        props.put("bootstrap.server", "node1:9092");
        props.put("group.id", "test");
        props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        //kafka消费者
        KafkaConsumer consumer = new KafkaConsumer<>(props);
        String topic = "kafkatopic";

        try {
            //获取topic的partition信息
            List<PartitionInfo> partitionInfos = consumer.partitionsFor(topic);
            ArrayList<Object> topicpartitions = new ArrayList<>();
            HashMap<Object, Object> timestampToSearch = new HashMap<>();

            SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
            Date now = new Date();

            //当前时刻的时间戳
            long nowTime = now.getTime();

            System.out.println("当前时间: " + df.format(now));
            //计算当前时间戳-24h后的结果
            long fetchDataTime = nowTime - 24 * 60 * 60 * 1000;

            for (PartitionInfo partitionInfo : partitionInfos) {
                //list里面增加
                topicpartitions.add(new TopicPartition(partitionInfo.topic(), partitionInfo.partition()));
                //hashmap里面添加
                timestampToSearch.put(new TopicPartition(partitionInfo.topic(), partitionInfo.partition()), fetchDataTime);

            }
            //给consumer分配任务,手动注册
            consumer.assign(topicpartitions);

            //获取每个partition一个小时之前的偏移量
            Map<TopicPartition, OffsetAndTimestamp> map = consumer.offsetsForTimes(timestampToSearch);

            OffsetAndTimestamp offsetAndTimestamp = null;

            System.out.println("开始设置各分区初始偏移量...");

            //每个partition和时间戳offset
            for (Map.Entry<TopicPartition, OffsetAndTimestamp> entry : map.entrySet()) {

                //如果设置的查询偏移量和时间点大于最大的索引记录时间,那么value就为空
                offsetAndTimestamp = entry.getValue();

                if (offsetAndTimestamp != null) {
                    int partition = entry.getKey().partition();
                    long timestamp = offsetAndTimestamp.timestamp();
                    long offset = offsetAndTimestamp.offset();
                    System.out.println("partition = " + partition +
                            ",time = " + df.format(new Date(timestamp))
                            + ",offset = " + offset);

                    //consumer消费指定partition的offset
                    //设置偏移量
                    consumer.seek(entry.getKey(), offset);

                }
            }
            System.out.println("设置各分区初始偏移量结束...");

            while (true) {
                ConsumerRecords<String, String> records = consumer.poll(1000);
                for (ConsumerRecord<String, String> record : records) {
                    System.out.println(record.toString());
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            consumer.close();
        }
    }
}