kafka-streams自定义窗口抑制

/**
 * kafka stream配置
 * @Author: Clown
 * @Version: 1.0
 * @Date: 2021/7/12 16:52
 */
@Configuration
public class KafkaStreamConfig {
    @Value("${kafka.stream.application.id}")
    private String applicationId;
    @Value("${kafka.consumer.servers}")
    private String bootstrapServers;
    @Value("${kafka.stream.thread.num}")
    private int threadNum;

    public static final String MY_STREAMS_CONFIG_BEAN_NAME = "myKafkaStreamsConfig";
    public static final String MY_STREAMS_BUILDER_BEAN_NAME = "myKafkaStreamsBuilder";

    @Bean(name = MY_STREAMS_CONFIG_BEAN_NAME)
    public KafkaStreamsConfiguration myStreamsConfiguration() {
        return new KafkaStreamsConfiguration(initConfigProps(applicationId));
    }

    @Bean(name = MY_STREAMS_BUILDER_BEAN_NAME)
    public StreamsBuilderFactoryBean myKafkaStreamsBuilder(
            @Qualifier(MY_STREAMS_CONFIG_BEAN_NAME)
                    ObjectProvider<KafkaStreamsConfiguration> streamsConfigProvider,
            ObjectProvider<StreamsBuilderFactoryBeanCustomizer> customizerProvider) {

        KafkaStreamsConfiguration streamsConfig = streamsConfigProvider.getIfAvailable();
        if (streamsConfig != null) {
            StreamsBuilderFactoryBean fb = new StreamsBuilderFactoryBean(streamsConfig);
            StreamsBuilderFactoryBeanCustomizer customizer = customizerProvider.getIfUnique();
            if (customizer != null) {
                customizer.configure(fb);
            }
            return fb;
        }
        else {
            throw new UnsatisfiedDependencyException(KafkaStreamsDefaultConfiguration.class.getName(),
                    MY_STREAMS_BUILDER_BEAN_NAME, "streamsConfig", "There is no '" +
                    MY_STREAMS_CONFIG_BEAN_NAME + "' " + KafkaStreamsConfiguration.class.getName() +
                    " bean in the application context.\n" +
                    "Consider declaring one or don't use @EnableKafkaStreams.");
        }
    }

    private Map<String, Object> initConfigProps(String applicationId) {
        Map<String, Object> props = Maps.newHashMap();
        props.put(StreamsConfig.APPLICATION_ID_CONFIG, applicationId);
        props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
        props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
        props.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, threadNum);
        props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
        props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());

        props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");
        props.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000);
        props.put(StreamsConfig.REPLICATION_FACTOR_CONFIG, 1);
        props.put(StreamsConfig.RETRY_BACKOFF_MS_CONFIG, 5000);
        props.put(StreamsConfig.STATE_CLEANUP_DELAY_MS_CONFIG, 24 * 60 * 60 *1000);
        props.put(StreamsConfig.SEND_BUFFER_CONFIG, 3 * 16 * 1024 * 1024);
        props.put(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, EventTimeExtractor.class.getName());
        props.put(StreamsConfig.DEFAULT_PRODUCTION_EXCEPTION_HANDLER_CLASS_CONFIG, StreamProductionExceptionHandler.class.getName());
        props.put(StreamsConfig.DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG, StreamDeserializationExceptionHandler.class.getName());
        return props;
    }
}

/**
 * 事件时间生成器
 * @Author: Clown
 * @Version: 1.0
 * @Date: 2021/8/26 14:59
 */
public class EventTimeExtractor implements TimestampExtractor {
    @Override
    public long extract(ConsumerRecord<Object, Object> record, long partitionTime) {
        return System.currentTimeMillis();
    }
}

/**
 * 数据流反序列化异常处理器
 * @Author: Clown
 * @Version: 1.0
 * @Date: 2021/8/31 14:10
 */
public class StreamDeserializationExceptionHandler implements DeserializationExceptionHandler {
    private static final Logger log = LoggerFactory.getLogger(StreamDeserializationExceptionHandler.class);
    @Override
    public DeserializationHandlerResponse handle(ProcessorContext context, ConsumerRecord<byte[], byte[]> record, Exception exception) {
        log.error("stream deserialization error", exception);
        return DeserializationHandlerResponse.CONTINUE;
    }

    @Override
    public void configure(Map<String, ?> configs) {

    }
}

/**
 * 数据流生产者异常处理器
 * @Author: Clown
 * @Version: 1.0
 * @Date: 2021/8/31 13:57
 */
public class StreamProductionExceptionHandler implements ProductionExceptionHandler {
    private static final Logger log = LoggerFactory.getLogger(StreamProductionExceptionHandler.class);
    @Override
    public ProductionExceptionHandlerResponse handle(ProducerRecord<byte[], byte[]> record, Exception exception) {
        log.error("stream production error", exception);
        return ProductionExceptionHandlerResponse.CONTINUE;
    }

    @Override
    public void configure(Map<String, ?> configs) {

    }
}

/**
 * 抑制窗口信息
 * @Author: Clown
 * @Version: 1.0
 * @Date: 2021/8/27 13:44
 */
public class SuppressWindowInfo<T> {
    private Long windowEndTime;
    private T windowData;

    public Long getWindowEndTime() {
        return windowEndTime;
    }

    public SuppressWindowInfo<T> setWindowEndTime(Long windowEndTime) {
        this.windowEndTime = windowEndTime;
        return this;
    }

    public T getWindowData() {
        return windowData;
    }

    public SuppressWindowInfo<T> setWindowData(T windowData) {
        this.windowData = windowData;
        return this;
    }
}

/**
 * 抑制窗口
 * @Author: Clown
 * @Version: 1.0
 * @Date: 2021/8/27 14:21
 */
public class SuppressWindowTransform {
    private static final Logger log = LoggerFactory.getLogger(SuppressWindowTransform.class);
    private String storeName;
    private Long scheduleInterval;

    public SuppressWindowTransform(String storeName, Long scheduleInterval) {
        this.storeName = storeName;
        this.scheduleInterval = scheduleInterval;
    }

    public TransformerSupplier<String, String, KeyValue<String, String>> getTransformSupplier() {
        return () -> new Transformer<String, String, KeyValue<String, String>>() {
            /**
             * 每个stream线程持有一个ProcessorContext
             */
            private ProcessorContext context;
            /**
             * 每个ProcessorContext中的stateStore对应storeName topic中的一个partition
             */
            private KeyValueStore<String, String> kvStore;
            private Cancellable cancellable;

            @Override
            public void init(ProcessorContext context) {
                if (StringUtils.isEmpty(storeName)) {
                    throw new IllegalArgumentException("store name can not be empty");
                }
                if (scheduleInterval <= 0) {
                    throw new IllegalArgumentException("schedule interval must be greater than 0");
                }
                this.context = context;
                kvStore = (KeyValueStore<String, String>) context.getStateStore(storeName);
                cancellable = context.schedule(Duration.ofMillis(scheduleInterval), PunctuationType.WALL_CLOCK_TIME, this::wallClockTimePunctuator);
            }

            /**
             * 收集窗口数据
             * @param key windowKey+windowEndTime, 使用"_"分隔
             * @param value 窗口数据
             * @return
             */
            @Override
            public KeyValue<String, String> transform(String key, String value) {
                if (StringUtils.isEmpty(value)) {
                    return null;
                }
                kvStore.put(key, value);
                return null;
            }

            /**
             * 使用当前时间判断窗口是否关闭
             * @param timestamp
             */
            void wallClockTimePunctuator(Long timestamp){
                log.info("++++ 开始关闭窗口 ++++");
                int count = 0;
                int close = 0;
                long beginTime = System.currentTimeMillis();
                KeyValue<String, String> keyValue = null;
                try (KeyValueIterator<String, String> iterator = kvStore.all()) {
                    while (iterator.hasNext()) {
                        keyValue = iterator.next();
                        String windowKey = keyValue.key;
                        String windowValue = keyValue.value;
                        if (StringUtils.isEmpty(windowValue)) {
                            kvStore.delete(windowKey);
                            continue;
                        }
                        SuppressWindowInfo suppressWindowInfo = JSON.parseObject(windowValue, SuppressWindowInfo.class);
                        long windowEndTime = suppressWindowInfo.getWindowEndTime();
                        if (windowEndTime <= timestamp) {
                            String windowData = JSON.toJSONString(suppressWindowInfo.getWindowData());
                            context.forward(windowKey, windowData);
                            kvStore.delete(windowKey);
                            close++;
                        }
                        count++;
                    }
                    kvStore.flush();
                } catch (Exception e) {
                    if (keyValue != null) {
                        kvStore.delete(keyValue.key);
                    }
                    log.error("窗口检测程序异常", e);
                }
                log.info("++++ 关闭窗口，轮询次数={}，关闭次数={}，剩余={}，轮询耗时={} ++++", count, close, count-close, System.currentTimeMillis()- beginTime);
            }

            @Override
            public void close() {
                cancellable.cancel();
            }
        };
    }
}

/**
 * 我的统计
 * @Author: Clown
 * @Version: 1.0
 * @Date: 2021/7/12 17:23
 */
@Configuration
public class MyStatisticsKafkaStream {
    private static final Logger log = LoggerFactory.getLogger(MyStatisticsKafkaStream.class);
    @Value("${kafka.consumer.topic.my}")
    private String myTopic;
    @Value("${kafka.stream.topic.my.statistic}")
    private String myStatisticsTopic;
    @Value("${kafka.stream.windows.duration}")
    private int windowsDuration;

    private static final JsonSerializer<MyStatisticsMustDto> MY_STATISTIC_MUST_SERIALIZER = new JsonSerializer<>();
    private static final JsonDeserializer<MyStatisticsMustDto> MY_STATISTIC_MUST_DESERIALIZER = new JsonDeserializer<>(MyStatisticsMustDto.class);
    private static final Serde<MyStatisticsMustDto> MY_STATISTIC_MUST_SERDE = Serdes.serdeFrom(MY_STATISTIC_MUST_SERIALIZER, MY_STATISTIC_MUST_DESERIALIZER);

    private static final JsonSerializer<MyStatisticsDto> MY_STATISTIC_SERIALIZER = new JsonSerializer<>();
    private static final JsonDeserializer<MyStatisticsDto> MY_STATISTIC_DESERIALIZER = new JsonDeserializer<>(MyStatisticsDto.class);
    private static final Serde<MyStatisticsDto> MY_STATISTIC_SERDE = Serdes.serdeFrom(MY_STATISTIC_SERIALIZER, MY_STATISTIC_DESERIALIZER);
    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_DEFAULT);

    private static final String MY_STATISTIC_STORE = "statistics_store";
    private static final String SUPPRESS_WINDOW_STORE = "suppress_window_store";
    private static final String KEY_DELIMITER = "_";

    StoreBuilder<KeyValueStore<String, String>> keyValueStoreBuilder =
              Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore(SUPPRESS_WINDOW_STORE),
                      Serdes.String(),
                      Serdes.String());

    /**
     * 我的统计
     * @param streamsBuilder
     * @return
     */
    @Bean(name = "myStatisticsStream")
    public KStream<String, String> myStatistics(@Qualifier(MY_STREAMS_BUILDER_BEAN_NAME)StreamsBuilder streamsBuilder) {
        streamsBuilder.addStateStore(keyValueStoreBuilder);
        KStream<String, String> kStream = streamsBuilder.stream(myTopic, Consumed.with(Serdes.String(), Serdes.String()));
        KGroupedStream<String, MyStatisticsMustDto> kGroupedStream = kStream.map((k, v) -> {
            IovMyDto myInfo = JSON.parseArray(v, IovMyDto.class).get(0);
            MyStatisticsMustDto myStatisticsMust = new MyStatisticsMustDto();
            myStatisticsMust.setVin(myInfo.getVin());
            myStatisticsMust.setDate(Long.parseLong(DateUtil.format(DateUtil.date(myInfo.getDate()), DatePattern.PURE_DATE_PATTERN)));
            myStatisticsMust.setDeviceCode(myInfo.getDeviceCode());
            myStatisticsMust.setDeviceType(DeviceTypeEnum.getEnum(String.valueOf(myInfo.getDeviceType())) == null ? 0 : DeviceTypeEnum.getEnum(String.valueOf(myInfo.getDeviceType())).mysqlCode);
            myStatisticsMust.setVehicleId(myInfo.getVehicleId());
            myStatisticsMust.setPlateNo(myInfo.getPlateNo());
            myStatisticsMust.setVehicleType(myInfo.getVehicleType());
            myStatisticsMust.setAreaId(myInfo.getAreaId());
            myStatisticsMust.setSubtotalMileage(myInfo.getSubtotalMileage() > 10000 ? 0 : myInfo.getSubtotalMileage());
                    return KeyValue.pair(String.join(KEY_DELIMITER, myInfo.getVin(), DateUtil.format(DateUtil.date(myInfo.getDate()), DatePattern.PURE_DATE_PATTERN)), myStatisticsMust);
                })
                .groupByKey(Grouped.with(Serdes.String(), MY_STATISTIC_MUST_SERDE));

        kGroupedStream.windowedBy(TimeWindows.of(Duration.ofMinutes(windowsDuration)).grace(Duration.ZERO))
                .aggregate(
                        MyStatisticsDto::new,
                        (k, v, vr) -> {
                            vr.setVin(v.getVin());
                            vr.setDate(v.getDate());
                            vr.setDeviceCode(v.getDeviceCode());
                            vr.setDeviceType(v.getDeviceType());
                            vr.setVehicleId(v.getVehicleId());
                            vr.setPlateNo(v.getPlateNo());
                            vr.setVehicleType(v.getVehicleType());
                            vr.setAreaId(v.getAreaId());
                            vr.setMyCount(vr.getMyCount() + 1);
                            vr.setMileageCount(vr.getMileageCount() + v.getSubtotalMileage());
                            return vr;
                        }
                        , Materialized.<String, MyStatisticsDto, WindowStore<Bytes, byte[]>>as(MY_STATISTIC_STORE).withKeySerde(Serdes.String()).withValueSerde(MY_STATISTIC_SERDE)
                )
                .toStream()
                .map((k, v) -> {
                    long windowEndTime = k.window().end();
                    SuppressWindowInfo<MyStatisticsDto> suppressWindowInfo = new SuppressWindowInfo();
                    suppressWindowInfo.setWindowEndTime(windowEndTime);
                    suppressWindowInfo.setWindowData(v);
                    String suppressWindowKey = String.join(KEY_DELIMITER, k.key(), String.valueOf(windowEndTime));
                    String suppressWindowValue = null;
                    try {
                        suppressWindowValue = OBJECT_MAPPER.writeValueAsString(suppressWindowInfo);
                    } catch (JsonProcessingException e) {
                        log.error("我的统计窗口抑制数据序列化异常", e);
                    }
                    return KeyValue.pair(suppressWindowKey, suppressWindowValue);
                })
                .transform(new SuppressWindowTransform(SUPPRESS_WINDOW_STORE, (long)windowsDuration * 60 * 1000).getTransformSupplier(), SUPPRESS_WINDOW_STORE)
                .to(myStatisticsTopic);
        return kStream;
    }
}