Kafka Hopping Time Window

最新推荐文章于 2025-07-31 11:48:45 发布

原创最新推荐文章于 2025-07-31 11:48:45 发布 · 801 阅读

0 ·

CC 4.0 BY-SA版权

Kafka 专栏收录该内容

3 篇文章

订阅专栏

本文介绍了Kafka中的跳跃时间窗口（Hopping time windows），它基于时间间隔，窗口大小可重叠。通过定义窗口大小和前进间隔，可以实现每隔五秒钟统计过去三十秒内的value指标。同时，使用.grace(Duration.ZERO)处理数据延迟，suppress方法用于抑制输出，等待窗口关闭再发送数据。总结中提到，同一时刻存在窗口的数量取决于窗口大小与前进间隔的比例，并指出相同key的最后一条消息可能无法被统计到的问题。

跳跃时间窗口Hopping time windows是基于时间间隔的窗口。它们为固定大小(可能)重叠的窗口建模。跳跃窗口由两个属性定义:窗口的size及其前进间隔advance。前进间隔指定一个窗口相对于前一个窗口向前移动多少。

案例：每隔五秒钟统计当前时间的前三十秒内 value的最大值、最小值和平均值。

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.example.demo.vo.WindowResult;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.KafkaStreams;
import org.apache.kafka.streams.StreamsBuilder;
import org.apache.kafka.streams.StreamsConfig;
import org.apache.kafka.streams.kstream.KStream;
import org.apache.kafka.streams.kstream.Materialized;
import org.apache.kafka.streams.kstream.Suppressed;
import org.apache.kafka.streams.kstream.TimeWindows;
import org.springframework.stereotype.Component;

import java.time.Duration;
import java.time.Instant;
import java.util.Properties;
import java.util.concurrent.TimeUnit;

import static org.apache.kafka.streams.kstream.Suppressed.BufferConfig.unbounded;

@Component
public class HoppingTimeWindowDemo {

    public static void main(String[] args) {
        final static int TIME_WINDOW_SECONDS = 30;
        final static int ADVANCED_BY_SECONDS = 10;

        Properties props = new Properties();
        props.put(StreamsConfig.APPLICATION_ID_CONFIG, "hopping-time-window");
        props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
        props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
        props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
        props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
        props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);

        StreamsBuilder builder = new StreamsBuilder();

        KStream<String, String> valueStream = builder.stream("value");

        valueStream.selectKey((key, value) -> {
            String newKey;
            try {
                JSONObject json = JSON.parseObject(value);
                newKey = json.getString("uid");
            } catch (Exception ex) {
                return key;
            }
            return newKey;
        }).groupByKey().windowedBy(TimeWindows.of(Duration.ofSeconds(TIME_WINDOW_SECONDS)).advanceBy(Duration.ofSeconds(ADVANCED_BY_SECONDS)).grace(Duration.ZERO))
                .aggregate(() -> {
                    WindowResult result = new WindowResult(0, 0, 0, 0, 0, 0, null);
                    return JSONObject.toJSONString(result);
                }, (aggKey, newValue, aggValue) -> {
                    // topic中的消息格式为{"uid": 1,"value":19}
                    WindowResult result = JSONObject.parseObject(aggValue, WindowResult.class);
                    Long newValueLong = null;
                    try {
                        JSONObject json = JSON.parseObject(newValue);
                        newValueLong = json.getLong("value");
                    } catch (ClassCastException ex) {

                    }
                    if (result.getMin() == 0 || result.getMin() > newValueLong) {
                        result.setMin(newValueLong);
                    }
                    if (result.getMax() == 0 || result.getMax() < newValueLong) {
                        result.setMax(newValueLong);
                    }
                    result.setUid(Integer.valueOf(aggKey));
                    result.setCount(result.getCount() + 1);
                    result.setSum(result.getSum() + newValueLong);
                    result.setAvg(result.getSum() / result.getCount());
                    return JSONObject.toJSONString(result);
                }, Materialized.with(Serdes.String(), Serdes.String()))
                .suppress(Suppressed.untilWindowCloses(unbounded())).toStream().mapValues((key, value) -> {
                //获取窗口的结束时间，由于时区问题，需要给获取到的时间加上八小时
            Instant end = key.window().endTime().plusMillis(TimeUnit.HOURS.toMillis(8));
            WindowResult result = JSONObject.parseObject(value, WindowResult.class);
            result.setWindowEnd(end.toString());
            return JSONObject.toJSONString(result);
        }).to("window-result");

        final KafkaStreams streams = new KafkaStreams(builder.build(), props);

        streams.start();

    }
}