Apache Flink Java 示例：实时温度监控（连接流）
本示例展示如何使用 Apache Flink 的连接流(ConnectedStreams)功能实现实时温度监控系统。该系统将处理两种数据流：温度传感器数据和设备告警规则数据，实现动态阈值监控和异常检测。
应用场景说明
- 温度数据流：从传感器接收实时温度读数
- 规则数据流：接收设备告警规则的动态更新
- 连接处理：将温度数据与最新规则动态关联
- 告警输出：当温度超过动态设定的阈值时触发告警
完整实现代码
1. 数据模型定义
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.state.*;
import org.apache.flink.api.common.typeinfo.TypeHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.tuple.*;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.*;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.co.*;
import org.apache.flink.util.Collector;
import java.time.Duration;
// 温度传感器事件
public class TemperatureEvent {
private String sensorId; // 传感器ID
private double temperature; // 温度值
private long timestamp; // 事件时间戳
// 构造方法/getters/setters
public TemperatureEvent(String sensorId, double temperature, long timestamp) {
this.sensorId = sensorId;
this.temperature = temperature;
this.timestamp = timestamp;
}
public String getSensorId() { return sensorId; }
public double getTemperature() { return temperature; }
public long getTimestamp() { return timestamp; }
}
// 设备告警规则
public class AlertRule {
private String deviceId; // 设备ID
private double minTemp; // 最低温度阈值
private double maxTemp; // 最高温度阈值
private long timestamp; // 规则更新时间
// 构造方法/getters/setters
public AlertRule(String deviceId, double minTemp, double maxTemp, long timestamp) {
this.deviceId = deviceId;
this.minTemp = minTemp;
this.maxTemp = maxTemp;
this.timestamp = timestamp;
}
public String getDeviceId() { return deviceId; }
public double getMinTemp() { return minTemp; }
public double getMaxTemp() { return maxTemp; }
public long getTimestamp() { return timestamp; }
}
// 温度告警事件
public class TemperatureAlert {
private String deviceId; // 设备ID
private double temperature; // 实际温度
private double threshold; // 超过的阈值
private String alertType; // 告警类型（HIGH/LOW）
private long timestamp; // 告警时间
public TemperatureAlert(String deviceId, double temperature,
double threshold, String alertType, long timestamp) {
this.deviceId = deviceId;
this.temperature = temperature;
this.threshold = threshold;
this.alertType = alertType;
this.timestamp = timestamp;
}
@Override
public String toString() {
return String.format("[ALERT] %s - %s: %.2f°C (Threshold: %.2f°C) @ %s",
deviceId, alertType, temperature, threshold, new Date(timestamp));
}
}
2. 核心处理逻辑 - 连接流处理器
public class TemperatureMonitor {
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(4);
// 1. 创建温度数据流（模拟）
DataStream<TemperatureEvent> tempStream = env
.addSource(new TemperatureSource())
.name("Temperature Source")
.uid("temperature-source");
// 2. 创建规则数据流（模拟）
DataStream<AlertRule> ruleStream = env
.addSource(new RuleSource())
.name("Rule Source")
.uid("rule-source");
// 3. 连接两条流（基于sensorId/deviceId）
ConnectedStreams<TemperatureEvent, AlertRule> connectedStreams = tempStream
.connect(ruleStream)
.keyBy(TemperatureEvent::getSensorId, AlertRule::getDeviceId);
// 4. 应用连接流处理函数
DataStream<TemperatureAlert> alerts = connectedStreams
.process(new TemperatureAlertProcessor())
.name("Temperature Alert Processor")
.uid("temperature-alert-processor");
// 5. 输出告警信息
alerts.print();
env.execute("Real-time Temperature Monitoring");
}
// 连接流处理函数（核心逻辑）
public static class TemperatureAlertProcessor
extends KeyedCoProcessFunction<String, TemperatureEvent, AlertRule, TemperatureAlert> {
// 状态存储：当前设备的告警规则
private ValueState<AlertRule> currentRuleState;
// 状态存储：历史最高温和最低温（用于温度突变检测）
private ValueState<Tuple2<Double, Double>> historyTempState;
// 状态存储：最后一次告警时间（用于防止警报洪泛）
private ValueState<Long> lastAlertTimeState;
@Override
public void open(Configuration parameters) {
// 初始化告警规则状态
ValueStateDescriptor<AlertRule> ruleDescriptor =
new ValueStateDescriptor<>("current-rule", AlertRule.class);
currentRuleState = getRuntimeContext().getState(ruleDescriptor);
// 初始化历史温度状态（max, min）
ValueStateDescriptor<Tuple2<Double, Double>> tempDescriptor =
new ValueStateDescriptor<>("history-temp",
TypeInformation.of(new TypeHint<Tuple2<Double, Double>>() {}));
historyTempState = getRuntimeContext().getState(tempDescriptor);
// 初始化告警时间状态
ValueStateDescriptor<Long> timeDescriptor =
new ValueStateDescriptor<>("last-alert-time", Long.class);
lastAlertTimeState = getRuntimeContext().getState(timeDescriptor);
}
// 处理温度事件
@Override
public void processElement1(TemperatureEvent tempEvent,
Context context,
Collector<TemperatureAlert> out) throws Exception {
// 1. 获取当前设备规则（可能为null）
AlertRule rule = currentRuleState.value();
String sensorId = context.getCurrentKey();
// 2. 如果规则存在，检查温度是否超出阈值
if (rule != null) {
double currentTemp = tempEvent.getTemperature();
long currentTime = System.currentTimeMillis();
// 防止警报洪泛（每分钟最多告警一次）
Long lastAlertTime = lastAlertTimeState.value();
if (lastAlertTime != null && (currentTime - lastAlertTime) < 60000) {
return; // 60秒内已告警过
}
// 3. 检查高温告警
if (currentTemp > rule.getMaxTemp()) {
lastAlertTimeState.update(currentTime);
out.collect(new TemperatureAlert(
sensorId,
currentTemp,
rule.getMaxTemp(),
"HIGH",
currentTime
));
}
// 4. 检查低温告警
else if (currentTemp < rule.getMinTemp()) {
lastAlertTimeState.update(currentTime);
out.collect(new TemperatureAlert(
sensorId,
currentTemp,
rule.getMinTemp(),
"LOW",
currentTime
));
}
// 5. 检查温度突变（与历史数据比较）
Tuple2<Double, Double> historyTemp = historyTempState.value();
if (historyTemp != null) {
double maxHistory = historyTemp.f0;
double minHistory = historyTemp.f1;
// 检测温度骤升（2分钟内升高超过5℃）
double deltaUp = currentTemp - maxHistory;
if (deltaUp > 5.0) {
out.collect(new TemperatureAlert(
sensorId,
currentTemp,
maxHistory,
"SPIKE_UP",
currentTime
));
}
// 检测温度骤降（2分钟内下降超过5℃）
double deltaDown = minHistory - currentTemp;
if (deltaDown > 5.0) {
out.collect(new TemperatureAlert(
sensorId,
currentTemp,
minHistory,
"SPIKE_DOWN",
currentTime
));
}
}
// 6. 更新历史温度状态
double newMax = Math.max(currentTemp, historyTemp != null ? historyTemp.f0 : currentTemp);
double newMin = Math.min(currentTemp, historyTemp != null ? historyTemp.f1 : currentTemp);
historyTempState.update(Tuple2.of(newMax, newMin));
}
}
// 处理规则更新事件
@Override
public void processElement2(AlertRule rule,
Context context,
Collector<TemperatureAlert> out) throws Exception {
// 更新当前设备的告警规则
currentRuleState.update(rule);
System.out.println("Rule updated for " + rule.getDeviceId() +
": Min=" + rule.getMinTemp() + ", Max=" + rule.getMaxTemp());
}
}
// ========================= 模拟数据源 =========================
// 模拟温度数据源
private static class TemperatureSource extends RichParallelSourceFunction<TemperatureEvent> {
private volatile boolean running = true;
private final Random random = new Random();
private static final String[] DEVICE_IDS = {"device-001", "device-002", "device-003", "device-004"};
private static final double[] BASE_TEMPS = {20.0, 22.0, 18.0, 25.0}; // 各设备基础温度
@Override
public void run(SourceContext<TemperatureEvent> ctx) throws Exception {
while (running) {
for (int i = 0; i < DEVICE_IDS.length; i++) {
String deviceId = DEVICE_IDS[i];
// 模拟温度波动（基础温度±10度）
double temp = BASE_TEMPS[i] + (random.nextDouble() * 20 - 10);
// 10%概率产生异常值
if (random.nextDouble() < 0.1) {
temp += (random.nextBoolean() ? 15 : -15);
}
ctx.collect(new TemperatureEvent(deviceId, temp, System.currentTimeMillis()));
}
Thread.sleep(500); // 每0.5秒发送一次所有设备数据
}
}
@Override
public void cancel() {
running = false;
}
}
// 模拟规则数据源
private static class RuleSource extends RichParallelSourceFunction<AlertRule> {
private volatile boolean running = true;
private final Random random = new Random();
private static final String[] DEVICE_IDS = {"device-001", "device-002", "device-003", "device-004"};
@Override
public void run(SourceContext<AlertRule> ctx) throws Exception {
// 初始规则
ctx.collect(new AlertRule("device-001", 15.0, 30.0, System.currentTimeMillis()));
ctx.collect(new AlertRule("device-002", 18.0, 35.0, System.currentTimeMillis()));
ctx.collect(new AlertRule("device-003", 12.0, 28.0, System.currentTimeMillis()));
ctx.collect(new AlertRule("device-004", 20.0, 40.0, System.currentTimeMillis()));
Thread.sleep(30_000); // 等待30秒
// 模拟规则动态更新
while (running) {
for (String deviceId : DEVICE_IDS) {
// 随机更新阈值（±3度）
double minChange = random.nextDouble() * 6 - 3;
double maxChange = random.nextDouble() * 6 - 3;
// 新规则范围：15-35度
double newMin = Math.max(15.0, 20.0 + minChange);
double newMax = Math.min(35.0, 30.0 + maxChange);
ctx.collect(new AlertRule(deviceId, newMin, newMax, System.currentTimeMillis()));
}
Thread.sleep(60_000); // 每分钟更新一次规则
}
}
@Override
public void cancel() {
running = false;
}
}
}
3. 连接流处理流程详解
4. 状态管理策略
-
告警规则状态 (
currentRuleState)- 存储每个设备的最新告警规则
- 通过规则数据流(
processElement2)更新 - 用于温度数据流的阈值检查
-
历史温度状态 (
historyTempState)- 存储每个设备的最高和最低温度
- 用于检测温度突变事件
- 随着新温度数据的到来持续更新
-
告警时间状态 (
lastAlertTimeState)- 防止警报洪泛机制
- 确保每分钟最多触发一次阈值告警
- 不影响温度突变告警的实时性
5. 生产环境增强建议
// 配置RocksDB状态后端（处理大规模状态数据）
env.setStateBackend(new EmbeddedRocksDBStateBackend());
// 启用检查点（保证状态一致性）
env.enableCheckpointing(60_000); // 60秒一次
// Kafka数据源配置（实际生产环境使用）
Properties tempProps = new Properties();
tempProps.setProperty("bootstrap.servers", "kafka-broker:9092");
DataStream<TemperatureEvent> kafkaTempStream = env
.addSource(new FlinkKafkaConsumer<>(
"temperature-topic",
new JsonDeserialization<>(TemperatureEvent.class),
tempProps
));
// 告警输出到时序数据库
alerts.addSink(new InfluxDBSink(
"http://influxdb:8086",
"temperature_alerts",
"admin",
"password"
));
// 添加监控仪表板
alerts.map(alert -> {
Map<String, Object> metrics = new HashMap<>();
metrics.put("device", alert.getDeviceId());
metrics.put("value", alert.getTemperature());
metrics.put("alertType", alert.getAlertType());
return metrics;
}).addSink(new ElasticsearchSink<>("monitoring-es:9200"));
6. 高级功能扩展
-
设备分组告警
// 添加设备分组信息到规则 public class AlertRule { private String deviceGroup; // 新增分组字段 // ... } // 分组级别告警 alerts.keyBy(alert -> alert.getDeviceGroup()) .process(new GroupAlertProcessor()); -
温度预测模型
// 使用状态存储历史温度序列 ListState<TemperatureEvent> temperatureHistoryState; // 在processElement1中 temperatureHistoryState.add(tempEvent); // 调用预测模型 if (needPredict()) { List<TemperatureEvent> history = Lists.newArrayList(temperatureHistoryState.get()); double prediction = predictionModel.predict(history); if (prediction > threshold) { // 生成预测告警 } } -
动态规则传播
// 广播规则流 MapStateDescriptor<String, AlertRule> ruleDescriptor = new MapStateDescriptor<>("rule-broadcast", String.class, AlertRule.class); BroadcastStream<AlertRule> broadcastRuleStream = ruleStream .broadcast(ruleDescriptor); // 连接广播流 tempStream.connect(broadcastRuleStream) .process(new BroadcastRuleProcessor());
连接流处理流程全貌
通过此示例，您可掌握 Flink 连接流的核心应用模式。这种技术特别适用于需要将动态配置与实时数据流结合的监控场景，例如物联网设备监控、实时风控系统和动态定价系统等。
扫一扫
742
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
