将list里的新旧数据做对比生成不存在的数据的新旧集合并转换为JsonObject

本文介绍了如何在工程中对比list中的新旧数据,找出不存在的数据,并将结果转换为JsonObject。首先建立工程,然后通过代码实现比对和转换操作,最终得到所需的数据集合。

摘要生成于 C知道 ,由 DeepSeek-R1 满血版支持, 前往体验 >

1.建立工程

2.代码

package test;

import java.util.ArrayList;
import java.util.List;

import org.junit.Test;

import net.sf.json.JSONObject;

public class compare {

@Test
public void test() {

    // 建立新的List
    List<String> newList = new ArrayList<String>();
    newList.add("1");
    newList.add("2");
    newList.add("3");
    newList.add("5");
    newList.add("7");
    newList.add("9");
    System.out.println(newList);
    System.out.println("-------------------------------");


    // 建立一个原有的List
    List<String> oldList = new ArrayList<String>();
    oldList.add("2");
    oldList.add("3");
    oldList.add("4");
    oldList.add("5");
    oldList.add("6");
    oldList.add("8");
    oldList.add("10");
    System.out.println(oldList);
    System.out.println("-------------------------------");


    // 建立一个存储newList比oldList里多出的东西的newListMore
    List<Strin
package com.tongchuang.realtime.mds; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.tongchuang.realtime.bean.ULEParamConfig; import com.tongchuang.realtime.util.KafkaUtils; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.common.state.; import org.apache.flink.api.common.time.Time; import org.apache.flink.api.common.typeinfo.BasicTypeInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.configuration.Configuration; import org.apache.flink.connector.kafka.source.KafkaSource; import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer; import org.apache.flink.streaming.api.datastream.; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.co.KeyedBroadcastProcessFunction; import org.apache.flink.streaming.api.functions.source.RichSourceFunction; import org.apache.flink.util.Collector; import java.io.Serializable; import java.sql.; import java.text.SimpleDateFormat; import java.util.; import java.util.Date; import java.util.concurrent.TimeUnit; public class ULEDataanomalyanalysis { public static void main(String[] args) throws Exception { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); // 创建Kafka消费者 KafkaSource<String> kafkaConsumer = KafkaUtils.getKafkaConsumer( "realdata_minute", "minutedata_uledataanomalyanalysis", OffsetsInitializer.latest() ); DataStreamSource<String> kafkaDS = env.fromSource( kafkaConsumer, WatermarkStrategy.noWatermarks(), "realdata_uledataanomalyanalysis" ); // 解析JSON拆分每个tag的数据 SingleOutputStreamOperator<JSONObject> splitStream = kafkaDS .map(JSON::parseObject) .flatMap((JSONObject value, Collector<JSONObject> out) -> { JSONObject data = value.getJSONObject("datas"); String time = value.getString("times"); for (String tag : data.keySet()) { JSONObject tagData = data.getJSONObject(tag); JSONObject newObj = new JSONObject(); newObj.put("time", time); newObj.put("tag", tag); newObj.put("ontime", tagData.getDouble("ontime")); newObj.put("avg", tagData.getDouble("avg")); out.collect(newObj); } }) .returns(TypeInformation.of(JSONObject.class)) .name("Split-By-Tag"); // 每5分钟加载参数配置 DataStream<ConfigCollection> configDataStream = env .addSource(new MysqlConfigSource()) .setParallelism(1) .filter(Objects::nonNull) .name("Config-Source"); // 将配置流转换为广播流 BroadcastStream<ConfigCollection> configBroadcastStream = configDataStream .broadcast(Descriptors.configStateDescriptor); // 按tag分组连接广播流 KeyedStream<JSONObject, String> keyedStream = splitStream .keyBy(json -> json.getString("tag")); BroadcastConnectedStream<JSONObject, ConfigCollection> connectedStream = keyedStream.connect(configBroadcastStream); // 异常检测处理 SingleOutputStreamOperator<JSONObject> anomalyStream = connectedStream .process(new OptimizedAnomalyDetectionFunction()) .name("Anomaly-Detection"); anomalyStream.print("异常检测结果"); // anomalyStream.map(JSON::toString).addSink(KafkaUtils.getKafkaSink("minutedata_uleanomaly")); env.execute("uledataanomalyanalysis"); } // 配置集合类 - 修复配置变化检测问题 public static class ConfigCollection implements Serializable { private static final long serialVersionUID = 1L; public final Map<String, List<ULEParamConfig>> tagToConfigs; public final Map<String, ULEParamConfig> encodeToConfig; private final long contentHash; // 添加内容哈希值用于精确检测变化 public ConfigCollection(Map<String, List<ULEParamConfig>> tagToConfigs, Map<String, ULEParamConfig> encodeToConfig) { this.tagToConfigs = new HashMap<>(tagToConfigs); this.encodeToConfig = new HashMap<>(encodeToConfig); // 计算内容哈希值(考虑所有配置项的内容) this.contentHash = calculateContentHash(); } private long calculateContentHash() { long hash = 0L; // 对每个配置项计算哈希(使用所有字段) for (ULEParamConfig config : encodeToConfig.values()) { hash = 31 * hash + Objects.hash( config.tag, config.encode, config.datatype, config.constantvalue, config.isonline, config.issync, config.syncparaencode, config.iszero, config.ishigh, config.highthreshold, config.islow, config.lowthreshold, config.duration ); } return hash; } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; ConfigCollection that = (ConfigCollection) o; return contentHash == that.contentHash; } @Override public int hashCode() { return (int) (contentHash ^ (contentHash >>> 32)); } public int configCount() { return encodeToConfig.size(); } } // MySQL配置源 - 优化配置加载和变化检测 public static class MysqlConfigSource extends RichSourceFunction<ConfigCollection> { private volatile boolean isRunning = true; private final long interval = TimeUnit.MINUTES.toMillis(5); private transient ConfigCollection lastConfig; private long lastConfigHash = 0; @Override public void run(SourceContext<ConfigCollection> ctx) throws Exception { while (isRunning) { ConfigCollection newConfig = loadParams(); if (newConfig != null) { long newHash = newConfig.contentHash; if (lastConfig == null || newHash != lastConfigHash) { ctx.collect(newConfig); lastConfig = newConfig; lastConfigHash = newHash; System.out.println("检测到配置变化,广播新配置 (哈希值: " + newHash + ")"); } else { System.out.println("配置未变化,跳过广播 (哈希值: " + newHash + ")"); } } else { System.out.println("配置加载失败,使用上次配置"); } Thread.sleep(interval); } } private ConfigCollection loadParams() { Map<String, List<ULEParamConfig>> tagToConfigs = new HashMap<>(5000); Map<String, ULEParamConfig> encodeToConfig = new HashMap<>(5000); String url = "jdbc:mysql://10.51.37.73:3306/eps?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8"; String user = "root"; String password = "6CKIm5jDVsLrahSw"; String query = "SELECT F_tag AS tag, F_enCode AS encode, F_dataTypes AS datatype, " + "F_isConstantValue AS constantvalue, F_isOnline AS isonline, " + "F_isSync AS issync, F_syncParaEnCode AS syncparaencode, " + "F_isZero AS iszero, F_isHigh AS ishigh, F_highThreshold AS highthreshold, " + "F_isLow AS islow, F_lowThreshold AS lowthreshold, F_duration AS duration " + "FROM t_equipmentparameter " + "WHERE F_enabledmark = '1' AND (F_isConstantValue ='1' OR F_isZero= '1' " + "OR F_isHigh = '1' OR F_isLow = '1' OR F_isOnline = '1' OR F_isSync = '1')"; try (Connection conn = DriverManager.getConnection(url, user, password); Statement stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery(query)) { while (rs.next()) { ULEParamConfig config = new ULEParamConfig(); config.tag = rs.getString("tag"); config.encode = rs.getString("encode"); config.datatype = rs.getString("datatype"); config.constantvalue = rs.getInt("constantvalue"); config.iszero = rs.getInt("iszero"); config.ishigh = rs.getInt("ishigh"); config.highthreshold = rs.getDouble("highthreshold"); config.islow = rs.getInt("islow"); config.lowthreshold = rs.getDouble("lowthreshold"); config.duration = rs.getLong("duration"); config.isonline = rs.getInt("isonline"); config.issync = rs.getInt("issync"); config.syncparaencode = rs.getString("syncparaencode"); String tag = config.tag; tagToConfigs.computeIfAbsent(tag, k -> new ArrayList<>(10)).add(config); encodeToConfig.put(config.encode, config); } System.out.println("加载配置: " + encodeToConfig.size() + " 个参数"); return new ConfigCollection(tagToConfigs, encodeToConfig); } catch (SQLException e) { System.err.println("加载参数配置错误:"); e.printStackTrace(); return null; } } @Override public void cancel() { isRunning = false; } } // 状态描述符 public static class Descriptors { public static final MapStateDescriptor<Void, ConfigCollection> configStateDescriptor = new MapStateDescriptor<>( "configState", TypeInformation.of(Void.class), TypeInformation.of(ConfigCollection.class) ); } // 优化后的异常检测函数(带状态TTL和离线检测优化) public static class OptimizedAnomalyDetectionFunction extends KeyedBroadcastProcessFunction<String, JSONObject, ConfigCollection, JSONObject> { // 状态管理 private transient MapState<String, AnomalyState> stateMap; // key=encode private transient MapState<String, Double> lastValuesMap; // key=tag private transient MapState<String, Long> lastDataTimeMap; // key=tag private transient MapState<String, Long> offlineTimerState; // key=encode private transient SimpleDateFormat timeFormat; @Override public void open(Configuration parameters) { // 状态TTL配置(30天自动清理) StateTtlConfig ttlConfig = StateTtlConfig.newBuilder(Time.days(3000)) .setUpdateType(StateTtlConfig.UpdateType.OnCreateAndWrite) .setStateVisibility(StateTtlConfig.StateVisibility.NeverReturnExpired) .cleanupFullSnapshot() .build(); // 初始化异常状态存储(启用TTL) MapStateDescriptor<String, AnomalyState> stateDesc = new MapStateDescriptor<>( "anomalyState", BasicTypeInfo.STRING_TYPE_INFO, TypeInformation.of(AnomalyState.class) ); stateDesc.enableTimeToLive(ttlConfig); stateMap = getRuntimeContext().getMapState(stateDesc); // 初始化最新值存储(启用TTL) MapStateDescriptor<String, Double> valuesDesc = new MapStateDescriptor<>( "lastValuesState", BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.DOUBLE_TYPE_INFO ); valuesDesc.enableTimeToLive(ttlConfig); lastValuesMap = getRuntimeContext().getMapState(valuesDesc); // 初始化最后数据时间存储(启用TTL) MapStateDescriptor<String, Long> timeDesc = new MapStateDescriptor<>( "lastDataTimeState", BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO ); timeDesc.enableTimeToLive(ttlConfig); lastDataTimeMap = getRuntimeContext().getMapState(timeDesc); // 初始化离线定时器状态(启用TTL) MapStateDescriptor<String, Long> timerDesc = new MapStateDescriptor<>( "offlineTimerState", BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO ); timerDesc.enableTimeToLive(ttlConfig); offlineTimerState = getRuntimeContext().getMapState(timerDesc); timeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm"); } @Override public void processElement(JSONObject data, ReadOnlyContext ctx, Collector<JSONObject> out) throws Exception { String tag = ctx.getCurrentKey(); String timeStr = data.getString("time"); long eventTime = timeFormat.parse(timeStr).getTime(); // 更新最后数据时间 lastDataTimeMap.put(tag, eventTime); // 获取广播配置 ConfigCollection configCollection = getBroadcastConfig(ctx); if (configCollection == null) { return; } List<ULEParamConfig> configs = configCollection.tagToConfigs.get(tag); if (configs == null || configs.isEmpty()) { return; } // ========== 清理无效状态 ========== Iterator<String> stateKeys = stateMap.keys().iterator(); while (stateKeys.hasNext()) { String encode = stateKeys.next(); boolean found = false; for (ULEParamConfig cfg : configs) { if (cfg.encode.equals(encode)) { found = true; break; } } if (!found) { stateMap.remove(encode); // 清理相关定时器 Long timer = offlineTimerState.get(encode); if (timer != null) { ctx.timerService().deleteEventTimeTimer(timer); offlineTimerState.remove(encode); } } } double value = 0; boolean valueSet = false; // 遍历配置项进行异常检测 for (ULEParamConfig config : configs) { if (!valueSet) { value = "436887485805570949".equals(config.datatype) ? data.getDouble("ontime") : data.getDouble("avg"); lastValuesMap.put(tag, value); valueSet = true; } // 获取或初始化状态 AnomalyState state = getOrCreateState(config.encode); // ========== 离线恢复检测 ========== if (config.isonline == 1) { AnomalyStatus status = state.getStatus(5); if (status.reported) { // 设备重新上线,发送恢复事件 reportAnomaly(5, 0, 0.0, timeStr, config, out); status.reset(); System.out.println("设备 " + tag + " 恢复在线,发送恢复事件"); } } // 处理异常类型 checkConstantValueAnomaly(config, value, timeStr, state, out); checkZeroValueAnomaly(config, value, timeStr, state, out); checkThresholdAnomaly(config, value, timeStr, state, out); checkSyncAnomaly(config, value, timeStr, state, configCollection, out); // 设置离线检测定时器(仅当配置开启离线检测时) if (config.isonline == 1) { setupOfflineTimer(tag, config, eventTime, ctx); } // 保存状态 stateMap.put(config.encode, state); } } // 设置离线检测定时器 private void setupOfflineTimer(String tag, ULEParamConfig config, long eventTime, ReadOnlyContext ctx) throws Exception { long durationMillis = config.duration * 60 * 1000; long timerTimestamp = eventTime + durationMillis; String timerKey = config.encode; // 使用encode作为唯一标识 // 清除旧定时器(如果存在) Long oldTimer = offlineTimerState.get(timerKey); if (oldTimer != null) { ctx.timerService().deleteEventTimeTimer(oldTimer); } // 注册新定时器(事件时间) ctx.timerService().registerEventTimeTimer(timerTimestamp); offlineTimerState.put(timerKey, timerTimestamp); } @Override public void onTimer(long timestamp, OnTimerContext ctx, Collector<JSONObject> out) throws Exception { String tag = ctx.getCurrentKey(); // 获取广播配置 ConfigCollection configCollection = getBroadcastConfig(ctx); if (configCollection == null) return; List<ULEParamConfig> configs = configCollection.tagToConfigs.get(tag); if (configs == null) return; // 找出匹配的定时器 List<ULEParamConfig> triggeredConfigs = new ArrayList<>(); for (ULEParamConfig config : configs) { if (config.isonline == 1) { Long storedTimer = offlineTimerState.get(config.encode); if (storedTimer != null && storedTimer == timestamp) { triggeredConfigs.add(config); } } } // 处理所有触发的配置项 for (ULEParamConfig config : triggeredConfigs) { Long lastEventTime = lastDataTimeMap.get(tag); AnomalyState state = getOrCreateState(config.encode); AnomalyStatus status = state.getStatus(5); // 5=离线类型 if (lastEventTime == null || lastEventTime < timestamp - config.duration * 60 * 1000) { // 触发离线异常 if (!status.reported) { reportAnomaly(5, 1, 0.0, timeFormat.format(new Date(timestamp)), config, out); status.reported = true; System.out.println("设备 " + tag + " 离线报告"); } } else { // 取消误报 if (status.reported) { reportAnomaly(5, 0, 0.0, timeFormat.format(new Date()), config, out); status.reported = false; System.out.println("设备 " + tag + " 离线恢复"); } } // 重置定时器 offlineTimerState.remove(config.encode); } } // 恒值检测 private void checkConstantValueAnomaly(ULEParamConfig config, double currentValue, String timeStr, AnomalyState state, Collector<JSONObject> out) { if (config.constantvalue != 1) return; try { AnomalyStatus status = state.getStatus(1); long durationThreshold = config.duration * 60 * 1000; Date timestamp = timeFormat.parse(timeStr); if (status.lastValue == null) { status.lastValue = currentValue; status.lastChangeTime = timestamp; return; } if (Math.abs(currentValue - status.lastValue) > 0.001) { status.lastValue = currentValue; status.lastChangeTime = timestamp; if (status.reported) { reportAnomaly(1, 0, currentValue, timeStr, config, out); } status.reset(); return; } long elapsed = timestamp.getTime() - status.lastChangeTime.getTime(); if (elapsed > durationThreshold) { if (!status.reported) { reportAnomaly(1, 1, currentValue, timeStr, config, out); status.reported = true; } } } catch (Exception e) { System.err.println("恒值检测错误: " + config.encode + " - " + e.getMessage()); } } // 零值检测 private void checkZeroValueAnomaly(ULEParamConfig config, double currentValue, String timeStr, AnomalyState state, Collector<JSONObject> out) { if (config.iszero != 1) return; try { AnomalyStatus status = state.getStatus(2); Date timestamp = timeFormat.parse(timeStr); boolean isZero = Math.abs(currentValue) < 0.001; if (isZero) { if (status.startTime == null) { status.startTime = timestamp; } else if (!status.reported) { long elapsed = timestamp.getTime() - status.startTime.getTime(); if (elapsed >= config.duration * 60 * 1000) { reportAnomaly(2, 1, currentValue, timeStr, config, out); status.reported = true; } } } else { if (status.reported) { reportAnomaly(2, 0, currentValue, timeStr, config, out); status.reset(); } else if (status.startTime != null) { status.startTime = null; } } } catch (Exception e) { System.err.println("零值检测错误: " + config.encode + " - " + e.getMessage()); } } // 阈值检测 private void checkThresholdAnomaly(ULEParamConfig config, double currentValue, String timeStr, AnomalyState state, Collector<JSONObject> out) { try { if (config.ishigh == 1) { AnomalyStatus highStatus = state.getStatus(3); processThresholdAnomaly(highStatus, currentValue, timeStr, currentValue > config.highthreshold, config, 3, out); } if (config.islow == 1) { AnomalyStatus lowStatus = state.getStatus(4); processThresholdAnomaly(lowStatus, currentValue, timeStr, currentValue < config.lowthreshold, config, 4, out); } } catch (Exception e) { System.err.println("阈值检测错误: " + config.encode + " - " + e.getMessage()); } } private void processThresholdAnomaly(AnomalyStatus status, double currentValue, String timeStr, boolean isAnomaly, ULEParamConfig config, int anomalyType, Collector<JSONObject> out) { try { Date timestamp = timeFormat.parse(timeStr); if (isAnomaly) { if (status.startTime == null) { status.startTime = timestamp; } else if (!status.reported) { long elapsed = timestamp.getTime() - status.startTime.getTime(); if (elapsed >= config.duration * 60 * 1000) { reportAnomaly(anomalyType, 1, currentValue, timeStr, config, out); status.reported = true; } } } else { if (status.reported) { reportAnomaly(anomalyType, 0, currentValue, timeStr, config, out); status.reset(); } else if (status.startTime != null) { status.startTime = null; } } } catch (Exception e) { System.err.println("阈值处理错误: " + config.encode + " - " + e.getMessage()); } } // 同步检测 private void checkSyncAnomaly(ULEParamConfig config, double currentValue, String timeStr, AnomalyState state, ConfigCollection configCollection, Collector<JSONObject> out) { if (config.issync != 1 || config.syncparaencode == null) return; try { AnomalyStatus status = state.getStatus(6); Date timestamp = timeFormat.parse(timeStr); ULEParamConfig relatedConfig = configCollection.encodeToConfig.get(config.syncparaencode); if (relatedConfig == null) return; String relatedTag = null; for (Map.Entry<String, List<ULEParamConfig>> entry : configCollection.tagToConfigs.entrySet()) { if (entry.getValue().contains(relatedConfig)) { relatedTag = entry.getKey(); break; } } if (relatedTag == null) return; Double relatedValue = lastValuesMap.get(relatedTag); if (relatedValue == null) return; boolean isAnomaly = (Math.abs(currentValue - 1.0) < 0.001) && (Math.abs(relatedValue) < 0.001); if (isAnomaly) { if (status.startTime == null) { status.startTime = timestamp; } else if (!status.reported) { long elapsed = timestamp.getTime() - status.startTime.getTime(); if (elapsed >= config.duration * 60 * 1000) { reportAnomaly(6, 1, currentValue, timeStr, config, out); status.reported = true; } } } else { if (status.reported) { reportAnomaly(6, 0, currentValue, timeStr, config, out); status.reset(); } else if (status.startTime != null) { status.startTime = null; } } } catch (Exception e) { System.err.println("同步检测错误: " + config.encode + " - " + e.getMessage()); } } // 报告异常 private void reportAnomaly(int anomalyType, int statusFlag, double value, String time, ULEParamConfig config, Collector<JSONObject> out) { JSONObject event = new JSONObject(); event.put("tag", config.tag); event.put("paracode", config.encode); event.put("abnormaltype", anomalyType); event.put("statusflag", statusFlag); event.put("datavalue", value); event.put("triggertime", time); out.collect(event); } @Override public void processBroadcastElement(ConfigCollection newConfig, Context ctx, Collector<JSONObject> out) { BroadcastState<Void, ConfigCollection> state = ctx.getBroadcastState(Descriptors.configStateDescriptor); try { state.put(null, newConfig); // 仅更新广播状态 } catch (Exception e) { e.printStackTrace(); } System.out.println("广播配置更新完成, 配置项: " + newConfig.configCount()); } // ========== 辅助方法 ========== private ConfigCollection getBroadcastConfig(ReadOnlyContext ctx) throws Exception { return ctx.getBroadcastState(Descriptors.configStateDescriptor).get(null); } private AnomalyState getOrCreateState(String encode) throws Exception { AnomalyState state = stateMap.get(encode); if (state == null) { state = new AnomalyState(); } return state; } } // 异常状态类 public static class AnomalyState implements Serializable { private static final long serialVersionUID = 1L; private final Map<Integer, AnomalyStatus> statusMap = new HashMap<>(); public AnomalyStatus getStatus(int type) { return statusMap.computeIfAbsent(type, k -> new AnomalyStatus()); } } // 异常状态详情 public static class AnomalyStatus implements Serializable { private static final long serialVersionUID = 1L; public Date startTime; // 异常开始时间 public Double lastValue; // 用于恒值检测 public Date lastChangeTime; // 值最后变化时间 public boolean reported; // 是否已报告 public void reset() { startTime = null; lastValue = null; lastChangeTime = null; reported = false; } } }上述代码需完善内容,1、离线监测未实现,当配置中有条的tag更改,且流数据中没有该tag,未报出离线异常;2、当参数调整时,已有的报警状态参数,配置成启用相应的报警如何处理;3、当参数配置删除时,已有的报警状态参数处理。,请完善上述内容,生成完整优化后可运行代码。考虑通过 ‘Context’ 中的方法 ‘timerService’和‘ReadOnlyContext’ 中的方法 ‘getTimerService’方法实现
最新发布
08-02
package com.tongchuang.realtime.mds; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.tongchuang.realtime.bean.ULEParamConfig; import com.tongchuang.realtime.util.KafkaUtils; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.common.state.; import org.apache.flink.api.common.time.Time; import org.apache.flink.api.common.typeinfo.BasicTypeInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.configuration.Configuration; import org.apache.flink.connector.kafka.source.KafkaSource; import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer; import org.apache.flink.streaming.api.datastream.; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.co.KeyedBroadcastProcessFunction; import org.apache.flink.streaming.api.functions.source.RichSourceFunction; import org.apache.flink.util.Collector; import java.io.Serializable; import java.sql.; import java.text.SimpleDateFormat; import java.util.; import java.util.Date; import java.util.concurrent.TimeUnit; public class ULEDataanomalyanalysis { public static void main(String[] args) throws Exception { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); // 创建Kafka消费者 KafkaSource<String> kafkaConsumer = KafkaUtils.getKafkaConsumer( "realdata_minute", "minutedata_uledataanomalyanalysis", OffsetsInitializer.latest() ); DataStreamSource<String> kafkaDS = env.fromSource( kafkaConsumer, WatermarkStrategy.noWatermarks(), "realdata_uledataanomalyanalysis" ); // 解析JSON拆分每个tag的数据 SingleOutputStreamOperator<JSONObject> splitStream = kafkaDS .map(JSON::parseObject) .flatMap((JSONObject value, Collector<JSONObject> out) -> { JSONObject data = value.getJSONObject("datas"); String time = value.getString("times"); for (String tag : data.keySet()) { JSONObject tagData = data.getJSONObject(tag); JSONObject newObj = new JSONObject(); newObj.put("time", time); newObj.put("tag", tag); newObj.put("ontime", tagData.getDouble("ontime")); newObj.put("avg", tagData.getDouble("avg")); out.collect(newObj); } }) .returns(TypeInformation.of(JSONObject.class)); // 每5分钟加载参数配置 DataStream<ConfigCollection> configDataStream = env .addSource(new MysqlConfigSource()) .setParallelism(1) .filter(Objects::nonNull); // 将配置流转换为广播流 BroadcastStream<ConfigCollection> configBroadcastStream = configDataStream .broadcast(Descriptors.configStateDescriptor); // 按tag分组连接广播流 KeyedStream<JSONObject, String> keyedStream = splitStream .keyBy(json -> json.getString("tag")); BroadcastConnectedStream<JSONObject, ConfigCollection> connectedStream = keyedStream.connect(configBroadcastStream); // 异常检测处理 SingleOutputStreamOperator<JSONObject> anomalyStream = connectedStream .process(new OptimizedAnomalyDetectionFunction()); anomalyStream.print("异常检测结果"); // anomalyStream.map(JSON::toString).addSink(KafkaUtils.getKafkaSink(“minutedata_uleanomaly”)); env.execute(“uledataanomalyanalysis”); } // 配置集合类 public static class ConfigCollection implements Serializable { public final Map<String, List<ULEParamConfig>> tagToConfigs; public final Map<String, ULEParamConfig> encodeToConfig; public ConfigCollection(Map<String, List<ULEParamConfig>> tagToConfigs, Map<String, ULEParamConfig> encodeToConfig) { this.tagToConfigs = tagToConfigs; this.encodeToConfig = encodeToConfig; } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; ConfigCollection that = (ConfigCollection) o; return Objects.equals(encodeToConfig.keySet(), that.encodeToConfig.keySet()); } @Override public int hashCode() { return Objects.hash(encodeToConfig.keySet()); } } // MySQL配置源 public static class MysqlConfigSource extends RichSourceFunction<ConfigCollection> { private volatile boolean isRunning = true; private final long interval = TimeUnit.MINUTES.toMillis(5); private transient ConfigCollection lastConfig; @Override public void run(SourceContext<ConfigCollection> ctx) throws Exception { while (isRunning) { ConfigCollection newConfig = loadParams(); if (lastConfig == null || !newConfig.equals(lastConfig)) { ctx.collect(newConfig); lastConfig = newConfig; System.out.println("检测到配置变化,广播新配置"); } else { System.out.println("配置未变化,跳过广播"); } Thread.sleep(interval); } } private ConfigCollection loadParams() { Map<String, List<ULEParamConfig>> tagToConfigs = new HashMap<>(); Map<String, ULEParamConfig> encodeToConfig = new HashMap<>(); String url = "jdbc:mysql://10.51.37.73:3306/eps?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8"; String user = "root"; String password = "6CKIm5jDVsLrahSw"; String query = "SELECT F_tag AS tag, F_enCode AS encode, F_dataTypes AS datatype, " + "F_isConstantValue AS constantvalue, F_isOnline AS isonline, " + "F_isSync AS issync, F_syncParaEnCode AS syncparaencode, " + "F_isZero AS iszero, F_isHigh AS ishigh, F_highThreshold AS highthreshold, " + "F_isLow AS islow, F_lowThreshold AS lowthreshold, F_duration AS duration " + "FROM t_equipmentparameter " + "WHERE F_enabledmark = '1' AND (F_isConstantValue ='1' OR F_isZero= '1' " + "OR F_isHigh = '1' OR F_isLow = '1' OR F_isOnline = '1' OR F_isSync = '1')"; try (Connection conn = DriverManager.getConnection(url, user, password); Statement stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery(query)) { while (rs.next()) { ULEParamConfig config = new ULEParamConfig(); config.tag = rs.getString("tag"); config.encode = rs.getString("encode"); config.datatype = rs.getString("datatype"); config.constantvalue = rs.getInt("constantvalue"); config.iszero = rs.getInt("iszero"); config.ishigh = rs.getInt("ishigh"); config.highthreshold = rs.getDouble("highthreshold"); config.islow = rs.getInt("islow"); config.lowthreshold = rs.getDouble("lowthreshold"); config.duration = rs.getLong("duration"); config.isonline = rs.getInt("isonline"); config.issync = rs.getInt("issync"); config.syncparaencode = rs.getString("syncparaencode"); String tag = config.tag; tagToConfigs.computeIfAbsent(tag, k -> new ArrayList<>()).add(config); encodeToConfig.put(config.encode, config); } System.out.println("加载配置: " + encodeToConfig.size() + " 个参数"); } catch (SQLException e) { System.err.println("加载参数配置错误:"); e.printStackTrace(); return null; } return new ConfigCollection(tagToConfigs, encodeToConfig); } @Override public void cancel() { isRunning = false; } } // 状态描述符 public static class Descriptors { public static final MapStateDescriptor<Void, ConfigCollection> configStateDescriptor = new MapStateDescriptor<>( "configState", TypeInformation.of(Void.class), TypeInformation.of(ConfigCollection.class) ); } // 优化后的异常检测函数(带状态TTL和离线检测优化) public static class OptimizedAnomalyDetectionFunction extends KeyedBroadcastProcessFunction<String, JSONObject, ConfigCollection, JSONObject> { // 状态管理 private transient MapState<String, AnomalyState> stateMap; // key=encode private transient MapState<String, Double> lastValuesMap; // key=tag private transient MapState<String, Long> lastDataTimeMap; // key=tag private transient MapState<String, Long> offlineTimerState; // key=encode private transient SimpleDateFormat timeFormat; @Override public void open(Configuration parameters) { // 状态TTL配置(30天自动清理) StateTtlConfig ttlConfig = StateTtlConfig.newBuilder(Time.days(30)) .setUpdateType(StateTtlConfig.UpdateType.OnCreateAndWrite) .setStateVisibility(StateTtlConfig.StateVisibility.NeverReturnExpired) .build(); // 初始化异常状态存储(启用TTL) MapStateDescriptor<String, AnomalyState> stateDesc = new MapStateDescriptor<>( "anomalyState", BasicTypeInfo.STRING_TYPE_INFO, TypeInformation.of(AnomalyState.class) ); stateDesc.enableTimeToLive(ttlConfig); stateMap = getRuntimeContext().getMapState(stateDesc); // 初始化最新值存储(启用TTL) MapStateDescriptor<String, Double> valuesDesc = new MapStateDescriptor<>( "lastValuesState", BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.DOUBLE_TYPE_INFO ); valuesDesc.enableTimeToLive(ttlConfig); lastValuesMap = getRuntimeContext().getMapState(valuesDesc); // 初始化最后数据时间存储(启用TTL) MapStateDescriptor<String, Long> timeDesc = new MapStateDescriptor<>( "lastDataTimeState", BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO ); timeDesc.enableTimeToLive(ttlConfig); lastDataTimeMap = getRuntimeContext().getMapState(timeDesc); // 初始化离线定时器状态(启用TTL) MapStateDescriptor<String, Long> timerDesc = new MapStateDescriptor<>( "offlineTimerState", BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO ); timerDesc.enableTimeToLive(ttlConfig); offlineTimerState = getRuntimeContext().getMapState(timerDesc); timeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm"); } @Override public void processElement(JSONObject data, ReadOnlyContext ctx, Collector<JSONObject> out) throws Exception { String tag = ctx.getCurrentKey(); String timeStr = data.getString("time"); long eventTime = timeFormat.parse(timeStr).getTime(); // 更新最后数据时间 lastDataTimeMap.put(tag, eventTime); // 获取广播配置 ConfigCollection configCollection = getBroadcastConfig(ctx); if (configCollection == null) { return; } List<ULEParamConfig> configs = configCollection.tagToConfigs.get(tag); if (configs == null || configs.isEmpty()) { return; } // ========== 清理无效状态 ========== Iterator<String> stateKeys = stateMap.keys().iterator(); while (stateKeys.hasNext()) { String encode = stateKeys.next(); boolean found = false; for (ULEParamConfig cfg : configs) { if (cfg.encode.equals(encode)) { found = true; break; } } if (!found) { stateMap.remove(encode); // 清理相关定时器 Long timer = offlineTimerState.get(encode); if (timer != null) { ctx.timerService().deleteEventTimeTimer(timer); offlineTimerState.remove(encode); } } } double value = 0; boolean valueSet = false; // 遍历配置项进行异常检测 for (ULEParamConfig config : configs) { if (!valueSet) { value = "436887485805570949".equals(config.datatype) ? data.getDouble("ontime") : data.getDouble("avg"); lastValuesMap.put(tag, value); valueSet = true; } // 获取或初始化状态 AnomalyState state = getOrCreateState(config.encode); // ========== 离线恢复检测 ========== if (config.isonline == 1) { AnomalyStatus status = state.getStatus(5); if (status.reported) { // 设备重新上线,发送恢复事件 reportAnomaly(5, 0, 0.0, timeStr, config, out); status.reset(); System.out.println("设备 " + tag + " 恢复在线,发送恢复事件"); } } // 处理异常类型 checkConstantValueAnomaly(config, value, timeStr, state, out); checkZeroValueAnomaly(config, value, timeStr, state, out); checkThresholdAnomaly(config, value, timeStr, state, out); checkSyncAnomaly(config, value, timeStr, state, configCollection, out); // 设置离线检测定时器(仅当配置开启离线检测时) if (config.isonline == 1) { setupOfflineTimer(tag, config, eventTime, ctx); } // 保存状态 stateMap.put(config.encode, state); } } // 设置离线检测定时器 private void setupOfflineTimer(String tag, ULEParamConfig config, long eventTime, ReadOnlyContext ctx) throws Exception { long durationMillis = config.duration * 60 * 1000; long timerTimestamp = eventTime + durationMillis; String timerKey = config.encode; // 使用encode作为唯一标识 // 清除旧定时器(如果存在) Long oldTimer = offlineTimerState.get(timerKey); if (oldTimer != null) { ctx.timerService().deleteEventTimeTimer(oldTimer); } // 注册新定时器(事件时间) ctx.timerService().registerEventTimeTimer(timerTimestamp); offlineTimerState.put(timerKey, timerTimestamp); // System.out.println("为标签 " + tag + " 设置离线检测定时器: " + timeFormat.format(new Date(timerTimestamp))); } @Override public void onTimer(long timestamp, OnTimerContext ctx, Collector<JSONObject> out) throws Exception { String tag = ctx.getCurrentKey(); System.out.println("离线检测定时器触发: " + tag + " at " + timeFormat.format(new Date(timestamp))); // 获取广播配置 ConfigCollection configCollection = getBroadcastConfig(ctx); if (configCollection == null) return; List<ULEParamConfig> configs = configCollection.tagToConfigs.get(tag); if (configs == null) return; // 找出匹配的定时器 List<ULEParamConfig> triggeredConfigs = new ArrayList<>(); for (ULEParamConfig config : configs) { if (config.isonline == 1) { Long storedTimer = offlineTimerState.get(config.encode); if (storedTimer != null && storedTimer == timestamp) { triggeredConfigs.add(config); } } } // 处理所有触发的配置项 for (ULEParamConfig config : triggeredConfigs) { Long lastEventTime = lastDataTimeMap.get(tag); AnomalyState state = getOrCreateState(config.encode); AnomalyStatus status = state.getStatus(5); // 5=离线类型 if (lastEventTime == null || lastEventTime < timestamp - config.duration * 60 * 1000) { // 触发离线异常 if (!status.reported) { reportAnomaly(5, 1, 0.0, timeFormat.format(new Date(timestamp)), config, out); status.reported = true; System.out.println("设备 " + tag + " 离线报告"); } } else { // 取消误报 if (status.reported) { reportAnomaly(5, 0, 0.0, timeFormat.format(new Date()), config, out); status.reported = false; System.out.println("设备 " + tag + " 离线恢复"); } } // 重置定时器 offlineTimerState.remove(config.encode); } } // 恒值检测 private void checkConstantValueAnomaly(ULEParamConfig config, double currentValue, String timeStr, AnomalyState state, Collector<JSONObject> out) { if (config.constantvalue != 1) return; try { AnomalyStatus status = state.getStatus(1); long durationThreshold = config.duration * 60 * 1000; Date timestamp = timeFormat.parse(timeStr); if (status.lastValue == null) { status.lastValue = currentValue; status.lastChangeTime = timestamp; return; } if (Math.abs(currentValue - status.lastValue) > 0.001) { status.lastValue = currentValue; status.lastChangeTime = timestamp; if (status.reported) { reportAnomaly(1, 0, currentValue, timeStr, config, out); } status.reset(); return; } long elapsed = timestamp.getTime() - status.lastChangeTime.getTime(); if (elapsed > durationThreshold) { if (!status.reported) { reportAnomaly(1, 1, currentValue, timeStr, config, out); status.reported = true; } } } catch (Exception e) { System.err.println("恒值检测错误: " + config.encode + " - " + e.getMessage()); } } // 零值检测 private void checkZeroValueAnomaly(ULEParamConfig config, double currentValue, String timeStr, AnomalyState state, Collector<JSONObject> out) { if (config.iszero != 1) return; try { AnomalyStatus status = state.getStatus(2); Date timestamp = timeFormat.parse(timeStr); boolean isZero = Math.abs(currentValue) < 0.001; if (isZero) { if (status.startTime == null) { status.startTime = timestamp; } else if (!status.reported) { long elapsed = timestamp.getTime() - status.startTime.getTime(); if (elapsed >= config.duration * 60 * 1000) { reportAnomaly(2, 1, currentValue, timeStr, config, out); status.reported = true; } } } else { if (status.reported) { reportAnomaly(2, 0, currentValue, timeStr, config, out); status.reset(); } else if (status.startTime != null) { status.startTime = null; } } } catch (Exception e) { System.err.println("零值检测错误: " + config.encode + " - " + e.getMessage()); } } // 阈值检测 private void checkThresholdAnomaly(ULEParamConfig config, double currentValue, String timeStr, AnomalyState state, Collector<JSONObject> out) { try { if (config.ishigh == 1) { AnomalyStatus highStatus = state.getStatus(3); processThresholdAnomaly(highStatus, currentValue, timeStr, currentValue > config.highthreshold, config, 3, out); } if (config.islow == 1) { AnomalyStatus lowStatus = state.getStatus(4); processThresholdAnomaly(lowStatus, currentValue, timeStr, currentValue < config.lowthreshold, config, 4, out); } } catch (Exception e) { System.err.println("阈值检测错误: " + config.encode + " - " + e.getMessage()); } } private void processThresholdAnomaly(AnomalyStatus status, double currentValue, String timeStr, boolean isAnomaly, ULEParamConfig config, int anomalyType, Collector<JSONObject> out) { try { Date timestamp = timeFormat.parse(timeStr); if (isAnomaly) { if (status.startTime == null) { status.startTime = timestamp; } else if (!status.reported) { long elapsed = timestamp.getTime() - status.startTime.getTime(); if (elapsed >= config.duration * 60 * 1000) { reportAnomaly(anomalyType, 1, currentValue, timeStr, config, out); status.reported = true; } } } else { if (status.reported) { reportAnomaly(anomalyType, 0, currentValue, timeStr, config, out); status.reset(); } else if (status.startTime != null) { status.startTime = null; } } } catch (Exception e) { System.err.println("阈值处理错误: " + config.encode + " - " + e.getMessage()); } } // 同步检测 private void checkSyncAnomaly(ULEParamConfig config, double currentValue, String timeStr, AnomalyState state, ConfigCollection configCollection, Collector<JSONObject> out) { if (config.issync != 1 || config.syncparaencode == null) return; try { AnomalyStatus status = state.getStatus(6); Date timestamp = timeFormat.parse(timeStr); ULEParamConfig relatedConfig = configCollection.encodeToConfig.get(config.syncparaencode); if (relatedConfig == null) return; String relatedTag = null; for (Map.Entry<String, List<ULEParamConfig>> entry : configCollection.tagToConfigs.entrySet()) { if (entry.getValue().contains(relatedConfig)) { relatedTag = entry.getKey(); break; } } if (relatedTag == null) return; Double relatedValue = lastValuesMap.get(relatedTag); if (relatedValue == null) return; boolean isAnomaly = (Math.abs(currentValue - 1.0) < 0.001) && (Math.abs(relatedValue) < 0.001); if (isAnomaly) { if (status.startTime == null) { status.startTime = timestamp; } else if (!status.reported) { long elapsed = timestamp.getTime() - status.startTime.getTime(); if (elapsed >= config.duration * 60 * 1000) { reportAnomaly(6, 1, currentValue, timeStr, config, out); status.reported = true; } } } else { if (status.reported) { reportAnomaly(6, 0, currentValue, timeStr, config, out); status.reset(); } else if (status.startTime != null) { status.startTime = null; } } } catch (Exception e) { System.err.println("同步检测错误: " + config.encode + " - " + e.getMessage()); } } // 报告异常 private void reportAnomaly(int anomalyType, int statusFlag, double value, String time, ULEParamConfig config, Collector<JSONObject> out) { JSONObject event = new JSONObject(); event.put("tag", config.tag); event.put("paracode", config.encode); event.put("abnormaltype", anomalyType); event.put("statusflag", statusFlag); event.put("datavalue", value); event.put("triggertime", time); out.collect(event); // System.out.println("报告异常: " + event.toJSONString()); } @Override public void processBroadcastElement(ConfigCollection newConfig, Context ctx, Collector<JSONObject> out) { BroadcastState<Void, ConfigCollection> state = ctx.getBroadcastState(Descriptors.configStateDescriptor); try { state.put(null, newConfig); // 仅更新广播状态 } catch (Exception e) { e.printStackTrace(); } System.out.println("广播配置更新完成, 配置项: " + newConfig.encodeToConfig.size()); } // ========== 辅助方法 ========== private ConfigCollection getBroadcastConfig(ReadOnlyContext ctx) throws Exception { return ctx.getBroadcastState(Descriptors.configStateDescriptor).get(null); } private AnomalyState getOrCreateState(String encode) throws Exception { AnomalyState state = stateMap.get(encode); if (state == null) { state = new AnomalyState(); stateMap.put(encode, state); } return state; } } // 异常状态类 public static class AnomalyState implements Serializable { private final Map<Integer, AnomalyStatus> statusMap = new HashMap<>(); public AnomalyStatus getStatus(int type) { return statusMap.computeIfAbsent(type, k -> new AnomalyStatus()); } } // 异常状态详情 public static class AnomalyStatus implements Serializable { public Date startTime; // 异常开始时间 public Double lastValue; // 用于恒值检测 public Date lastChangeTime; // 值最后变化时间 public boolean reported; // 是否已报告 public void reset() { startTime = null; lastValue = null; lastChangeTime = null; reported = false; } } } 优化上述代码,参数更新时全量更新广播,参数在5000条左右,生成完整优化代码,条数可能变但具体配置数值变化
08-01
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值