package com.tongchuang.realtime.mds;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.tongchuang.realtime.bean.ULEParamConfig;
import com.tongchuang.realtime.util.KafkaUtils;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.state.*;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.common.typeinfo.TypeHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.connector.kafka.source.KafkaSource;
import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer;
import org.apache.flink.streaming.api.datastream.*;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.co.KeyedBroadcastProcessFunction;
import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
import org.apache.flink.util.Collector;
import java.sql.*;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.Date;
import java.util.concurrent.TimeUnit;
public class ULEDataanomalyanalysis {
public static void main(String[] args) throws Exception {
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
KafkaSource<String> kafkaConsumer = KafkaUtils.getKafkaConsumer("realdata_minute", "minutedata_uledataanomalyanalysis", OffsetsInitializer.latest());
DataStreamSource<String> kafkaDS = env.fromSource(kafkaConsumer, WatermarkStrategy.noWatermarks(), "realdata_uledataanomalyanalysis");
// 解析JSON并拆分每个tag的数据
SingleOutputStreamOperator<JSONObject> splitStream = kafkaDS.map(JSON::parseObject)
.flatMap((JSONObject value, Collector<JSONObject> out) -> {
JSONObject data = value.getJSONObject("datas");
String time = value.getString("times");
for (String tag : data.keySet()) {
JSONObject tagData = data.getJSONObject(tag);
JSONObject newObj = new JSONObject();
newObj.put("time", time);
newObj.put("tag", tag);
newObj.put("ontime", tagData.getDouble("ontime"));
newObj.put("avg", tagData.getDouble("avg"));
out.collect(newObj);
}
})
.returns(TypeInformation.of(JSONObject.class));
// 每5分钟加载参数配置 - 创建DataStream
DataStream<Map<String, ULEParamConfig>> configDataStream = env
.addSource(new MysqlConfigSource())
.setParallelism(1);
// 将DataStream转换为BroadcastStream
BroadcastStream<Map<String, ULEParamConfig>> configBroadcastStream =
configDataStream.broadcast(Descriptors.configStateDescriptor);
// 按tag分组并连接广播流
KeyedStream<JSONObject, String> keyedStream = splitStream
.keyBy(json -> json.getString("tag"));
BroadcastConnectedStream<JSONObject, Map<String, ULEParamConfig>> connectedStream =
keyedStream.connect(configBroadcastStream);
// 异常检测处理
SingleOutputStreamOperator<JSONObject> anomalyStream = connectedStream
.process(new AnomalyDetectionFunction());
anomalyStream.print("异常检测结果");
env.execute("uledataanomalyanalysis");
}
// MySQL配置源
public static class MysqlConfigSource extends RichSourceFunction<Map<String, ULEParamConfig>> {
private volatile boolean isRunning = true;
private final long interval = TimeUnit.MINUTES.toMillis(5);
@Override
public void run(SourceContext<Map<String, ULEParamConfig>> ctx) throws Exception {
while (isRunning) {
ctx.collect(loadParams());
Thread.sleep(interval);
}
}
private Map<String, ULEParamConfig> loadParams() {
Map<String, ULEParamConfig> configMap = new HashMap<>();
String url = "jdbc:mysql://10.51.37.73:3306/eps?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8";
String user = "root";
String password = "6CKIm5jDVsLrahSw";
String query = "select F_tag tag , F_enCode encode, F_dataTypes datatype,F_isConstantValue constantvalue,F_isOnline isonline,F_isSync issync,F_syncParaEncode syncparaencode,\n" +
"F_runTag runtag,F_isZero iszero,F_isHigh ishigh,F_highThreshold highthreshold,F_isLow islow,F_lowThreshold lowthreshold, F_duration duration \n" +
"from t_equipmentparameter\n" +
"where F_enabledmark = '1' and (F_isConstantValue ='1' or F_isZero= '1' or F_isHigh = '1' or F_isLow = '1' or F_isOnline = '1' or F_isSync = '1' )";
try (Connection conn = DriverManager.getConnection(url, user, password);
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(query)) {
while (rs.next()) {
ULEParamConfig config = new ULEParamConfig();
config.encode = rs.getString("encode");
config.datatype = rs.getString("datatype");
config.constantvalue = rs.getInt("constantvalue");
config.iszero = rs.getInt("iszero");
config.ishigh = rs.getInt("ishigh");
config.highthreshold = rs.getDouble("highthreshold");
config.islow = rs.getInt("islow");
config.lowthreshold = rs.getDouble("lowthreshold");
config.duration = rs.getLong("duration");
// 使用runtag优先,没有则用tag
String runtag = rs.getString("runtag");
String tag = rs.getString("tag");
String key = (runtag != null && !runtag.isEmpty()) ? runtag : tag;
configMap.put(key, config);
}
System.out.println("Loaded " + configMap.size() + " parameter configurations");
} catch (SQLException e) {
System.err.println("Error loading parameters from MySQL:");
e.printStackTrace();
}
return configMap;
}
@Override
public void cancel() {
isRunning = false;
}
}
// 状态描述符
public static class Descriptors {
public static final MapStateDescriptor<Void, Map<String, ULEParamConfig>> configStateDescriptor =
new MapStateDescriptor<>(
"configState",
TypeInformation.of(Void.class),
TypeInformation.of(new TypeHint<Map<String, ULEParamConfig>>() {})
);
}
// 异常检测函数
public static class AnomalyDetectionFunction
extends KeyedBroadcastProcessFunction<String, JSONObject, Map<String, ULEParamConfig>, JSONObject> {
private transient MapState<String, AnomalyState> stateMap;
private transient SimpleDateFormat timeFormat;
@Override
public void open(Configuration parameters) {
MapStateDescriptor<String, AnomalyState> stateDesc = new MapStateDescriptor<>(
"anomalyState",
BasicTypeInfo.STRING_TYPE_INFO,
TypeInformation.of(AnomalyState.class)
);
stateMap = getRuntimeContext().getMapState(stateDesc);
timeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm");
}
@Override
public void processElement(JSONObject data, ReadOnlyContext ctx, Collector<JSONObject> out) throws Exception {
String tag = ctx.getCurrentKey();
String timeStr = data.getString("time");
// 获取广播配置
ReadOnlyBroadcastState<Void, Map<String, ULEParamConfig>> broadcastState =
ctx.getBroadcastState(Descriptors.configStateDescriptor);
Map<String, ULEParamConfig> configMap = broadcastState.get(null);
if (configMap == null) {
System.out.println("No config available for tag: " + tag);
return;
}
ULEParamConfig config = configMap.get(tag);
if (config == null) {
// 没有该tag的配置
return;
}
// 获取当前值
double value = "436887485805570949".equals(config.datatype) ?
data.getDouble("ontime") : data.getDouble("avg");
// 获取或初始化状态
AnomalyState state = stateMap.get(tag);
if (state == null) {
state = new AnomalyState();
}
// 处理异常类型
checkAndReportAnomaly(config.constantvalue, 1, value, timeStr, config, state, tag, out);
checkAndReportAnomaly(config.iszero, 2, value, timeStr, config, state, tag, out);
checkAndReportAnomaly(config.ishigh, 3, value, timeStr, config, state, tag, out);
checkAndReportAnomaly(config.islow, 4, value, timeStr, config, state, tag, out);
// 保存状态
stateMap.put(tag, state);
}
private void checkAndReportAnomaly(int enabled, int anomalyType, double currentValue,
String timeStr, ULEParamConfig config, AnomalyState state,
String tag, Collector<JSONObject> out) {
if (enabled != 1) return;
try {
AnomalyStatus status = state.getStatus(anomalyType);
long durationThreshold = config.duration * 60 * 1000; // 分钟转毫秒
Date timestamp = timeFormat.parse(timeStr);
// 检查异常条件
boolean isAnomaly = false;
switch (anomalyType) {
case 1: // 恒值检测
if (status.lastValue == null) {
status.lastValue = currentValue;
status.lastChangeTime = timestamp;
} else if (Math.abs(currentValue - status.lastValue) > 0.0) {
// 值发生变化
status.lastValue = currentValue;
status.lastChangeTime = timestamp;
}
isAnomaly = (timestamp.getTime() - status.lastChangeTime.getTime()) > durationThreshold;
break;
case 2: // 零值
isAnomaly = Math.abs(currentValue) == 0.0; // 接近0
break;
case 3: // 高阈值
isAnomaly = currentValue > config.highthreshold;
break;
case 4: // 低阈值
isAnomaly = currentValue < config.lowthreshold;
break;
}
// 状态转换处理
if (isAnomaly) {
if (status.startTime == null) {
status.startTime = timestamp;
System.out.println("[" + tag + "] Anomaly started at " + timeStr);
} else if (!status.reported &&
(timestamp.getTime() - status.startTime.getTime()) >= durationThreshold) {
// 达到持续时长,报告异常
reportAnomaly(anomalyType, 1, currentValue, timeStr, config, out);
status.reported = true;
System.out.println("[" + tag + "] Reported anomaly type " + anomalyType);
}
} else if (status.startTime != null) { // 异常恢复
if (status.reported) {
reportAnomaly(anomalyType, 0, currentValue, timeStr, config, out);
System.out.println("[" + tag + "] Anomaly recovered");
}
status.reset();
}
} catch (Exception e) {
System.err.println("Error processing anomaly for tag " + tag + ": " + e.getMessage());
e.printStackTrace();
}
}
private void reportAnomaly(int anomalyType, int statusFlag, double value,
String time, ULEParamConfig config, Collector<JSONObject> out) {
JSONObject event = new JSONObject();
event.put("paracode", config.encode);
event.put("abnormaltype", anomalyType);
event.put("statusflag", statusFlag);
event.put("datavalue", value);
event.put("triggertime", time);
out.collect(event);
}
@Override
public void processBroadcastElement(Map<String, ULEParamConfig> newConfig, Context ctx, Collector<JSONObject> out) {
BroadcastState<Void, Map<String, ULEParamConfig>> state =
ctx.getBroadcastState(Descriptors.configStateDescriptor);
try {
state.put(null, newConfig);
System.out.println("Updated configuration: " + newConfig.size() + " parameters");
} catch (Exception e) {
System.err.println("Failed to update broadcast state:");
e.printStackTrace();
}
}
}
// 异常状态类
public static class AnomalyState {
private final Map<Integer, AnomalyStatus> statusMap = new HashMap<>();
public AnomalyStatus getStatus(int type) {
return statusMap.computeIfAbsent(type, k -> new AnomalyStatus());
}
}
// 异常状态详情
public static class AnomalyStatus {
public Date startTime; // 异常开始时间
public Double lastValue; // 用于恒值检测
public Date lastChangeTime; // 值最后变化时间
public boolean reported; // 是否已报告
public void reset() {
startTime = null;
reported = false;
// 保留lastValue和lastChangeTime用于恒值检测
}
}
}在上述代码基础上完善,需要每5分钟从数据库加载参数配置currentParams,在分钟流数据获取时,先赛选保留参数配置中所有的tag或runtag值,然后根据每条配置,对每分钟的数据进行恒值(数值不变化)、0值(数值为0.0)、上下阈值(超过上阈值、低于下阈值)、离线(没有改tag的分钟数据)、同步(该点位值为1.0,但根据该点位对应的syncparaencode查找encode对应的tag值为0.0)异常分析,当出现异常,且异常持续时间超过配置的duration值时,生成异常消息,异常恢复时也生成消息,发送kafka,格式为paracode、abnormaltype、statusflag、datavalue、triggertime。具体数据取值根据配置数据类型datatype为436887485805570949的,取ontime,其他取avg
最新发布