/*
* Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved.
*/
package com.huawei.it.gaia.apigc.service.task;
import static com.huawei.it.gaia.apigc.domain.utils.DateUtils.DATE_FORMAT_COMPACT;
import static com.huawei.it.gaia.apigc.domain.utils.DateUtils.DATE_FORMAT_FULL;
import static com.huawei.it.gaia.apigc.domain.utils.DateUtils.DATE_FORMAT_SHORT;
import static com.huawei.it.gaia.apigc.domain.utils.DateUtils.formatDate;
import static com.huawei.it.gaia.apigc.service.common.consts.TableNameConstants.APIGC_TOPO_MICRO_REL;
import static com.huawei.it.gaia.apigc.service.common.consts.TableNameConstants.TBL_CLOUD_MONITOR_RECORD;
import static com.huawei.it.gaia.apigc.service.common.consts.TableOwnerConstants.DU_BO_BO;
import static com.huawei.it.gaia.apigc.service.common.consts.TaskConstants.API_RUNNING_SOURCE_MONITOR;
import static com.huawei.it.gaia.apigc.service.common.consts.TaskConstants.SCHEDULE_TASK_EXECUTOR;
import static com.huawei.it.gaia.apigc.service.common.consts.TaskConstants.SCHEDULE_TASK_EXECUTOR_KEY;
import com.huawei.it.gaia.apigc.client.apigc.monitor.MonitorSearchClient;
import com.huawei.it.gaia.apigc.client.apigc.monitor.ro.MonitorSearchRO;
import com.huawei.it.gaia.apigc.client.apigc.samdata.microsvcrel.MicroSvcClient;
import com.huawei.it.gaia.apigc.client.apigc.samdata.microsvcrel.ro.MicroSvcRelRO;
import com.huawei.it.gaia.apigc.domain.api.gateway.IApiAssetsGateway;
import com.huawei.it.gaia.apigc.domain.api.model.ApiAssetsModel;
import com.huawei.it.gaia.apigc.domain.common.exception.ApplicationException;
import com.huawei.it.gaia.apigc.domain.monitor.gateway.IApiLogMatchRelGateway;
import com.huawei.it.gaia.apigc.domain.monitor.gateway.ICloudMonitorRecordGateway;
import com.huawei.it.gaia.apigc.domain.monitor.gateway.IGaiaMonitorGateway;
import com.huawei.it.gaia.apigc.domain.monitor.gateway.IGaiaMonitorTransferGateway;
import com.huawei.it.gaia.apigc.domain.monitor.gateway.IMicroAppRelGateway;
import com.huawei.it.gaia.apigc.domain.monitor.gateway.IMicroSvcRelGateway;
import com.huawei.it.gaia.apigc.domain.monitor.model.ApiLogMatchRelEntity;
import com.huawei.it.gaia.apigc.domain.monitor.model.GaiaMonitorEntity;
import com.huawei.it.gaia.apigc.domain.monitor.model.MicroSvcRelEntity;
import com.huawei.it.gaia.apigc.domain.monitor.model.SyncTaskEntity;
import com.huawei.it.gaia.apigc.domain.utils.DateUtils;
import com.huawei.it.gaia.apigc.service.common.annotation.ScheduleTaskConfig;
import com.huawei.it.gaia.apigc.service.common.taskmanage.AsyncTaskBaseService;
import com.huawei.it.gaia.apigc.service.task.dto.ApiServiceStaticsDTO;
import com.huawei.it.gaia.apigc.service.task.dto.ListApiPageResultDTO;
import com.huawei.it.gaia.apigc.service.task.dto.ListApiStaticsDTO;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* The type Monitor day log task.
*
* @author d30046373
* @since 2024-03-02
*/
@Service
@Slf4j
@ScheduleTaskConfig(createTables = {APIGC_TOPO_MICRO_REL, TBL_CLOUD_MONITOR_RECORD}, dependsOnTables = {},
owner = DU_BO_BO)
public class MonitorDayLogTask extends AsyncTaskBaseService {
private static final String LAST_UPDATE_USER_CN = "System";
/**
* 查询数据底座微服务实例映射关系页码
*/
private static final int SAM_DATA_PAGE_SIZE = 100;
private static final long ONE_MILLISECOND = 1000L;
/**
* The constant 毫秒单位.
*/
private static final int MILE_SECONDS = 1000;
/**
* 查询GaiaMonitor 初始页大小
*/
private static final int PAGE_SIZE = 4000;
/**
* The constant 一天的毫秒数.
*/
private static final int MILE_SECONDS_BY_DAY = 86400000;
@Autowired
private IGaiaMonitorTransferGateway gaiaMonitorTransferGateway;
@Autowired
private IApiAssetsGateway apiAssetsGateway;
@Autowired
private IGaiaMonitorGateway gaiaMonitorGateway;
@Autowired
private ICloudMonitorRecordGateway cloudMonitorRecordGateway;
@Autowired
private IMicroAppRelGateway microAppRelGateway;
@Autowired
private IMicroSvcRelGateway microSvcRelGateway;
@Autowired
private IApiLogMatchRelGateway apiLogMatchRelGateway;
private final Map<String, String> subAppNameShortNameMap = new HashMap<>();
// 待匹配的API集合
private List<ApiAssetsModel> apiLogMatchUrlByLogSource = new ArrayList<>();
@Autowired
private MicroSvcClient microSvcClient;
@Autowired
private MonitorSearchClient monitorSearchClient;
@Override
public void doService(Map<String, String> map) throws ApplicationException {
log.info("begin MonitorDayLogTask");
// 删除transfer表的老化数据
deleteExpiredData();
// 同步调用链中微服务实例
asyncInitNotMatchMicroRelData();
// 同步数据底座微服务实例映射关系
asyncItAssetsMicroRelData();
execute(map);
log.info("finish MonitorDayLogTask");
}
private void asyncItAssetsMicroRelData() {
int currPage = 1;
Map<String, MicroSvcRelEntity> microSvcRelId2MsId = microSvcRelGateway.queryIdFromMicroSvcRel()
.stream()
.filter(node -> StringUtils.isNotBlank(node.getMicroServiceId()))
.collect(Collectors.toMap(MicroSvcRelEntity::getId, Function.identity(), (key1, key2) -> key1));
// 获取底座微服务实例映射关系
int timeout = 0;
int totalNum = 0;
try {
long lastUpdateTime = DateUtils.getUTCSecond() * 1000;
HashMap<String, String> map = new HashMap<>();
map.put("effectiveDate", formatDate(DateUtils.getBeforeDay(-1), DATE_FORMAT_SHORT));
do {
List<MicroSvcRelRO> serviceSamDataMicroSvcRelList = microSvcClient.getSamDataMicroSvcRel(
SAM_DATA_PAGE_SIZE, currPage, map);
List<MicroSvcRelEntity> microSvcRelEntities = new ArrayList<>();
serviceSamDataMicroSvcRelList.forEach(item -> {
MicroSvcRelEntity microSvcRelEntity = getMicroSvcRelEntity(microSvcRelId2MsId, lastUpdateTime,
item);
if (microSvcRelEntity == null) {
return;
}
microSvcRelEntities.add(microSvcRelEntity);
});
if (CollectionUtils.isEmpty(serviceSamDataMicroSvcRelList)) {
break;
}
if (timeout > 10000) {
log.error("get samData microSvcRel data timeout!");
break;
}
timeout++;
currPage++;
totalNum += microSvcRelEntities.size();
if (!CollectionUtils.isEmpty(microSvcRelEntities)) {
microSvcRelGateway.insertMicroSvcRelList(microSvcRelEntities, "IT资产数据");
}
} while (true);
} catch (Exception e) {
log.error("get samData microSvcRel data failed!");
}
log.info("get samData microSvcRel data total count = {}", totalNum);
}
private MicroSvcRelEntity getMicroSvcRelEntity(Map<String, MicroSvcRelEntity> microSvcRelId2MsId,
long lastUpdateTime, MicroSvcRelRO item) {
String appId = item.getAppId();
String apmServiceName = item.getApmServiceName();
MicroSvcRelEntity microSvcRelEntity = new MicroSvcRelEntity();
String id = UUID.nameUUIDFromBytes((appId + apmServiceName).getBytes(StandardCharsets.UTF_8)).toString();
microSvcRelEntity.setId(id);
microSvcRelEntity.setAppId(appId);
microSvcRelEntity.setSubAppName(apmServiceName);
microSvcRelEntity.setMicroServiceId(item.getMsId());
// 数据存在,微服务
if (microSvcRelId2MsId.containsKey(id)) {
MicroSvcRelEntity svcRelVO = microSvcRelId2MsId.get(id);
if (!StringUtils.equals("System", svcRelVO.getLastUpdateUserCN()) || StringUtils.equals(item.getMsId(),
svcRelVO.getMicroServiceId())) {
return null;
}
}
microSvcRelEntity.setLastUpdateDate(lastUpdateTime);
microSvcRelEntity.setLastUpdateUserCN(LAST_UPDATE_USER_CN);
String activeSyncDate = DateUtils.getBeforeDayAndFormat(-1, DATE_FORMAT_SHORT);
microSvcRelEntity.setActiveSyncDate(activeSyncDate);
return microSvcRelEntity;
}
private void asyncInitNotMatchMicroRelData() {
Map<String, String> microSvcRelMap = microAppRelGateway.findAllMicroAppRelMap();
if (MapUtils.isEmpty(microSvcRelMap)) {
return;
}
List<MicroSvcRelEntity> MicroSvcRelEntityList = new ArrayList<>();
long lastUpdateTime = DateUtils.getUTCSecond() * 1000;
String activeSyncDate = DateUtils.getBeforeDayAndFormat(-1, DATE_FORMAT_SHORT);
microSvcRelMap.forEach((key, val) -> {
String id = UUID.nameUUIDFromBytes((val + key).getBytes(StandardCharsets.UTF_8)).toString();
MicroSvcRelEntity microSvcRelEntity = new MicroSvcRelEntity();
microSvcRelEntity.setId(id);
microSvcRelEntity.setAppId(val);
microSvcRelEntity.setSubAppName(key);
microSvcRelEntity.setActiveSyncDate(activeSyncDate);
microSvcRelEntity.setLastUpdateUserCN(LAST_UPDATE_USER_CN);
microSvcRelEntity.setLastUpdateDate(lastUpdateTime);
MicroSvcRelEntityList.add(microSvcRelEntity);
});
microSvcRelGateway.insertMicroSvcRelList(MicroSvcRelEntityList, "设计开发中心");
log.info("insert into microSvc rel count = {}", microSvcRelMap.size());
}
/**
* Execute.
*
* @param parameters the parameters
* @throws ApplicationException the application exception
*/
public void execute(Map<String, String> parameters) throws ApplicationException {
// 定时任务最后执行时间
SyncTaskEntity syncTaskEntity = cloudMonitorRecordGateway.selectLastQueryTime(API_RUNNING_SOURCE_MONITOR);
// 增加老化逻辑,只保存近90天数据
// 开始时间
long startTime = getStartTime(parameters, syncTaskEntity);
// 结束时间
long endTime = DateUtils.getTodayStartTime() / MILE_SECONDS * MILE_SECONDS;
if (parameters.containsKey("endTime")) {
long temTime = parseLong(parameters.get("endTime"));
if (temTime != 0) {
endTime = temTime;
}
}
initMicroNameMap();
getMonitorData(parameters, startTime, endTime);
subAppNameShortNameMap.clear();
log.info("end");
}
private void deleteExpiredData() {
// 保留10天内啄木鸟调用数据
String date = DateUtils.getBeforeDay(-10);
gaiaMonitorTransferGateway.deleteTaskLogBeforeDate(date);
}
private void getMonitorData(Map<String, String> parameters, long startTime, long endTime) {
String userCN = parameters.getOrDefault(SCHEDULE_TASK_EXECUTOR_KEY, SCHEDULE_TASK_EXECUTOR);
// 初始化数据持久化状态为,1:成功
while (startTime < endTime) {
int totalRows = 0;
String date = formatDate(new Date(startTime), DATE_FORMAT_COMPACT);
ListApiStaticsDTO listApiStaticsDTO = getListApiStaticsQO(startTime);
if (CollectionUtils.isEmpty(listApiStaticsDTO.getMicroServiceShortNameList())) {
insertOrUpdateSyncLogTask(parameters, startTime, userCN, totalRows);
log.info("insert into monitor logs count = {}", totalRows);
startTime = startTime + MILE_SECONDS_BY_DAY;
joinTransferData(date, API_RUNNING_SOURCE_MONITOR);
}
// 获取待查询APM列表,循环单个查询,避免monitor接口响应失败导致数据丢失
List<String> microServiceShortNames = listApiStaticsDTO.getMicroServiceShortNameList();
totalRows = getApiListFromMonitor(microServiceShortNames, listApiStaticsDTO, totalRows, date);
// 前端传值id为0时候此方法为添加,不为0或者不为空则为添加
insertOrUpdateSyncLogTask(parameters, startTime, userCN, totalRows);
log.info("insert into monitor logs count = {}", totalRows);
startTime = startTime + MILE_SECONDS_BY_DAY;
joinTransferData(date, API_RUNNING_SOURCE_MONITOR);
}
}
private int getApiListFromMonitor(List<String> microServiceShortNames, ListApiStaticsDTO listApiStaticsDTO,
int totalRows, String date) {
List<String> activeNameList = new ArrayList<>();
for (String item : microServiceShortNames) {
try {
// 查询分页数据,获取数据总量
listApiStaticsDTO.setMicroServiceShortNameList(Collections.singletonList(item));
listApiStaticsDTO.setCurrentpage(1);
listApiStaticsDTO.setPagesize(PAGE_SIZE);
ListApiPageResultDTO listApiPageResultVO = getElasticSearch(listApiStaticsDTO);
// 如果大于PAGE_SIZE的, 才一次性拿全量
if (listApiPageResultVO.getRows() > PAGE_SIZE) {
listApiStaticsDTO.setPagesize(listApiPageResultVO.getRows());
listApiPageResultVO = getElasticSearch(listApiStaticsDTO);
}
log.info("sync monitor getElasticSearch, APM = {}, size = {}", item, listApiPageResultVO.getRows());
List<ApiServiceStaticsDTO> resultData = listApiPageResultVO.getResultData();
if (!CollectionUtils.isEmpty(resultData)) {
totalRows += resultData.size();
Map<String, List<ApiServiceStaticsDTO>> groupByMonitorResult = resultData.stream()
.collect(Collectors.groupingBy(ApiServiceStaticsDTO::getMicroServiceId));
groupByMonitorResult.forEach((microServiceName, resultList) -> {
insertMonitorLogByMicroService(microServiceName, resultList, date);
});
activeNameList.add(item);
}
} catch (Exception e) {
log.error("APM = {} ,sync monitor api has error, msg = {}", item, e);
}
}
if (!CollectionUtils.isEmpty(activeNameList)) {
String activeSyncDate = DateUtils.getBeforeDayAndFormat(-1, DATE_FORMAT_SHORT);
// 更新API运行有效同步时间ACTIVE_SYNC_DATE
microSvcRelGateway.updateActiveSyncDateByApplicationName(activeSyncDate, activeNameList);
}
return totalRows;
}
private ListApiPageResultDTO getElasticSearch(ListApiStaticsDTO listApiStaticsDTO) {
ListApiPageResultDTO result = new ListApiPageResultDTO();
String baseUrl = getParams(listApiStaticsDTO);
List<ApiServiceStaticsDTO> apiServiceStaticsVOs = new ArrayList<>();
if (CollectionUtils.isEmpty(listApiStaticsDTO.getMicroServiceShortNameList())) {
result.setResultData(apiServiceStaticsVOs);
return result;
}
List<String> microServiceShortNameList = listApiStaticsDTO.getMicroServiceShortNameList();
MonitorSearchRO monitorSearchRO = monitorSearchClient.getMonitorSearch(baseUrl, microServiceShortNameList);
if (ObjectUtils.isEmpty(monitorSearchRO.getData())) {
log.error("getElasticSearch monitorSearchRO is null");
return result;
}
result.setRows(Integer.parseInt(monitorSearchRO.getData().getPageDTO().getTotalCount()));
buildResultData(apiServiceStaticsVOs, monitorSearchRO.getData(), listApiStaticsDTO.getTo());
result.setResultData(apiServiceStaticsVOs);
return result;
}
private String getParams(ListApiStaticsDTO listApiStaticsDTO) {
String other = "?startTime=" + listApiStaticsDTO.getFrom() + "&endTime=" + listApiStaticsDTO.getTo()
+ "&curPage=" + listApiStaticsDTO.getCurrentpage() + "&pageSize=" + listApiStaticsDTO.getPagesize();
if (StringUtils.isNotBlank(listApiStaticsDTO.getUrl())) {
other = other + "&uri=" + listApiStaticsDTO.getUrl();
}
if (StringUtils.equals("callsNumber", listApiStaticsDTO.getSortfield())) {
String sort = listApiStaticsDTO.getSort() == 0 ? "&sorts=-count" : "&sorts=count";
other = other + sort;
} else if (StringUtils.equals("averageCostTime", listApiStaticsDTO.getSortfield())) {
String sort = listApiStaticsDTO.getSort() == 0 ? "&sorts=-avgRes" : "&sorts=avgRes";
other = other + sort;
}
return other;
}
private void buildResultData(List<ApiServiceStaticsDTO> apiServiceStaticsVOs,
MonitorSearchRO.MonitorSearchInfo monitorSearchInfo, Long to) {
// to: 第二天凌晨; 最后调用时间:第二天凌晨 减 1000 毫秒
long lastCallTime = to - ONE_MILLISECOND;
List<MonitorSearchRO.MonitorSearchRecord> records = monitorSearchInfo.getRecords();
if (ObjectUtils.isEmpty(records) || records.isEmpty()) {
return;
}
records.forEach(record -> {
Long countSum = Long.valueOf(record.getCallsNumber());
Long failCountSum = Long.valueOf(record.getFailedNumber());
Long unqualifiedCountSum = Long.valueOf(record.getSlowCount());
long unqualifiedNum = ObjectUtils.isEmpty(unqualifiedCountSum) ? 0L : unqualifiedCountSum;
long callNum = ObjectUtils.isEmpty(countSum) ? 0L : countSum;
long qualifiedNum = callNum - unqualifiedNum;
long failCountNum = ObjectUtils.isEmpty(failCountSum) ? 0L : failCountSum;
long successNum = callNum - failCountNum;
ApiServiceStaticsDTO apiServiceStaticsDTO = new ApiServiceStaticsDTO();
buildBaseInfo(record, apiServiceStaticsDTO);
apiServiceStaticsDTO.setCallsNumber(callNum);
apiServiceStaticsDTO.setQualifiedNumber(qualifiedNum);
apiServiceStaticsDTO.setSuccessNumber(successNum);
apiServiceStaticsDTO.setSuccessRate(callNum == 0 ? 0 : (float) successNum / callNum);
apiServiceStaticsDTO.setQualifiedNumber(qualifiedNum);
apiServiceStaticsDTO.setQualifiedRate(callNum == 0 ? 0 : (float) qualifiedNum / callNum);
apiServiceStaticsDTO.setLastCallTimeStamp(lastCallTime);
apiServiceStaticsVOs.add(apiServiceStaticsDTO);
});
}
private void buildBaseInfo(MonitorSearchRO.MonitorSearchRecord record, ApiServiceStaticsDTO apiServiceStaticsDTO) {
apiServiceStaticsDTO.setAverageCostTime(Float.parseFloat(record.getAverageCostTime()));
apiServiceStaticsDTO.setFullPath(record.getFullPath());
apiServiceStaticsDTO.setMethod(record.getHttpMethod());
apiServiceStaticsDTO.setResp0To300Num(Long.parseLong(record.getThreeHundredMsCount()));
apiServiceStaticsDTO.setResp0To500Num(Long.parseLong(record.getFiveHundredMsCount()));
// 先在微服务id中保存微服务别名,后期在转换成微服务
apiServiceStaticsDTO.setMicroServiceId(record.getMicroServiceAbbreviation());
}
private void joinTransferData(String date, String source) {
if (API_RUNNING_SOURCE_MONITOR.equals(source)) {
// 查询数据
List<GaiaMonitorEntity> gaiaMonitorEntities = gaiaMonitorTransferGateway.getGaiaMonitorTransferList(date);
if (CollectionUtils.isEmpty(gaiaMonitorEntities)) {
log.info("gaiaMonitorTransferList is empty. date is {}", date);
return;
}
List<GaiaMonitorEntity> result = getJoinedTransferData(gaiaMonitorEntities);
// 数据集合插入tbl_gaiamonitor_day_log_2020数据库
gaiaMonitorGateway.createGaiaMonitor(result);
// 删除已经处理的数据
gaiaMonitorTransferGateway.deleteGaiaMonitorTransferData(date);
}
}
private long getSumCount(List<GaiaMonitorEntity> entities, Function<GaiaMonitorEntity, Long> function) {
return entities.stream().map(function::apply).filter(Objects::nonNull).mapToLong(Long::longValue).sum();
}
private List<GaiaMonitorEntity> getJoinedTransferData(List<GaiaMonitorEntity> gaiaMonitorEntities) {
Map<String, List<GaiaMonitorEntity>> map = gaiaMonitorEntities.stream()
.collect(Collectors.groupingBy(GaiaMonitorEntity::getAssetsId));
List<GaiaMonitorEntity> result = new ArrayList<>();
for (Map.Entry<String, List<GaiaMonitorEntity>> entry : map.entrySet()) {
List<GaiaMonitorEntity> entities = entry.getValue();
GaiaMonitorEntity entity = new GaiaMonitorEntity();
BeanUtils.copyProperties(entities.get(0), entity);
entity.setFullPath(entities.stream().map(GaiaMonitorEntity::getFullPath).collect(Collectors.joining(";")));
entity.setSuccessNum(getSumCount(entities, GaiaMonitorEntity::getSuccessNum));
long totalMun = getSumCount(entities, GaiaMonitorEntity::getTotalNum);
entity.setTotalNum(totalMun);
entity.setResp0To300Num(getSumCount(entities, GaiaMonitorEntity::getResp0To300Num));
entity.setResp0To500Num(getSumCount(entities, GaiaMonitorEntity::getResp0To500Num));
entity.setResp300To500Num(getSumCount(entities, GaiaMonitorEntity::getResp300To500Num));
entity.setResp500To1000Num(getSumCount(entities, GaiaMonitorEntity::getResp500To1000Num));
entity.setRespOver1000Num(getSumCount(entities, GaiaMonitorEntity::getRespOver1000Num));
long totalTime = entities.stream().mapToLong(item -> item.getAvgRespTime() * item.getTotalNum()).sum();
entity.setAvgRespTime(calcRate(totalTime, totalMun).intValue());
entity.setLastCallTime(entities.stream().mapToLong(GaiaMonitorEntity::getLastCallTime).max().orElse(0));
result.add(entity);
}
return result;
}
private BigDecimal calcRate(long child, long parent) {
if (parent == 0) {
return BigDecimal.ZERO;
}
BigDecimal rate = new BigDecimal(child).divide(new BigDecimal(parent), 4, BigDecimal.ROUND_HALF_UP);
if (rate.compareTo(BigDecimal.ONE) == 0 && child != parent) {
return new BigDecimal("0.9999");
} else {
return rate;
}
}
private void insertMonitorLogByMicroService(String monitorMicroServiceName, List<ApiServiceStaticsDTO> resultList,
String date) {
// 通过monitor接口返回的微服务名称找微服务的ID,调用链映射中的微服务别名直接从apigc_topo_micro_rel表中取数
String microServiceId = subAppNameShortNameMap.getOrDefault(monitorMicroServiceName, "");
if (CollectionUtils.isEmpty(resultList)) {
return;
}
List<GaiaMonitorEntity> gaiaMonitorEntities = new ArrayList<>();
for (ApiServiceStaticsDTO apiServiceStaticsDTO : resultList) {
GaiaMonitorEntity gaiaMonitorEntity = new GaiaMonitorEntity();
gaiaMonitorEntity.setDate(date);
gaiaMonitorEntity.setFullPath(subFullPathString(apiServiceStaticsDTO.getFullPath()));
String method = apiServiceStaticsDTO.getMethod();
gaiaMonitorEntity.setMethod(StringUtils.isEmpty(method) ? "" : method);
gaiaMonitorEntity.setMicroServiceName(monitorMicroServiceName);
gaiaMonitorEntity.setMicroServiceId(microServiceId);
gaiaMonitorEntity.setTotalNum(apiServiceStaticsDTO.getCallsNumber());
gaiaMonitorEntity.setSuccessNum(apiServiceStaticsDTO.getSuccessNumber());
gaiaMonitorEntity.setResp0To300Num(apiServiceStaticsDTO.getResp0To300Num());
gaiaMonitorEntity.setResp0To500Num(apiServiceStaticsDTO.getResp0To500Num());
gaiaMonitorEntity.setAvgRespTime((int) apiServiceStaticsDTO.getAverageCostTime());
gaiaMonitorEntity.setLastCallTime(apiServiceStaticsDTO.getLastCallTimeStamp());
gaiaMonitorEntities.add(gaiaMonitorEntity);
}
getMatchedAssetsApiID(gaiaMonitorEntities);
// 插入数据到tbl_gaiamonitor_day_log_transfer
if (!CollectionUtils.isEmpty(gaiaMonitorEntities)) {
gaiaMonitorTransferGateway.createGaiaMonitorTransfer(gaiaMonitorEntities);
}
}
private void getMatchedAssetsApiID(List<GaiaMonitorEntity> gaiaMonitorEntities) {
long todayStartTime = DateUtils.getTodayStartTime() / 1000 * 1000;
// 请求方式分组,一次查询匹配多条
Map<String, List<GaiaMonitorEntity>> groupByMethod = gaiaMonitorEntities.stream()
.collect(Collectors.groupingBy(GaiaMonitorEntity::getMethod));
groupByMethod.forEach((method, list) -> {
GaiaMonitorEntity gaiaMonitorEntity = list.get(0);
String microServiceId = gaiaMonitorEntity.getMicroServiceId();
ApiLogMatchRelEntity apiLogMatchRelEntity = new ApiLogMatchRelEntity();
apiLogMatchRelEntity.setHttpMethod(method);
apiLogMatchRelEntity.setMicroServiceId(microServiceId);
apiLogMatchRelEntity.setSource(API_RUNNING_SOURCE_MONITOR);
// 初始化单次待匹配API列表
apiLogMatchUrlByLogSource = apiAssetsGateway.getApiLogMatchUrlByLogSource(method, microServiceId);
list.forEach(item -> {
matchedMonitorLogData(todayStartTime, apiLogMatchRelEntity, item);
});
// 清理单次缓存
apiLogMatchUrlByLogSource.clear();
});
}
private void matchedMonitorLogData(long todayStartTime, ApiLogMatchRelEntity apiLogMatchRelEntity,
GaiaMonitorEntity item) {
String logFullPath = item.getFullPath();
// 查询历史记录表
apiLogMatchRelEntity.setUrl(logFullPath);
String matchedAssetsId = getMatchedAssetsApiId(apiLogMatchRelEntity);
ApiAssetsModel apiAssetsModel = apiAssetsGateway.findApiById(matchedAssetsId);
if (StringUtils.isNotEmpty(matchedAssetsId) && ObjectUtils.isNotEmpty(apiAssetsModel) && apiAssetsNotChanged(
logFullPath, apiAssetsModel)) {
item.setAssetsId(matchedAssetsId);
apiLogMatchRelEntity.setLastUpdateDate(todayStartTime);
apiLogMatchRelEntity.setAssetsId(matchedAssetsId);
apiLogMatchRelGateway.insertOrUpdateMatchModel(apiLogMatchRelEntity);
return;
}
// 匹配API列表信息
String assetsId = getMatchedAssetsId(logFullPath, apiLogMatchUrlByLogSource);
item.setAssetsId(assetsId);
if (StringUtils.isNotEmpty(assetsId)) {
apiLogMatchRelEntity.setLastUpdateDate(todayStartTime);
apiLogMatchRelEntity.setAssetsId(assetsId);
apiLogMatchRelGateway.insertOrUpdateMatchModel(apiLogMatchRelEntity);
}
}
/**
* monitor 已匹配的API资产数据,进行再次校验
*
* @param logFullPath logFullPath
* @param apiAssetsInfoVO apiAssetsInfoVO
* @return boolean
*/
private boolean apiAssetsNotChanged(String logFullPath, ApiAssetsModel apiAssetsInfoVO) {
List<String> accessPathList = new ArrayList<>();
accessPathList.add(apiAssetsInfoVO.getPath());
if (StringUtils.isNotEmpty(apiAssetsInfoVO.getAccessChannel())) {
accessPathList.addAll(Arrays.asList(apiAssetsInfoVO.getAccessChannel().split(";")));
}
return accessPathList.contains(logFullPath);
}
private String getMatchedAssetsId(String logFullUrl, List<ApiAssetsModel> apiLogMatchUrlByLogSource) {
// 1、 获取待匹配API资产的访问路径信息
Map<String, String> accessChannelMap = apiLogMatchUrlByLogSource.stream()
.filter(item -> StringUtils.isNotEmpty(item.getAccessChannel()))
.collect(Collectors.toMap(ApiAssetsModel::getAccessChannel, apiAssetsDO -> apiAssetsDO.getId(),
(key1, key2) -> key1));
Map<String, String> compareAccessChannelMap = new HashMap<>();
apiLogMatchUrlByLogSource.forEach(item -> {
if (StringUtils.isNotEmpty(item.getPath())) {
compareAccessChannelMap.put(item.getPath(), item.getId());
}
});
accessChannelMap.forEach((accessChannel, assetsId) -> {
String[] split = accessChannel.split(";");
for (String item : split) {
compareAccessChannelMap.put(item, assetsId);
}
});
if (compareAccessChannelMap.containsKey(logFullUrl)) {
return compareAccessChannelMap.get(logFullUrl);
}
// 2、通过path 字段筛,APId path 与日志访问路径末尾匹配
List<ApiAssetsModel> compareResult = apiLogMatchUrlByLogSource.stream()
.filter(item -> item != null && StringUtils.endsWith(logFullUrl, item.getPath()))
.collect(Collectors.toList());
if (!CollectionUtils.isEmpty(compareResult) && compareResult.size() == 1) {
return compareResult.get(0).getId();
}
return StringUtils.EMPTY;
}
private String getMatchedAssetsApiId(ApiLogMatchRelEntity apiLogMatchRelDO) {
ApiLogMatchRelEntity apiLogMatchRelEntity = apiLogMatchRelGateway.findApiLogMatchRelDO(apiLogMatchRelDO);
if (ObjectUtils.isNotEmpty(apiLogMatchRelEntity)) {
return apiLogMatchRelEntity.getAssetsId();
}
return StringUtils.EMPTY;
}
private ListApiStaticsDTO getListApiStaticsQO(long startTime) {
ListApiStaticsDTO listApiStaticsDTO = new ListApiStaticsDTO();
listApiStaticsDTO.setFrom(startTime);
listApiStaticsDTO.setTo(startTime + MILE_SECONDS_BY_DAY);
List<MicroSvcRelEntity> microSvcRelEntities = microSvcRelGateway.queryIdFromMicroSvcRel();
List<String> microServiceShorterNames = microSvcRelEntities.stream()
.map(MicroSvcRelEntity::getSubAppName)
.collect(Collectors.toList());
listApiStaticsDTO.setMicroServiceShortNameList(microServiceShorterNames);
return listApiStaticsDTO;
}
private long parseLong(String source) {
try {
return Long.parseLong(source);
} catch (NumberFormatException e) {
return 0;
}
}
private long getStartTime(Map<String, String> parameters, SyncTaskEntity syncTaskEntity) {
long startTime = DateUtils.getYearStartTime();
if (ObjectUtils.isNotEmpty(syncTaskEntity)) {
String date = syncTaskEntity.getDate();
if ("0".equals(syncTaskEntity.getFlag())) {
startTime = DateUtils.getDateStartTime(date);
} else {
startTime = DateUtils.getDateStartTime(date) + MILE_SECONDS_BY_DAY;
}
}
if (null != parameters.get("startTime")) {
startTime = Long.parseLong(parameters.get("startTime"));
}
return startTime;
}
private String subFullPathString(String fullPath) {
String path = fullPath;
if (!StringUtils.isEmpty(fullPath) && fullPath.length() > 800) {
path = fullPath.substring(fullPath.length() - 800);
log.info("full_path length > 800,subString after 800 length");
}
return path;
}
private void insertOrUpdateSyncLogTask(Map<String, String> parameters, long startTime, String userCN,
Integer resultNum) {
SyncTaskEntity syncTaskEntity = new SyncTaskEntity();
syncTaskEntity.setTaskType(API_RUNNING_SOURCE_MONITOR);
syncTaskEntity.setRuntime(DateUtils.getTodayDate(DATE_FORMAT_FULL));
syncTaskEntity.setDate(startTime + "");
syncTaskEntity.setUser(userCN);
syncTaskEntity.setInsertCount(resultNum.longValue());
syncTaskEntity.setFlag(resultNum > 0 ? "1" : "0");
// 自动or手动触发定时任务
String id = parameters.get("id");
if (!StringUtils.isEmpty(id) && !"0".equals(id)) {
syncTaskEntity.setTaskId(id);
cloudMonitorRecordGateway.updateSyncTask(syncTaskEntity);
} else {
cloudMonitorRecordGateway.insertOrUpdate(Collections.singletonList(syncTaskEntity));
}
}
private void initMicroNameMap() {
List<MicroSvcRelEntity> services = microSvcRelGateway.queryIdFromMicroSvcRel();
for (MicroSvcRelEntity service : services) {
if (StringUtils.isEmpty(service.getMicroServiceId())) {
continue;
}
subAppNameShortNameMap.put(service.getSubAppName(), service.getMicroServiceId());
}
}
}
详细解释下asyncItAssetsMicroRelData和asyncInitNotMatchMicroRelData
最新发布