package com.jkt.sjzy.data.batch;
import com.jkt.sjzy.data.batch.runner.FetchFromApiRunner;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.scheduling.annotation.EnableScheduling;
import java.util.Arrays;
/**
* Hello world!
*git branch -M master
*/
@SpringBootApplication
@EnableScheduling
@MapperScan(basePackages = {"com.jkt.sjzy.data.batch.mapper"})
public class BatchDataApplication
{
public static void main( String[] args )
{
ConfigurableApplicationContext context = SpringApplication.run(BatchDataApplication.class, args);
FetchFromApiRunner runner = context.getBean(FetchFromApiRunner.class);
boolean isFirstExecution = Arrays.asList(args).contains("--first-execution");
runner.fetchFromApi(isFirstExecution);
System.exit(0);
}
}
package com.jkt.sjzy.data.batch.handler.savemode;
import cn.hutool.core.date.DateUtil;
import cn.hutool.db.PageResult;
import com.alibaba.fastjson2.JSONObject;
import com.jkt.sjzy.data.batch.domain.entities.DataApiConfig;
import com.jkt.sjzy.data.batch.domain.entities.DataApiTokenConfig;
import com.jkt.sjzy.data.batch.enums.ApiType;
import com.jkt.sjzy.data.batch.handler.fetch.FetchDataFactory;
import com.jkt.sjzy.data.batch.handler.fetch.IFetchDataHandler;
import com.jkt.sjzy.data.batch.handler.token.ITokenHandler;
import com.jkt.sjzy.data.batch.handler.token.TokenHandlerFactory;
import com.jkt.sjzy.data.batch.handler.write.IWriteDataHandler;
import okhttp3.Headers;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
/**
* @Author: wangruiming
* @Date: 2025/6/26 17:26
*/
public abstract class SaveDataTemplate implements ISaveDataHandler {
@Override
public void saveData(DataApiTokenConfig tokenConfig, DataApiConfig apiConfig, ApiType apiType, String beginDate, String endDate) {
IWriteDataHandler writeDataHandler = preStep(apiConfig, apiType);
IFetchDataHandler<?> init = FetchDataFactory.init(apiConfig.getFetchDataClazz());
init.fetchData(apiType, tokenConfig, apiConfig, writeDataHandler, beginDate, endDate);
}
/**
* 前置操作步骤
*
* @param apiConfig
* @param apiType
*/
public abstract IWriteDataHandler preStep(DataApiConfig apiConfig, ApiType apiType);
}
package com.jkt.sjzy.data.batch.handler.fetch;
import java.lang.reflect.InvocationTargetException;
/**
* @Author: wangruiming
* @Date: 2025/6/26 17:52
*/
public class FetchDataFactory {
public static IFetchDataHandler<?> init(String clazz){
try {
Class<?> aClass = Class.forName(clazz);
return (IFetchDataHandler<?>) aClass.getConstructor().newInstance();
} catch (ClassNotFoundException | InvocationTargetException | InstantiationException | IllegalAccessException |
NoSuchMethodException e) {
throw new RuntimeException(e);
}
}
}
package com.jkt.sjzy.data.batch.handler.fetch;
import com.jkt.sjzy.data.batch.domain.entities.DataApiConfig;
import com.jkt.sjzy.data.batch.domain.entities.DataApiTokenConfig;
import com.jkt.sjzy.data.batch.enums.ApiType;
import com.jkt.sjzy.data.batch.handler.write.IWriteDataHandler;
/**
* @Author: wangruiming
* @Date: 2025/6/26 17:50
*/
public interface IFetchDataHandler<T> {
void fetchData(ApiType apiType, DataApiTokenConfig tokenConfig, DataApiConfig config, IWriteDataHandler handler, String beginDate, String endDate);
}
package com.jkt.sjzy.data.batch.handler.fetch.cck;
import cn.hutool.core.util.StrUtil;
import cn.hutool.crypto.digest.DigestUtil;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.google.common.util.concurrent.RateLimiter;
import com.jkt.sjzy.data.batch.domain.dao.cck.CckPageResult;
import com.jkt.sjzy.data.batch.domain.entities.DataApiConfig;
import com.jkt.sjzy.data.batch.domain.entities.DataApiTokenConfig;
import com.jkt.sjzy.data.batch.domain.entities.JobLastRunningLog;
import com.jkt.sjzy.data.batch.enums.ApiType;
import com.jkt.sjzy.data.batch.exception.TokenExpiredException;
import com.jkt.sjzy.data.batch.handler.fetch.IFetchDataHandler;
import com.jkt.sjzy.data.batch.handler.token.ITokenHandler;
import com.jkt.sjzy.data.batch.handler.token.TokenHandlerFactory;
import com.jkt.sjzy.data.batch.handler.write.IWriteDataHandler;
import com.jkt.sjzy.data.batch.service.JobLastRunningLogService;
import com.jkt.sjzy.data.batch.service.ServiceRegistry;
import com.jkt.sjzy.data.batch.util.JsonUtil;
import com.jkt.sjzy.data.batch.util.RequestUtil;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import okhttp3.Headers;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.env.Environment;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import static com.alibaba.fastjson2.JSONObject.parseObject;
import static com.jkt.sjzy.data.batch.constant.Constant.*;
/**
* @Author: wangruiming
* @Date: 2025/6/26 17:53
*/
@Slf4j
public class CckFetchDataHandler<T> implements IFetchDataHandler<T> {
private final Environment environment;
private final JobLastRunningLogService jobLastRunningLogService;
public CckFetchDataHandler(Environment environment, JobLastRunningLogService jobLastRunningLogService) {
this.environment = environment;
this.jobLastRunningLogService = jobLastRunningLogService;
}
@Override
public void fetchData(ApiType apiType, DataApiTokenConfig tokenConfig, DataApiConfig config, IWriteDataHandler handler, String beginDate, String endDate) {
Set<String> idSet = new HashSet<>();
String timeRangeType = getTimeRangeType();
LocalDateTime now = LocalDateTime.now();
int pageNum = 1;
//刷新token
ITokenHandler tokenHandler = TokenHandlerFactory.getTokenHandler(tokenConfig);
if (!tokenHandler.validateToken()) {
tokenHandler.refreshToken();
}
// 处理分页接口
if (config.getPaged()) {
// String beginDate = "2019-01-01";
// String endDate = DateUtil.format(LocalDateTime.now(), "yyyy-MM-dd");
while (true) {
List<Object> allData = new ArrayList<>();
CckPageResult pageResult = fetchPagedData(apiType, config, tokenHandler, pageNum, beginDate, endDate);
if (pageResult == null || pageResult.getResources().isEmpty()) {
break;
}
for (JSONObject data : pageResult.getResources()) {
String id = extractId(data, apiType);
if (id != null && idSet.add(id)) {
T bean = processDataToBean(data, config, apiType);
if (bean != null) {
allData.add(bean);
}
}
data = null;
}
handler.writeData(allData);
if (pageNum >= pageResult.getTotalPageNo()) {
break;
}
pageNum++;
}
}
// 处理单日接口
else {
LocalDate startDate = LocalDate.of(2025, 1, 1);
LocalDate endDateObj = parseDate(endDate);
LocalDate beginDateObj = parseDate(beginDate);
if (beginDateObj.isBefore(startDate)) {
beginDateObj = startDate;
}
// Iterator<LocalDate> dateIterator = startDate.datesUntil(endDate.plusDays(1)).iterator();
List<LocalDate> allDates = beginDateObj.datesUntil(endDateObj.plusDays(1)).collect(Collectors.toList());
int threadCount = Math.min(Runtime.getRuntime().availableProcessors() * 2, 10);
ExecutorService executor = Executors.newFixedThreadPool(threadCount);
Semaphore semaphore = new Semaphore(5); // 限制最大并发请求数
RateLimiter rateLimiter = RateLimiter.create(5.0); // 每秒最多5个请求
BlockingQueue<T> resultQueue = new LinkedBlockingQueue<>();
CountDownLatch latch = new CountDownLatch(allDates.size());
// while (dateIterator.hasNext()) {
// 提交所有日期的请求任务
for (LocalDate date : allDates) {
executor.submit(() -> {
try {
semaphore.acquire();
rateLimiter.acquire();
// LocalDate currentDate = dateIterator.next();
String dateStr = date.format(DateTimeFormatter.ISO_LOCAL_DATE);
// 获取单日数据
JSONObject data = fetchSingleDayData(apiType, tokenHandler, config, dateStr);
if (data != null) {
String id = extractId(data, apiType);
if (id != null && idSet.add(id)) {
T bean = processDataToBean(data, config, apiType);
if (bean != null) {
resultQueue.put(bean);
}
data = null;
}
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error("处理日期 {} 时出错", date, e);
} finally {
semaphore.release();
latch.countDown();
}
});
}
try {
latch.await();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
log.error("等待任务完成时被中断", e);
} finally {
executor.shutdown();
}
List<Object> allData = new ArrayList<>(resultQueue);
handler.writeData(allData);
}
jobLastRunningLogService.saveOrUpdate(new JobLastRunningLog(DigestUtil.md5Hex(config.getTargetTable()), apiType.getDesc(), now, (long) idSet.size()));
}
/**
* 将JSON数据转换为对应的Bean对象
*/
@SuppressWarnings("unchecked")
private <T> T processDataToBean(JSONObject data, DataApiConfig apiConfig, ApiType apiType) {
if (data == null) {
return null;
}
// 转换为小写下划线命名
try {
Class<?> clazz = Class.forName(DAO_CLAZZ_PATH.formatted(apiConfig.getProjectCode()) + apiType.getBeanClazz());
JsonUtil.jsonToUnderLineKey(data);
return (T) JSONObject.parseObject(data.toJSONString(), clazz);
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
}
/**
* 获取单日数据
*/
private JSONObject fetchSingleDayData(ApiType apiType, ITokenHandler tokenHandler, DataApiConfig config, String date)
{
AtomicInteger retryCount = new AtomicInteger(0);
while (retryCount.getAndIncrement() < MAX_PAGE_RETRIES) {
try {
String url = config.getUrl() + "?date=" + date;
Headers headers = Headers.of("x-token", tokenHandler.getToken());
String responseBody = RequestUtil.requestRaw("GET", url, headers, "");
return parseSingleResult(responseBody, apiType);
} catch (TokenExpiredException e) {
tokenHandler.refreshToken();
} catch (Exception e) {
if (retryCount.get() < MAX_PAGE_RETRIES) {
tokenHandler.sleepWithBackoff(retryCount.get());
}
}
}
return null;
}
/**
* 解析单日结果
*/
private JSONObject parseSingleResult(String response, ApiType apiType) {
if (StrUtil.isBlank(response)) {
return null;
}
try {
JSONObject responseObj = parseObject(response);
if (responseObj == null) {
return null;
}
// 检查响应状态码
int ret = responseObj.getIntValue("ret");
if (ret != 0) {
log.warn("{}接口返回非成功状态: {}", apiType.getDesc(), ret);
return null;
}
// 解析嵌套的data结构
if (responseObj.containsKey("data")) {
JSONObject outerData = responseObj.getJSONObject("data");
if (outerData != null && outerData.containsKey("data")) {
return outerData.getJSONObject("data");
}
}
} catch (Exception e) {
log.error("解析{}单日数据失败: {}", apiType.getDesc(), e.getMessage());
}
return null;
}
/**
* 获取分页数据
*/
private CckPageResult fetchPagedData(ApiType apiType, DataApiConfig config, ITokenHandler tokenHandler, int pageNum,
String beginDate, String endDate) {
AtomicInteger retryCount = new AtomicInteger(0);
while (retryCount.getAndIncrement() < MAX_PAGE_RETRIES) {
try {
String url = config.getUrl() + "?beginDate=" + beginDate +
"&pageNo=" + pageNum + "&pageSize=" + PAGE_SIZE +
"&endDate=" + endDate;
Headers headers = Headers.of("x-token", tokenHandler.getToken());
String responseBody = RequestUtil.requestRaw("GET", url, headers, "");
return parsePageResult(responseBody, apiType);
} catch (TokenExpiredException e) {
tokenHandler.refreshToken();
} catch (Exception e) {
tokenHandler.sleepWithBackoff(retryCount.get());
}
}
return null;
}
/**
* 从数据中提取唯一标识
*/
private String extractId(JSONObject data, ApiType apiType) {
if (data == null) {
return null;
}
// 根据不同接口类型提取不同的ID字段
switch (apiType) {
case ACTIVITY:
return data.getString("ID");
case EVENT:
return data.getString("ID");
case TRAINING_VIDEO:
return data.getString("ID");
case TRAINING_TEXT:
return data.getString("ID");
case EVENT_INFO:
return data.getString("DATE");
case PASSENGER_INFO:
return data.getString("DATETIME");
case ACTIVITY_INFO:
return data.getString("DATE");
default:
return data.getString("ID");
}
}
//日期格式参数化
private static LocalDate parseDate(String dateStr) {
return dateStr.contains("T")
? LocalDateTime.parse(dateStr).toLocalDate()
: LocalDate.parse(dateStr);
}
private String getTimeRangeType() {
return environment.getProperty("data.fetch.timeRangeType", "NORMAL");
}
/**
* 解析分页结果
*/
private CckPageResult parsePageResult(String response, ApiType apiType) {
if (StrUtil.isBlank(response)) {
return new CckPageResult(new ArrayList<>(), 0, 1);
}
List<JSONObject> dataList = new ArrayList<>();
int totalCount = 0;
int totalPage = 1;
try {
JSONObject responseObj = parseObject(response);
if (responseObj == null) {
return new CckPageResult(dataList, totalCount, totalPage);
}
// 嵌套的data.data层
JSONObject dataObj = responseObj.getJSONObject("data").getJSONObject("data");
totalCount = dataObj.getIntValue("count");
int pageSize = dataObj.getIntValue("pageSize");
totalPage = (totalCount + pageSize - 1) / pageSize;
JSONArray dataArray = dataObj.getJSONArray("list");
if (dataArray != null) {
for (int i = 0; i < dataArray.size(); i++) {
JSONObject data = dataArray.getJSONObject(i);
if (data != null) {
dataList.add(data);
}
}
}
} catch (Exception e) {
log.error("解析{}分页数据失败: {}", apiType.getDesc(), e.getMessage());
}
return new CckPageResult(dataList, totalCount, totalPage);
}
}
线上运行的代码出现OOM问题，经排查发现是由于Fastjson在转换对象为JSON字符串时引发的。Fastjson在处理大量数据时会分配大量内存，可能导致内存溢出。考虑到其安全性和性能问题，建议改用Jackson的writeValueAsString()方法来替代，以避免类似问题。此外，文章还强调了在选择JSON库时应考虑其稳定性和性能。
扫一扫
1627
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
