多线程插入数据库
package com.jd.adapter.processor.huarun.so;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import javax.annotation.PreDestroy;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.*;
@Slf4j
@Component
public class HuaRunBatchInsertSimpleHelper {
private static final int CORE_POOL_SIZE = 5;
private static final int MAX_POOL_SIZE = 10;
private static final int QUEUE_CAPACITY = 1000;
private static final long KEEP_ALIVE_TIME = 60L;
private final ExecutorService executorService = new ThreadPoolExecutor(
CORE_POOL_SIZE,
MAX_POOL_SIZE,
KEEP_ALIVE_TIME,
TimeUnit.SECONDS,
new LinkedBlockingQueue<>(QUEUE_CAPACITY),
new ThreadFactory() {
private int count = 0;
@Override
public Thread newThread(Runnable r) {
return new Thread(r, "huarun-batch-insert-simple-" + (++count));
}
},
new ThreadPoolExecutor.CallerRunsPolicy()
);
public <T> BatchInsertResult processBatchInsert(List<T> dataList, int batchSize) {
if (CollectionUtils.isEmpty(dataList)) {
return new BatchInsertResult(0, 0, Collections.emptyList());
}
List<List<T>> partitions = partition(dataList, batchSize);
List<CompletableFuture<BatchInsertResult>> futures = new ArrayList<>();
for (List<T> batch : partitions) {
CompletableFuture<BatchInsertResult> future = CompletableFuture.supplyAsync(
() -> doInsert(batch),
executorService
);
futures.add(future);
}
CompletableFuture<Void> allFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
try {
allFutures.get(1, TimeUnit.HOURS);
} catch (InterruptedException e) {
log.error("批量插入任务被中断", e);
Thread.currentThread().interrupt();
return new BatchInsertResult(0, dataList.size(), Collections.singletonList("Task Interrupted"));
} catch (ExecutionException | TimeoutException e) {
log.error("批量插入任务执行异常或超时", e);
}
return collectResults(futures);
}
private <T> BatchInsertResult doInsert(List<T> batch) {
int success = 0;
int fail = 0;
List<String> errors = new ArrayList<>();
try {
success = batch.size();
} catch (Exception e) {
log.error("本批次插入异常, size: {}", batch.size(), e);
fail = batch.size();
errors.add("Batch insert failed: " + e.getMessage());
}
return new BatchInsertResult(success, fail, errors);
}
private BatchInsertResult collectResults(List<CompletableFuture<BatchInsertResult>> futures) {
int totalSuccess = 0;
int totalFail = 0;
List<String> totalErrors = new ArrayList<>();
for (CompletableFuture<BatchInsertResult> future : futures) {
try {
if (future.isDone() && !future.isCompletedExceptionally()) {
BatchInsertResult result = future.get();
totalSuccess += result.getSuccessCount();
totalFail += result.getFailCount();
if (!CollectionUtils.isEmpty(result.getErrorMessages())) {
totalErrors.addAll(result.getErrorMessages());
}
} else {
totalErrors.add("Task failed or was cancelled unexpectedly");
}
} catch (Exception e) {
log.error("获取子任务结果异常", e);
totalErrors.add("Get result exception: " + e.getMessage());
}
}
return new BatchInsertResult(totalSuccess, totalFail, totalErrors);
}
private <T> List<List<T>> partition(List<T> list, int size) {
if (list == null) return Collections.emptyList();
List<List<T>> partitions = new ArrayList<>();
for (int i = 0; i < list.size(); i += size) {
partitions.add(new ArrayList<>(list.subList(i, Math.min(i + size, list.size()))));
}
return partitions;
}
@PreDestroy
public void destroy() {
if (executorService != null && !executorService.isShutdown()) {
executorService.shutdown();
try {
if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
executorService.shutdownNow();
}
} catch (InterruptedException e) {
executorService.shutdownNow();
}
}
}
@Data
@AllArgsConstructor
@NoArgsConstructor
public static class BatchInsertResult {
private int successCount;
private int failCount;
private List<String> errorMessages;
}
}
进阶版
package com.jd.adapter.processor.huarun.so;
import com.jd.adapter.util.ThreadLogUtil;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import javax.annotation.PreDestroy;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.*;
import java.util.function.Consumer;
@Slf4j
@Component
public class HuaRunBatchInsertHelper {
private static final int CORE_POOL_SIZE = 5;
private static final int MAX_POOL_SIZE = 10;
private static final int QUEUE_CAPACITY = 1000;
private static final long KEEP_ALIVE_TIME = 60L;
private final ExecutorService executorService = new ThreadPoolExecutor(
CORE_POOL_SIZE,
MAX_POOL_SIZE,
KEEP_ALIVE_TIME,
TimeUnit.SECONDS,
new LinkedBlockingQueue<>(QUEUE_CAPACITY),
new ThreadFactory() {
private int count = 0;
@Override
public Thread newThread(Runnable r) {
return new Thread(r, "huarun-batch-insert-" + (++count));
}
},
new ThreadPoolExecutor.CallerRunsPolicy()
);
public <T> BatchInsertResult processBatchInsert(List<T> dataList, int batchSize, Consumer<List<T>> batchConsumer) {
if (CollectionUtils.isEmpty(dataList)) {
return new BatchInsertResult(0, 0, Collections.emptyList());
}
String parentTraceId = ThreadLogUtil.getTid();
List<List<T>> partitions = partition(dataList, batchSize);
List<CompletableFuture<BatchInsertResult>> futures = new ArrayList<>();
for (List<T> batch : partitions) {
CompletableFuture<BatchInsertResult> future = CompletableFuture.supplyAsync(
() -> {
ThreadLogUtil.requestStart(parentTraceId);
try {
return doInsert(batch, batchConsumer);
} finally {
ThreadLogUtil.requestEnd();
}
},
executorService
);
futures.add(future);
}
CompletableFuture<Void> allFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
try {
allFutures.get(2, TimeUnit.HOURS);
} catch (InterruptedException e) {
log.error("批量插入任务被中断", e);
Thread.currentThread().interrupt();
return new BatchInsertResult(0, dataList.size(), Collections.singletonList("Task Interrupted: " + e.getMessage()));
} catch (ExecutionException | TimeoutException e) {
log.error("批量插入任务执行异常或超时", e);
}
return collectResults(futures);
}
private <T> BatchInsertResult doInsert(List<T> batch, Consumer<List<T>> batchConsumer) {
int success = 0;
int fail = 0;
List<String> errors = new ArrayList<>();
try {
batchConsumer.accept(batch);
success = batch.size();
} catch (Exception e) {
log.error("本批次插入异常, size: {}", batch.size(), e);
fail = batch.size();
errors.add("Batch insert failed: " + e.getMessage());
}
return new BatchInsertResult(success, fail, errors);
}
private BatchInsertResult collectResults(List<CompletableFuture<BatchInsertResult>> futures) {
int totalSuccess = 0;
int totalFail = 0;
List<String> totalErrors = new ArrayList<>();
for (CompletableFuture<BatchInsertResult> future : futures) {
try {
if (future.isDone() && !future.isCompletedExceptionally() && !future.isCancelled()) {
BatchInsertResult result = future.get();
totalSuccess += result.getSuccessCount();
totalFail += result.getFailCount();
if (!CollectionUtils.isEmpty(result.getErrorMessages())) {
totalErrors.addAll(result.getErrorMessages());
}
} else {
totalErrors.add("Task failed, timed out or was cancelled");
}
} catch (Exception e) {
log.error("获取子任务结果异常", e);
totalErrors.add("Get result exception: " + e.getMessage());
}
}
return new BatchInsertResult(totalSuccess, totalFail, totalErrors);
}
private <T> List<List<T>> partition(List<T> list, int size) {
if (list == null) return Collections.emptyList();
List<List<T>> partitions = new ArrayList<>();
for (int i = 0; i < list.size(); i += size) {
partitions.add(new ArrayList<>(list.subList(i, Math.min(i + size, list.size()))));
}
return partitions;
}
@PreDestroy
public void destroy() {
if (executorService != null && !executorService.isShutdown()) {
executorService.shutdown();
try {
if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
executorService.shutdownNow();
}
} catch (InterruptedException e) {
executorService.shutdownNow();
}
}
}
@Data
@AllArgsConstructor
@NoArgsConstructor
public static class BatchInsertResult {
private int successCount;
private int failCount;
private List<String> errorMessages;
}
}
