Java 线程池插入百万级数据

开发目的：提高百万级数据插入效率。

采取方案：利用ThreadPoolTaskExecutor多线程批量插入。

1. application-dev.properties添加线程池配置信息

# 异步线程配置
# 配置核心线程数
async.executor.thread.core_pool_size = 30
# 配置最大线程数
async.executor.thread.max_pool_size = 30
# 配置队列大小
async.executor.thread.queue_capacity = 99988
# 配置线程池中的线程的名称前缀
async.executor.thread.name.prefix = async-importDB-

2. spring容器注入线程池bean对象

@Configuration
@EnableAsync
@Slf4j
public class ExecutorConfig {
    
       // 获取当前机器的核数
    public static final int cpuNum = Runtime.getRuntime().availableProcessors();
     
    @Value("${async.executor.thread.core_pool_size}")
    private int corePoolSize;
    @Value("${async.executor.thread.max_pool_size}")
    private int maxPoolSize;
    @Value("${async.executor.thread.queue_capacity}")
    private int queueCapacity;
    @Value("${async.executor.thread.name.prefix}")
    private String namePrefix;
    @Bean(name = "asyncServiceExecutor")
    public Executor asyncServiceExecutor() {
        log.warn("start asyncServiceExecutor");
        //在这里修改
        ThreadPoolTaskExecutor executor = new VisiableThreadPoolTaskExecutor();
        //配置核心线程数 
        
        executor.setCorePoolSize(cpuNum * 2);
        //配置最大线程数
        executor.setMaxPoolSize(cpuNum * 4);
        //配置队列大小
        executor.setQueueCapacity(queueCapacity);
        //配置线程池中的线程的名称前缀
        executor.setThreadNamePrefix(namePrefix);
        // rejection-policy：当pool已经达到max size的时候，如何处理新任务
        // CALLER_RUNS：不在新线程中执行任务，而是有调用者所在的线程来执行
        executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
        //执行初始化
        executor.initialize();
        return executor;
    }
}

3. 创建异步线程 业务类

import java.util.List;
import java.util.concurrent.CountDownLatch;

public interface AsyncService {

    public void executeAsync(ZhuowangSendData sysData, CountDownLatch countDownLatch);
}

4. 实现了

import org.springframework.stereotype.Service;

import java.util.List;
import java.util.concurrent.CountDownLatch;
@Service
@Slf4j
public class AsyncServiceImpl implements AsyncService {

    @Autowired
    private ZhuowangSendDataMapper zhuowangSendDataMapper;

    @Override
    @Async("asyncServiceExecutor")
    public void executeAsync(ZhuowangSendData sysData, CountDownLatch countDownLatch) {
        try{
            //sysData实体类 步线程要做的事情、插入数据 
            zhuowangSendDataMapper.insertZhuowangSendDataMapper(sysData);
        }finally {
            countDownLatch.countDown();// 很关键, 无论上面程序是否异常必须执行countDown,否则await无法释放
        }
    }
}

5. 调用

    public int testMultiThread() {
         
        CountDownLatch countDownLatch = new CountDownLatch(lists.size());

        for (object listSub : lists) {
            //开始异步调用
            asyncService.executeAsync(listSub, countDownLatch );
        }
        try {
            countDownLatch.await(); //保证之前的所有的线程都执行完成，才会走下面的；
            // 这样就可以在下面拿到所有线程执行完的集合结果
        } catch (Exception e) {
            log.error("阻塞异常:"+e.getMessage());
        }
        return 1;
    }