Java实现多线程上传文件

原创于 2025-11-03 21:08:56 发布 · 273 阅读

3 ·

CC 4.0 BY-SA版权

文章标签：

#java #python #windows

Java 专栏收录该内容

40 篇文章

订阅专栏

以下是实现多线程上传文件的几种方式：

1. 使用 CompletableFuture 实现（推荐）

private void uploadFileBatch(String id, MultipartFile[] files, String type, SysUser user) throws IOException {
    long st = System.currentTimeMillis();
    log.info("目标管理 开始上传文件 : {}", id);
    
    // 创建 CompletableFuture 列表
    List<CompletableFuture<Void>> futures = Arrays.stream(files)
            .map(file -> CompletableFuture.runAsync(() -> {
                try {
                    uploadFile(id, file, type, user);
                } catch (IOException e) {
                    log.error("文件上传失败: {}", file.getOriginalFilename(), e);
                    throw new RuntimeException(e);
                }
            }))
            .collect(Collectors.toList());
    
    // 等待所有任务完成
    CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]))
            .join();
    
    long ed = System.currentTimeMillis();
    log.info("上传文件到MinIO成功, 耗时 {} s", (ed - st) / 1000);
}

2. 使用线程池 + CountDownLatch

// 在类中定义线程池
private final ExecutorService uploadExecutor = Executors.newFixedThreadPool(10);

private void uploadFileBatch(String id, MultipartFile[] files, String type, SysUser user) throws IOException {
    long st = System.currentTimeMillis();
    log.info("目标管理 开始上传文件 : {}", id);
    
    CountDownLatch latch = new CountDownLatch(files.length);
    
    for (MultipartFile file : files) {
        uploadExecutor.submit(() -> {
            try {
                uploadFile(id, file, type, user);
            } catch (IOException e) {
                log.error("文件上传失败: {}", file.getOriginalFilename(), e);
            } finally {
                latch.countDown();
            }
        });
    }
    
    try {
        // 等待所有任务完成，设置超时时间
        boolean completed = latch.await(5, TimeUnit.MINUTES);
        if (!completed) {
            log.warn("文件上传超时，可能还有任务未完成");
        }
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        log.error("文件上传被中断", e);
    }
    
    long ed = System.currentTimeMillis();
    log.info("上传文件到MinIO成功, 耗时 {} s", (ed - st) / 1000);
}

3. 使用 Spring 的 @Async 注解

3.1 配置异步支持

@Configuration
@EnableAsync
public class AsyncConfig {
    
    @Bean("uploadTaskExecutor")
    public Executor uploadTaskExecutor() {
        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
        executor.setCorePoolSize(5);
        executor.setMaxPoolSize(10);
        executor.setQueueCapacity(100);
        executor.setThreadNamePrefix("upload-thread-");
        executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
        executor.setWaitForTasksToCompleteOnShutdown(true);
        executor.setAwaitTerminationSeconds(60);
        executor.initialize();
        return executor;
    }
}

3.2 创建异步服务

@Service
public class AsyncUploadService {
    
    @Async("uploadTaskExecutor")
    public CompletableFuture<Void> uploadFileAsync(String id, MultipartFile file, String type, SysUser user) {
        try {
            // 这里调用你的 uploadFile 方法
            // uploadFile(id, file, type, user);
            return CompletableFuture.completedFuture(null);
        } catch (Exception e) {
            log.error("异步上传文件失败: {}", file.getOriginalFilename(), e);
            return CompletableFuture.failedFuture(e);
        }
    }
}

3.3 在批量上传中使用

@Autowired
private AsyncUploadService asyncUploadService;

private void uploadFileBatch(String id, MultipartFile[] files, String type, SysUser user) throws IOException {
    long st = System.currentTimeMillis();
    log.info("目标管理 开始上传文件 : {}", id);
    
    List<CompletableFuture<Void>> futures = Arrays.stream(files)
            .map(file -> asyncUploadService.uploadFileAsync(id, file, type, user))
            .collect(Collectors.toList());
    
    // 等待所有任务完成
    CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]))
            .exceptionally(ex -> {
                log.error("部分文件上传失败", ex);
                return null;
            })
            .join();
    
    long ed = System.currentTimeMillis();
    log.info("上传文件到MinIO成功, 耗时 {} s", (ed - st) / 1000);
}

4. 带进度监控的多线程上传

private void uploadFileBatch(String id, MultipartFile[] files, String type, SysUser user) throws IOException {
    long st = System.currentTimeMillis();
    log.info("目标管理 开始上传文件 : {}", id);
    
    // 使用线程安全的计数器
    AtomicInteger successCount = new AtomicInteger(0);
    AtomicInteger failCount = new AtomicInteger(0);
    CountDownLatch latch = new CountDownLatch(files.length);
    
    List<CompletableFuture<Void>> futures = Arrays.stream(files)
            .map(file -> CompletableFuture.runAsync(() -> {
                try {
                    uploadFile(id, file, type, user);
                    successCount.incrementAndGet();
                    log.info("文件上传成功: {}", file.getOriginalFilename());
                } catch (Exception e) {
                    failCount.incrementAndGet();
                    log.error("文件上传失败: {}", file.getOriginalFilename(), e);
                } finally {
                    latch.countDown();
                }
            }))
            .collect(Collectors.toList());
    
    try {
        // 等待所有任务完成
        CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]))
                .orTimeout(10, TimeUnit.MINUTES) // 设置超时
                .join();
    } catch (Exception e) {
        log.error("批量上传出现异常", e);
    }
    
    long ed = System.currentTimeMillis();
    log.info("上传文件到MinIO成功, 成功: {}个, 失败: {}个, 总耗时: {}s", 
            successCount.get(), failCount.get(), (ed - st) / 1000);
}

5. 完整的导入包列表

import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service;

import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;