Java文件打包下载处理方案及优化策略-优快云博客

Java文件打包下载处理方案

文件打包下载处理方案

1. 基础文件打包下载

使用Java内置Zip处理

@Service
public class BasicZipService {
  
    /**
     * 基础ZIP文件打包
     */
    public byte[] createZipFile(List<FileInfo> files) throws IOException {
        try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
             ZipOutputStream zos = new ZipOutputStream(baos)) {
          
            for (FileInfo fileInfo : files) {
                addFileToZip(zos, fileInfo);
            }
          
            zos.finish();
            return baos.toByteArray();
        }
    }
  
    /**
     * 添加文件到ZIP
     */
    private void addFileToZip(ZipOutputStream zos, FileInfo fileInfo) throws IOException {
        ZipEntry entry = new ZipEntry(fileInfo.getFileName());
        zos.putNextEntry(entry);
      
        try (InputStream is = new FileInputStream(fileInfo.getFilePath())) {
            byte[] buffer = new byte[1024];
            int length;
            while ((length = is.read(buffer)) > 0) {
                zos.write(buffer, 0, length);
            }
        }
      
        zos.closeEntry();
    }
}

@RestController
@RequestMapping("/api/files")
public class FileDownloadController {
  
    @Autowired
    private BasicZipService zipService;
  
    @GetMapping("/download/zip")
    public ResponseEntity<byte[]> downloadZip(@RequestParam List<String> fileIds) throws IOException {
        // 获取文件信息
        List<FileInfo> files = fileService.getFilesByIds(fileIds);
      
        // 创建ZIP文件
        byte[] zipData = zipService.createZipFile(files);
      
        // 设置响应头
        HttpHeaders headers = new HttpHeaders();
        headers.setContentType(MediaType.APPLICATION_OCTET_STREAM);
        headers.setContentDispositionFormData("attachment", "files.zip");
        headers.setContentLength(zipData.length);
      
        return new ResponseEntity<>(zipData, headers, HttpStatus.OK);
    }
}

使用Apache Commons Compress

@Service
public class CommonsCompressService {
  
    /**
     * 使用Apache Commons Compress创建ZIP
     */
    public byte[] createZipWithCommonsCompress(List<FileInfo> files) throws IOException {
        try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
             ZipArchiveOutputStream zos = new ZipArchiveOutputStream(baos)) {
          
            zos.setEncoding("UTF-8"); // 设置编码，解决中文乱码
          
            for (FileInfo fileInfo : files) {
                addFileToZipArchive(zos, fileInfo);
            }
          
            zos.finish();
            return baos.toByteArray();
        }
    }
  
    /**
     * 添加文件到ZIP归档
     */
    private void addFileToZipArchive(ZipArchiveOutputStream zos, FileInfo fileInfo) throws IOException {
        ZipArchiveEntry entry = new ZipArchiveEntry(fileInfo.getFileName());
        zos.putArchiveEntry(entry);
      
        try (InputStream is = new FileInputStream(fileInfo.getFilePath())) {
            IOUtils.copy(is, zos);
        }
      
        zos.closeArchiveEntry();
    }
}

2. 流式文件打包下载

流式ZIP创建

@Service
public class StreamingZipService {
  
    /**
     * 流式创建ZIP文件
     */
    public void createStreamingZip(List<FileInfo> files, OutputStream outputStream) throws IOException {
        try (ZipOutputStream zos = new ZipOutputStream(outputStream)) {
            zos.setLevel(Deflater.BEST_SPEED); // 设置压缩级别
          
            for (FileInfo fileInfo : files) {
                addFileToStreamingZip(zos, fileInfo);
            }
        }
    }
  
    /**
     * 流式添加文件到ZIP
     */
    private void addFileToStreamingZip(ZipOutputStream zos, FileInfo fileInfo) throws IOException {
        ZipEntry entry = new ZipEntry(fileInfo.getFileName());
        zos.putNextEntry(entry);
      
        try (InputStream is = new FileInputStream(fileInfo.getFilePath())) {
            byte[] buffer = new byte[8192]; // 使用更大的缓冲区
            int length;
            while ((length = is.read(buffer)) > 0) {
                zos.write(buffer, 0, length);
            }
        }
      
        zos.closeEntry();
    }
}

@RestController
@RequestMapping("/api/files")
public class StreamingDownloadController {
  
    @Autowired
    private StreamingZipService streamingZipService;
  
    @GetMapping("/download/streaming-zip")
    public ResponseEntity<StreamingResponseBody> downloadStreamingZip(@RequestParam List<String> fileIds) {
        List<FileInfo> files = fileService.getFilesByIds(fileIds);
      
        StreamingResponseBody responseBody = outputStream -> {
            try {
                streamingZipService.createStreamingZip(files, outputStream);
            } catch (IOException e) {
                throw new RuntimeException("文件打包失败", e);
            }
        };
      
        HttpHeaders headers = new HttpHeaders();
        headers.setContentType(MediaType.APPLICATION_OCTET_STREAM);
        headers.setContentDispositionFormData("attachment", "files.zip");
      
        return new ResponseEntity<>(responseBody, headers, HttpStatus.OK);
    }
}

3. 异步文件打包下载

异步任务处理

@Service
public class AsyncZipService {
  
    @Autowired
    private TaskExecutor taskExecutor;
  
    /**
     * 异步创建ZIP文件
     */
    public CompletableFuture<String> createZipAsync(List<FileInfo> files) {
        return CompletableFuture.supplyAsync(() -> {
            try {
                String zipPath = generateZipPath();
                createZipFile(files, zipPath);
                return zipPath;
            } catch (IOException e) {
                throw new RuntimeException("异步文件打包失败", e);
            }
        }, taskExecutor);
    }
  
    /**
     * 创建ZIP文件到指定路径
     */
    private void createZipFile(List<FileInfo> files, String zipPath) throws IOException {
        try (FileOutputStream fos = new FileOutputStream(zipPath);
             ZipOutputStream zos = new ZipOutputStream(fos)) {
          
            for (FileInfo fileInfo : files) {
                addFileToZip(zos, fileInfo);
            }
        }
    }
  
    /**
     * 生成ZIP文件路径
     */
    private String generateZipPath() {
        String timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss"));
        return "/tmp/zips/files_" + timestamp + ".zip";
    }
}

@RestController
@RequestMapping("/api/files")
public class AsyncDownloadController {
  
    @Autowired
    private AsyncZipService asyncZipService;
  
    @PostMapping("/download/async-zip")
    public ResponseEntity<Map<String, String>> createAsyncZip(@RequestBody List<String> fileIds) {
        List<FileInfo> files = fileService.getFilesByIds(fileIds);
      
        CompletableFuture<String> future = asyncZipService.createZipAsync(files);
      
        // 返回任务ID，客户端可以轮询下载状态
        String taskId = UUID.randomUUID().toString();
        taskService.registerTask(taskId, future);
      
        Map<String, String> response = new HashMap<>();
        response.put("taskId", taskId);
        response.put("status", "PROCESSING");
        response.put("message", "文件打包任务已提交，请稍后查询状态");
      
        return ResponseEntity.ok(response);
    }
  
    @GetMapping("/download/status/{taskId}")
    public ResponseEntity<Map<String, Object>> getTaskStatus(@PathVariable String taskId) {
        TaskStatus status = taskService.getTaskStatus(taskId);
      
        Map<String, Object> response = new HashMap<>();
        response.put("taskId", taskId);
        response.put("status", status.getStatus());
        response.put("progress", status.getProgress());
      
        if (status.isCompleted()) {
            response.put("downloadUrl", "/api/files/download/file/" + status.getFilePath());
        }
      
        return ResponseEntity.ok(response);
    }
}

4. 分块下载处理

分块下载实现

@Service
public class ChunkedDownloadService {
  
    /**
     * 分块下载文件
     */
    public ResponseEntity<StreamingResponseBody> downloadChunked(
            String filePath, 
            HttpServletRequest request) throws IOException {
      
        File file = new File(filePath);
        if (!file.exists()) {
            throw new FileNotFoundException("文件不存在");
        }
      
        long fileLength = file.length();
        long start = 0;
        long end = fileLength - 1;
      
        // 处理Range请求头
        String rangeHeader = request.getHeader("Range");
        if (rangeHeader != null && rangeHeader.startsWith("bytes=")) {
            String[] ranges = rangeHeader.substring(6).split("-");
            start = Long.parseLong(ranges[0]);
            if (ranges.length > 1 && !ranges[1].isEmpty()) {
                end = Long.parseLong(ranges[1]);
            }
        }
      
        long contentLength = end - start + 1;
      
        StreamingResponseBody responseBody = outputStream -> {
            try (RandomAccessFile raf = new RandomAccessFile(file, "r")) {
                raf.seek(start);
              
                byte[] buffer = new byte[8192];
                long remaining = contentLength;
              
                while (remaining > 0) {
                    int read = raf.read(buffer, 0, (int) Math.min(buffer.length, remaining));
                    if (read == -1) break;
                  
                    outputStream.write(buffer, 0, read);
                    remaining -= read;
                }
            }
        };
      
        HttpHeaders headers = new HttpHeaders();
        headers.setContentType(MediaType.APPLICATION_OCTET_STREAM);
        headers.setContentDispositionFormData("attachment", file.getName());
        headers.setContentLength(contentLength);
      
        if (rangeHeader != null) {
            headers.set("Content-Range", "bytes " + start + "-" + end + "/" + fileLength);
            headers.set("Accept-Ranges", "bytes");
            return new ResponseEntity<>(responseBody, headers, HttpStatus.PARTIAL_CONTENT);
        }
      
        return new ResponseEntity<>(responseBody, headers, HttpStatus.OK);
    }
}

大数据量文件打包处理方式

1. 分批处理策略

分批打包实现

@Service
public class BatchZipService {
  
    private static final int BATCH_SIZE = 100; // 每批处理100个文件
    private static final int MAX_MEMORY_SIZE = 100 * 1024 * 1024; // 100MB内存限制
  
    /**
     * 分批处理大文件列表
     */
    public void processLargeFileList(List<FileInfo> allFiles, String outputPath) throws IOException {
        int totalFiles = allFiles.size();
        int totalBatches = (int) Math.ceil((double) totalFiles / BATCH_SIZE);
      
        try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputPath))) {
            zos.setLevel(Deflater.BEST_SPEED);
          
            for (int batchIndex = 0; batchIndex < totalBatches; batchIndex++) {
                int startIndex = batchIndex * BATCH_SIZE;
                int endIndex = Math.min(startIndex + BATCH_SIZE, totalFiles);
              
                List<FileInfo> batch = allFiles.subList(startIndex, endIndex);
                processBatch(zos, batch, batchIndex + 1, totalBatches);
              
                // 强制刷新，释放内存
                zos.flush();
                System.gc(); // 建议垃圾回收
            }
        }
    }
  
    /**
     * 处理单个批次
     */
    private void processBatch(ZipOutputStream zos, List<FileInfo> batch, 
                            int batchNum, int totalBatches) throws IOException {
      
        log.info("处理批次 {}/{}，文件数量: {}", batchNum, totalBatches, batch.size());
      
        for (FileInfo fileInfo : batch) {
            addFileToZip(zos, fileInfo);
          
            // 检查内存使用情况
            if (isMemoryUsageHigh()) {
                log.warn("内存使用过高，强制垃圾回收");
                System.gc();
            }
        }
    }
  
    /**
     * 检查内存使用情况
     */
    private boolean isMemoryUsageHigh() {
        Runtime runtime = Runtime.getRuntime();
        long usedMemory = runtime.totalMemory() - runtime.freeMemory();
        long maxMemory = runtime.maxMemory();
      
        return (double) usedMemory / maxMemory > 0.8; // 内存使用超过80%
    }
}

内存映射文件处理

@Service
public class MemoryMappedZipService {
  
    /**
     * 使用内存映射处理大文件
     */
    public void createZipWithMemoryMapping(List<FileInfo> files, String outputPath) throws IOException {
        try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputPath))) {
            zos.setLevel(Deflater.BEST_SPEED);
          
            for (FileInfo fileInfo : files) {
                addFileWithMemoryMapping(zos, fileInfo);
            }
        }
    }
  
    /**
     * 使用内存映射添加文件
     */
    private void addFileWithMemoryMapping(ZipOutputStream zos, FileInfo fileInfo) throws IOException {
        File file = new File(fileInfo.getFilePath());
        if (!file.exists()) {
            log.warn("文件不存在: {}", fileInfo.getFilePath());
            return;
        }
      
        ZipEntry entry = new ZipEntry(fileInfo.getFileName());
        zos.putNextEntry(entry);
      
        // 使用内存映射文件，避免一次性加载到内存
        try (FileChannel channel = new FileInputStream(file).getChannel()) {
            long fileSize = channel.size();
            long position = 0;
            long chunkSize = 1024 * 1024; // 1MB块大小
          
            while (position < fileSize) {
                long remaining = fileSize - position;
                long currentChunkSize = Math.min(chunkSize, remaining);
              
                ByteBuffer buffer = channel.map(
                    FileChannel.MapMode.READ_ONLY, 
                    position, 
                    currentChunkSize
                );
              
                byte[] chunk = new byte[(int) currentChunkSize];
                buffer.get(chunk);
                zos.write(chunk);
              
                position += currentChunkSize;
              
                // 清理缓冲区
                buffer.clear();
            }
        }
      
        zos.closeEntry();
    }
}

2. 多线程并行处理

并行打包实现

@Service
public class ParallelZipService {
  
    @Autowired
    private TaskExecutor taskExecutor;
  
    private static final int THREAD_POOL_SIZE = 4;
    private static final int FILES_PER_THREAD = 50;
  
    /**
     * 并行处理文件打包
     */
    public void processParallelZip(List<FileInfo> files, String outputPath) throws IOException {
        int totalFiles = files.size();
        int threadCount = Math.min(THREAD_POOL_SIZE, (int) Math.ceil((double) totalFiles / FILES_PER_THREAD));
      
        // 创建临时文件列表
        List<String> tempZipFiles = new ArrayList<>();
      
        // 并行处理每个线程的文件
        List<CompletableFuture<String>> futures = new ArrayList<>();
      
        for (int i = 0; i < threadCount; i++) {
            int startIndex = i * FILES_PER_THREAD;
            int endIndex = Math.min(startIndex + FILES_PER_THREAD, totalFiles);
          
            if (startIndex < totalFiles) {
                List<FileInfo> threadFiles = files.subList(startIndex, endIndex);
                CompletableFuture<String> future = processThreadFiles(threadFiles, i);
                futures.add(future);
            }
        }
      
        // 等待所有线程完成
        CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
      
        // 收集临时文件
        for (CompletableFuture<String> future : futures) {
            try {
                String tempFile = future.get();
                if (tempFile != null) {
                    tempZipFiles.add(tempFile);
                }
            } catch (Exception e) {
                log.error("获取线程结果失败", e);
            }
        }
      
        // 合并所有临时ZIP文件
        mergeZipFiles(tempZipFiles, outputPath);
      
        // 清理临时文件
        cleanupTempFiles(tempZipFiles);
    }
  
    /**
     * 处理单个线程的文件
     */
    private CompletableFuture<String> processThreadFiles(List<FileInfo> files, int threadId) {
        return CompletableFuture.supplyAsync(() -> {
            try {
                String tempPath = "/tmp/zip_thread_" + threadId + "_" + 
                                System.currentTimeMillis() + ".zip";
                createZipFile(files, tempPath);
                return tempPath;
            } catch (IOException e) {
                log.error("线程 {} 处理失败", threadId, e);
                return null;
            }
        }, taskExecutor);
    }
  
    /**
     * 合并多个ZIP文件
     */
    private void mergeZipFiles(List<String> tempZipFiles, String outputPath) throws IOException {
        try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputPath))) {
            zos.setLevel(Deflater.BEST_SPEED);
          
            for (String tempZipFile : tempZipFiles) {
                mergeZipFile(zos, tempZipFile);
            }
        }
    }
  
    /**
     * 合并单个ZIP文件
     */
    private void mergeZipFile(ZipOutputStream zos, String zipFilePath) throws IOException {
        try (ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFilePath))) {
            ZipEntry entry;
            while ((entry = zis.getNextEntry()) != null) {
                // 创建新的条目，避免名称冲突
                String newName = "thread_" + entry.getName();
                ZipEntry newEntry = new ZipEntry(newName);
                zos.putNextEntry(newEntry);
              
                byte[] buffer = new byte[8192];
                int length;
                while ((length = zis.read(buffer)) > 0) {
                    zos.write(buffer, 0, length);
                }
              
                zos.closeEntry();
                zis.closeEntry();
            }
        }
    }
  
    /**
     * 清理临时文件
     */
    private void cleanupTempFiles(List<String> tempZipFiles) {
        for (String tempFile : tempZipFiles) {
            try {
                Files.deleteIfExists(Paths.get(tempFile));
            } catch (IOException e) {
                log.warn("删除临时文件失败: {}", tempFile, e);
            }
        }
    }
}

3. 流式处理大文件

流式大文件处理

@Service
public class StreamingLargeFileService {
  
    /**
     * 流式处理大文件列表
     */
    public void processLargeFilesStreaming(List<FileInfo> files, OutputStream outputStream) throws IOException {
        try (ZipOutputStream zos = new ZipOutputStream(outputStream)) {
            zos.setLevel(Deflater.BEST_SPEED);
          
            for (FileInfo fileInfo : files) {
                processLargeFileStreaming(zos, fileInfo);
            }
        }
    }
  
    /**
     * 流式处理单个大文件
     */
    private void processLargeFileStreaming(ZipOutputStream zos, FileInfo fileInfo) throws IOException {
        File file = new File(fileInfo.getFilePath());
        if (!file.exists()) {
            log.warn("文件不存在: {}", fileInfo.getFilePath());
            return;
        }
      
        ZipEntry entry = new ZipEntry(fileInfo.getFileName());
        zos.putNextEntry(entry);
      
        // 使用缓冲流处理大文件
        try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file))) {
            byte[] buffer = new byte[64 * 1024]; // 64KB缓冲区
            int length;
          
            while ((length = bis.read(buffer)) > 0) {
                zos.write(buffer, 0, length);
              
                // 定期刷新，避免内存积累
                if (length == buffer.length) {
                    zos.flush();
                }
            }
        }
      
        zos.closeEntry();
    }
}

难点分析

1. 内存管理难点

问题描述

大文件一次性加载到内存导致OutOfMemoryError
大量文件信息占用过多内存
垃圾回收不及时导致内存泄漏

解决方案

@Service
public class MemoryManagementService {
  
    /**
     * 内存友好的文件处理
     */
    public void processFilesWithMemoryManagement(List<FileInfo> files, String outputPath) throws IOException {
        // 使用软引用管理文件信息
        List<SoftReference<FileInfo>> softFileRefs = files.stream()
            .map(SoftReference::new)
            .collect(Collectors.toList());
      
        try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputPath))) {
            for (SoftReference<FileInfo> softRef : softFileRefs) {
                FileInfo fileInfo = softRef.get();
                if (fileInfo != null) {
                    processFileWithMemoryCheck(zos, fileInfo);
                }
              
                // 主动触发垃圾回收
                if (isMemoryPressureHigh()) {
                    System.gc();
                }
            }
        }
    }
  
    /**
     * 检查内存压力
     */
    private boolean isMemoryPressureHigh() {
        Runtime runtime = Runtime.getRuntime();
        long usedMemory = runtime.totalMemory() - runtime.freeMemory();
        long maxMemory = runtime.maxMemory();
      
        return (double) usedMemory / maxMemory > 0.7;
    }
  
    /**
     * 带内存检查的文件处理
     */
    private void processFileWithMemoryCheck(ZipOutputStream zos, FileInfo fileInfo) throws IOException {
        // 检查可用内存
        if (!hasEnoughMemory()) {
            log.warn("内存不足，等待垃圾回收");
            System.gc();
            Thread.sleep(100); // 短暂等待
        }
      
        addFileToZip(zos, fileInfo);
    }
  
    /**
     * 检查是否有足够内存
     */
    private boolean hasEnoughMemory() {
        Runtime runtime = Runtime.getRuntime();
        long freeMemory = runtime.freeMemory();
        return freeMemory > 50 * 1024 * 1024; // 至少50MB可用内存
    }
}

2. 性能优化难点

问题描述

大文件处理速度慢
网络传输瓶颈
磁盘I/O性能限制

解决方案

@Service
public class PerformanceOptimizationService {
  
    /**
     * 性能优化的文件处理
     */
    public void processFilesWithPerformanceOptimization(List<FileInfo> files, String outputPath) throws IOException {
        // 使用NIO提高I/O性能
        try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputPath))) {
            zos.setLevel(Deflater.BEST_SPEED); // 最快压缩级别
          
            for (FileInfo fileInfo : files) {
                processFileWithNIO(zos, fileInfo);
            }
        }
    }
  
    /**
     * 使用NIO处理文件
     */
    private void processFileWithNIO(ZipOutputStream zos, FileInfo fileInfo) throws IOException {
        ZipEntry entry = new ZipEntry(fileInfo.getFileName());
        zos.putNextEntry(entry);
      
        Path filePath = Paths.get(fileInfo.getFilePath());
        try (FileChannel channel = FileChannel.open(filePath, StandardOpenOption.READ)) {
            // 使用直接缓冲区提高性能
            ByteBuffer buffer = ByteBuffer.allocateDirect(64 * 1024); // 64KB直接缓冲区
          
            while (channel.read(buffer) != -1) {
                buffer.flip();
                byte[] data = new byte[buffer.remaining()];
                buffer.get(data);
                zos.write(data);
                buffer.clear();
            }
        }
      
        zos.closeEntry();
    }
}

3. 错误处理难点

问题描述

部分文件损坏导致整个打包失败
网络中断导致下载失败
磁盘空间不足导致写入失败

解决方案

@Service
public class ErrorHandlingService {
  
    /**
     * 带错误处理的文件打包
     */
    public void processFilesWithErrorHandling(List<FileInfo> files, String outputPath) throws IOException {
        List<FileProcessingResult> results = new ArrayList<>();
      
        try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputPath))) {
            for (FileInfo fileInfo : files) {
                try {
                    processFileWithRetry(zos, fileInfo);
                    results.add(new FileProcessingResult(fileInfo, true, null));
                } catch (Exception e) {
                    log.error("处理文件失败: {}", fileInfo.getFileName(), e);
                    results.add(new FileProcessingResult(fileInfo, false, e.getMessage()));
                  
                    // 继续处理其他文件，不中断整个流程
                    continue;
                }
            }
        }
      
        // 生成处理报告
        generateProcessingReport(results);
    }
  
    /**
     * 带重试的文件处理
     */
    private void processFileWithRetry(ZipOutputStream zos, FileInfo fileInfo) throws IOException {
        int maxRetries = 3;
        int retryCount = 0;
      
        while (retryCount < maxRetries) {
            try {
                addFileToZip(zos, fileInfo);
                return; // 成功则返回
            } catch (IOException e) {
                retryCount++;
                if (retryCount >= maxRetries) {
                    throw e; // 重试次数用完，抛出异常
                }
              
                log.warn("处理文件失败，第{}次重试: {}", retryCount, fileInfo.getFileName());
                Thread.sleep(1000 * retryCount); // 递增延迟
            }
        }
    }
  
    /**
     * 生成处理报告
     */
    private void generateProcessingReport(List<FileProcessingResult> results) {
        long successCount = results.stream().filter(FileProcessingResult::isSuccess).count();
        long failureCount = results.size() - successCount;
      
        log.info("文件处理完成 - 成功: {}, 失败: {}", successCount, failureCount);
      
        if (failureCount > 0) {
            List<FileProcessingResult> failures = results.stream()
                .filter(r -> !r.isSuccess())
                .collect(Collectors.toList());
          
            log.warn("失败的文件:");
            failures.forEach(f -> log.warn("  - {}: {}", f.getFileInfo().getFileName(), f.getErrorMessage()));
        }
    }
}

@Data
@AllArgsConstructor
public class FileProcessingResult {
    private FileInfo fileInfo;
    private boolean success;
    private String errorMessage;
}

4. 并发控制难点

问题描述

多线程访问共享资源导致数据不一致
线程池资源耗尽
死锁和竞态条件

解决方案

@Service
public class ConcurrencyControlService {
  
    private final Semaphore semaphore = new Semaphore(5); // 限制并发数
    private final ReentrantLock lock = new ReentrantLock();
  
    /**
     * 带并发控制的文件处理
     */
    public void processFilesWithConcurrencyControl(List<FileInfo> files, String outputPath) throws IOException {
        ExecutorService executor = Executors.newFixedThreadPool(3); // 限制线程池大小
      
        try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputPath))) {
            // 使用同步块保护ZIP输出流
            List<CompletableFuture<Void>> futures = files.stream()
                .map(fileInfo -> CompletableFuture.runAsync(() -> {
                    try {
                        semaphore.acquire(); // 获取信号量
                        try {
                            processFileWithLock(zos, fileInfo);
                        } finally {
                            semaphore.release(); // 释放信号量
                        }
                    } catch (Exception e) {
                        log.error("处理文件失败: {}", fileInfo.getFileName(), e);
                    }
                }, executor))
                .collect(Collectors.toList());
          
            // 等待所有任务完成
            CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
          
        } finally {
            executor.shutdown();
            try {
                if (!executor.awaitTermination(60, TimeUnit.SECONDS)) {
                    executor.shutdownNow();
                }
            } catch (InterruptedException e) {
                executor.shutdownNow();
                Thread.currentThread().interrupt();
            }
        }
    }
  
    /**
     * 带锁的文件处理
     */
    private void processFileWithLock(ZipOutputStream zos, FileInfo fileInfo) throws IOException {
        lock.lock();
        try {
            addFileToZip(zos, fileInfo);
        } finally {
            lock.unlock();
        }
    }
}

最佳实践

1. 配置优化

应用配置

# application.yml
file:
  download:
    # 文件处理配置
    batch-size: 100
    max-memory-usage: 80
    compression-level: 1
    buffer-size: 8192
  
    # 线程池配置
    thread-pool:
      core-size: 4
      max-size: 8
      queue-capacity: 100
      keep-alive-seconds: 60
  
    # 超时配置
    timeout:
      connection: 30000
      read: 60000
      write: 60000
  
    # 缓存配置
    cache:
      enabled: true
      max-size: 1000
      expire-after-write: 3600

配置类

@Configuration
@ConfigurationProperties(prefix = "file.download")
@Data
public class FileDownloadConfig {
  
    private int batchSize = 100;
    private int maxMemoryUsage = 80;
    private int compressionLevel = 1;
    private int bufferSize = 8192;
  
    private ThreadPoolConfig threadPool = new ThreadPoolConfig();
    private TimeoutConfig timeout = new TimeoutConfig();
    private CacheConfig cache = new CacheConfig();
  
    @Data
    public static class ThreadPoolConfig {
        private int coreSize = 4;
        private int maxSize = 8;
        private int queueCapacity = 100;
        private int keepAliveSeconds = 60;
    }
  
    @Data
    public static class TimeoutConfig {
        private int connection = 30000;
        private int read = 60000;
        private int write = 60000;
    }
  
    @Data
    public static class CacheConfig {
        private boolean enabled = true;
        private int maxSize = 1000;
        private int expireAfterWrite = 3600;
    }
}

2. 监控和日志

监控实现

@Component
public class FileDownloadMonitor {
  
    private final MeterRegistry meterRegistry;
    private final Counter successCounter;
    private final Counter failureCounter;
    private final Timer processingTimer;
    private final Gauge memoryGauge;
  
    public FileDownloadMonitor(MeterRegistry meterRegistry) {
        this.meterRegistry = meterRegistry;
        this.successCounter = Counter.builder("file.download.success")
            .description("文件下载成功次数")
            .register(meterRegistry);
        this.failureCounter = Counter.builder("file.download.failure")
            .description("文件下载失败次数")
            .register(meterRegistry);
        this.processingTimer = Timer.builder("file.download.processing.time")
            .description("文件处理时间")
            .register(meterRegistry);
        this.memoryGauge = Gauge.builder("file.download.memory.usage")
            .description("内存使用率")
            .register(meterRegistry, this, FileDownloadMonitor::getMemoryUsage);
    }
  
    public void recordSuccess() {
        successCounter.increment();
    }
  
    public void recordFailure() {
        failureCounter.increment();
    }
  
    public Timer.Sample startTimer() {
        return Timer.start(meterRegistry);
    }
  
    public void stopTimer(Timer.Sample sample) {
        sample.stop(processingTimer);
    }
  
    private double getMemoryUsage() {
        Runtime runtime = Runtime.getRuntime();
        long usedMemory = runtime.totalMemory() - runtime.freeMemory();
        long maxMemory = runtime.maxMemory();
        return (double) usedMemory / maxMemory * 100;
    }
}

日志配置

<!-- logback-spring.xml -->
<configuration>
    <appender name="FILE_DOWNLOAD" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <file>logs/file-download.log</file>
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <fileNamePattern>logs/file-download.%d{yyyy-MM-dd}.log</fileNamePattern>
            <maxHistory>30</maxHistory>
        </rollingPolicy>
        <encoder>
            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
        </encoder>
    </appender>
  
    <logger name="com.iflytek.enrollment.file" level="INFO" additivity="false">
        <appender-ref ref="FILE_DOWNLOAD"/>
        <appender-ref ref="CONSOLE"/>
    </logger>
</configuration>

性能优化

1. 压缩算法优化

压缩级别选择

@Service
public class CompressionOptimizationService {
  
    /**
     * 根据文件类型选择最佳压缩级别
     */
    public int getOptimalCompressionLevel(String fileName) {
        String extension = getFileExtension(fileName).toLowerCase();
      
        switch (extension) {
            case "txt":
            case "log":
            case "csv":
                return Deflater.BEST_COMPRESSION; // 文本文件使用最高压缩
            case "jpg":
            case "png":
            case "gif":
            case "pdf":
                return Deflater.NO_COMPRESSION; // 图片和PDF不压缩
            case "zip":
            case "rar":
            case "7z":
                return Deflater.NO_COMPRESSION; // 已压缩文件不重复压缩
            default:
                return Deflater.BEST_SPEED; // 其他文件使用最快压缩
        }
    }
  
    /**
     * 获取文件扩展名
     */
    private String getFileExtension(String fileName) {
        int lastDotIndex = fileName.lastIndexOf('.');
        return lastDotIndex > 0 ? fileName.substring(lastDotIndex + 1) : "";
    }
}

2. 缓存策略优化

文件信息缓存

@Service
public class FileCacheService {
  
    @Autowired
    private CacheManager cacheManager;
  
    /**
     * 缓存文件信息
     */
    public FileInfo getCachedFileInfo(String fileId) {
        Cache cache = cacheManager.getCache("fileInfo");
        Cache.ValueWrapper wrapper = cache.get(fileId);
      
        if (wrapper != null) {
            return (FileInfo) wrapper.get();
        }
      
        // 从数据库获取并缓存
        FileInfo fileInfo = fileRepository.findById(fileId);
        if (fileInfo != null) {
            cache.put(fileId, fileInfo);
        }
      
        return fileInfo;
    }
  
    /**
     * 批量缓存文件信息
     */
    public List<FileInfo> getCachedFileInfos(List<String> fileIds) {
        return fileIds.parallelStream()
            .map(this::getCachedFileInfo)
            .filter(Objects::nonNull)
            .collect(Collectors.toList());
    }
}

3. 网络传输优化

分块传输

@Service
public class ChunkedTransferService {
  
    private static final int CHUNK_SIZE = 1024 * 1024; // 1MB块大小
  
    /**
     * 分块传输大文件
     */
    public void transferFileInChunks(String filePath, OutputStream outputStream) throws IOException {
        File file = new File(filePath);
        long fileSize = file.length();
      
        try (FileInputStream fis = new FileInputStream(file)) {
            byte[] buffer = new byte[CHUNK_SIZE];
            long transferred = 0;
          
            while (transferred < fileSize) {
                int bytesRead = fis.read(buffer);
                if (bytesRead == -1) break;
              
                outputStream.write(buffer, 0, bytesRead);
                outputStream.flush();
              
                transferred += bytesRead;
              
                // 记录传输进度
                log.debug("文件传输进度: {}/{} bytes ({}%)", 
                    transferred, fileSize, (transferred * 100 / fileSize));
            }
        }
    }
}