【全栈】大文件切片上传实现进度条，关闭浏览器再次打开继续上传（不需要再次选择文件）

Zacks_xdc

已于 2025-04-09 09:00:23 修改

阅读量1.3k

点赞数 44

CC 4.0 BY-SA版权

文章标签：状态模式前端 javascript html5 express 后端

于 2025-04-08 17:39:48 首次发布

本文链接：https://blog.youkuaiyun.com/Zacks_xdc/article/details/147067395

项目场景：

客户对上传文件功能提出了一个需求：关闭浏览器再次打开系统时，能够继续上传上次没有上传完的文件，而且希望不需要选择原文件了。

问题描述

之前项目的切片断点续传是要重新选择文件才能继续上传，本次的需求就无法满足，于是通过使用 IndexedDB 的来记录文件信息，以便能够不需要重新选择文件。

一，存储方式

文件存在用户本地设备
1. 源文件始终存储在用户的本地设备（如电脑、手机）中，前端通过浏览器的 File API 直接读取文件内容。
2. 无需将文件存储到服务器或浏览器缓存，仅上传分块数据。
前端状态存储
1. 文件元数据（如文件名、总大小）和上传状态（已上传块索引）存储在浏览器的 IndexedDB 中。
2. 示例存储结构：
```
{
  "uploadId": "unique-task-id",
  "fileName": "video.mp4", 
  "fileSize": 10485760,
  "uploadedChunks": [0, 1, 2]
}
```

二，恢复上传的实现逻辑

页面加载时自动检测
1. 读取本地存储的 uploadId 和 uploadedChunks，结合文件元数据，计算未上传的块范围。
2. 无需用户重新选择文件，直接通过 File API 读取本地文件的剩余块。
分块上传机制
1. 前端使用 File.slice(start, end) 动态读取文件的未上传部分。
```
const chunk = file.slice(start,  end); // 动态读取文件块 
```
2. 服务端按 uploadId 存储临时块文件（如 uploads/{uploadId}/{blockNumber}.part），合并时生成完整文件。

三，关键优势

无需用户干预
- 用户关闭浏览器后，再次打开页面时自动恢复上传，无需重新选择文件。
安全与隐私
- 源文件仅存在于用户本地设备，未上传的块数据不会泄露到服务器。
高效性
- 仅上传未完成的块，减少网络流量和服务器负载。

解决方案：

项目代码无法拿出来，实现了一个最小可用版本。需要注意的是如果用户更换了浏览器或者电脑就无法继续上传了

先启动node服务，在terminal中输入以下代码，控制台出现文件上传服务已启动: http://localhost:3001就是启动成功了。

> cd server
> node uploadServer.js

使用浏览器打开index.html，即可查看效果。

项目依赖如下（package.json）：

{
  "dependencies": {
    "axios": "^1.8.4",
    "cors": "^2.8.5",
    "express": "^5.1.0",
    "fs-extra": "^11.3.0",
    "multer": "^1.4.5-lts.2",
    "spark-md5": "^3.0.2"
  }
}

目录结构如下：

在这里插入图片描述

源码如下：

index.html

<!DOCTYPE html>
<html lang="zh-CN">
  <head>
    <meta charset="UTF-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
    <title>大文件上传</title>
    <!-- <script src="https://cdn.jsdelivr.net/npm/spark-md5@3.0.2/spark-md5.min.js"></script> -->
    <!-- <script src="https://cdn.jsdelivr.net/npm/axios/dist/axios.min.js"></script> -->
    <script src="https://cdn.bootcdn.net/ajax/libs/spark-md5/3.0.2/spark-md5.min.js"></script>
    <script src="https://cdn.bootcdn.net/ajax/libs/axios/1.8.4/axios.min.js"></script>
    <style>
      .container {
        margin: 20px;
        padding: 20px;
        border: 1px solid #ddd;
        border-radius: 5px;
        max-width: 500px;
      }
      .progress-container {
        margin-top: 10px;
      }
      button:disabled {
        opacity: 0.5;
      }
    </style>
  </head>
  <body>
    <div class="container">
      <input type="file" id="fileInput" />
      <button id="uploadBtn">上传</button>
      <div
        class="progress-container"
        id="progressContainer"
        style="display: none"
      >
        <progress id="progressBar" value="0" max="100"></progress>
        <span id="progressText">0%</span>
      </div>
    </div>
    <script src="src/upload.js"></script>
  </body>
</html>

upload.js

// 配置常量
const CONFIG = {
  CHUNK_SIZE: 1 * 1024 * 1024, // 1MB
  API_BASE_URL: "http://localhost:3001",
  DB_NAME: "FileUploadDB",
  STORE_NAME: "files",
  RETRY_DELAY: 1000, // 1秒
  MAX_RETRIES: 3,
};

// DOM 元素
const DOM = {
  fileInput: document.getElementById("fileInput"),
  uploadBtn: document.getElementById("uploadBtn"),
  progressContainer: document.getElementById("progressContainer"),
  progressBar: document.getElementById("progressBar"),
  progressText: document.getElementById("progressText"),
};

// 状态管理
const state = {
  file: null,
  uploadedChunks: JSON.parse(localStorage.getItem("uploadedChunks") || "[]"),
  fileHash: localStorage.getItem("fileHash") || "",
  chunks: [],
  fileInfo: JSON.parse(localStorage.getItem("fileInfo") || "null"),
  db: null,
};

// IndexedDB 操作
const dbOperations = {
  async initDB() {
    return new Promise((resolve, reject) => {
      const request = indexedDB.open(CONFIG.DB_NAME, 1);

      request.onerror = () => reject(request.error);
      request.onsuccess = () => {
        state.db = request.result;
        resolve(state.db);
      };

      request.onupgradeneeded = (event) => {
        const db = event.target.result;
        if (!db.objectStoreNames.contains(CONFIG.STORE_NAME)) {
          db.createObjectStore(CONFIG.STORE_NAME);
        }
      };
    });
  },

  async saveFile(file, hash) {
    return new Promise((resolve, reject) => {
      const transaction = state.db.transaction(
        [CONFIG.STORE_NAME],
        "readwrite"
      );
      const store = transaction.objectStore(CONFIG.STORE_NAME);
      const request = store.put(file, hash);

      request.onsuccess = () => resolve();
      request.onerror = () => reject(request.error);
    });
  },

  async getFile(hash) {
    return new Promise((resolve, reject) => {
      const transaction = state.db.transaction([CONFIG.STORE_NAME], "readonly");
      const store = transaction.objectStore(CONFIG.STORE_NAME);
      const request = store.get(hash);

      request.onsuccess = () => resolve(request.result);
      request.onerror = () => reject(request.error);
    });
  },

  async deleteFile(hash) {
    return new Promise((resolve, reject) => {
      const transaction = state.db.transaction(
        [CONFIG.STORE_NAME],
        "readwrite"
      );
      const store = transaction.objectStore(CONFIG.STORE_NAME);
      const request = store.delete(hash);

      request.onsuccess = () => resolve();
      request.onerror = () => reject(request.error);
    });
  },
};

// 进度管理
const progressManager = {
  save(uploadedCount, totalChunks) {
    localStorage.setItem(
      "uploadProgress",
      JSON.stringify({
        uploadedCount,
        totalChunks,
        timestamp: Date.now(),
      })
    );
  },

  get() {
    const progress = localStorage.getItem("uploadProgress");
    return progress ? JSON.parse(progress) : null;
  },

  clear() {
    localStorage.removeItem("uploadProgress");
  },

  update(uploaded, total) {
    const percent = Math.floor((uploaded / total) * 100);
    DOM.progressBar.value = percent;
    DOM.progressText.textContent = `${percent}% (${uploaded}/${total})`;
  },
};

// 存储管理
const storageManager = {
  async clear() {
    if (state.fileHash) {
      await dbOperations.deleteFile(state.fileHash);
    }
    localStorage.removeItem("uploadedChunks");
    localStorage.removeItem("fileHash");
    localStorage.removeItem("fileInfo");
    progressManager.clear();
    state.fileHash = "";
    state.fileInfo = null;
    state.uploadedChunks = [];
  },
};

// 文件操作
const fileOperations = {
  createChunks(file) {
    const chunks = [];
    let current = 0;
    while (current < file.size) {
      chunks.push({
        file: file.slice(current, current + CONFIG.CHUNK_SIZE),
        index: chunks.length,
      });
      current += CONFIG.CHUNK_SIZE;
    }
    return chunks;
  },

  async calculateHash(file) {
    return new Promise((resolve) => {
      const spark = new SparkMD5.ArrayBuffer();
      const reader = new FileReader();

      reader.onload = (e) => {
        spark.append(e.target.result);
        resolve(spark.end());
      };

      reader.readAsArrayBuffer(file);
    });
  },
};

// API 操作
const apiOperations = {
  async verifyFile(filename, hash) {
    try {
      const { data } = await axios.post(`${CONFIG.API_BASE_URL}/verify`, {
        filename,
        hash,
      });
      return data;
    } catch (error) {
      throw new Error(error.response?.data?.error || error.message);
    }
  },

  async uploadChunk(chunk, hash, filename, index, chunkName) {
    const formData = new FormData();
    formData.append("chunk", chunk);
    formData.append("hash", hash);
    formData.append("filename", filename);
    formData.append("index", index);
    formData.append("chunkName", chunkName);

    try {
      const response = await axios.post(
        `${CONFIG.API_BASE_URL}/upload`,
        formData
      );
      return response.data;
    } catch (error) {
      throw new Error(error.response?.data?.error || error.message);
    }
  },

  async mergeFile(filename, size, hash, chunks) {
    try {
      const response = await axios.post(`${CONFIG.API_BASE_URL}/merge`, {
        filename,
        size,
        hash,
        chunks,
      });
      return response.data;
    } catch (error) {
      throw new Error(error.response?.data?.error || error.message);
    }
  },
};

// 事件处理
const eventHandlers = {
  async handleFileSelect(e) {
    state.file = e.target.files[0];
    const currentFileHash = await fileOperations.calculateHash(state.file);

    await dbOperations.saveFile(state.file, currentFileHash);

    if (state.fileInfo && state.fileHash) {
      if (currentFileHash === state.fileHash) {
        const shouldContinue = confirm(
          `检测到未完成的上传，是否继续上传"${state.fileInfo.name}"？`
        );
        if (shouldContinue) {
          DOM.uploadBtn.click();
          return;
        }
      }
      await storageManager.clear();
    }

    state.fileHash = currentFileHash;
    state.fileInfo = {
      name: state.file.name,
      size: state.file.size,
      type: state.file.type,
      lastModified: state.file.lastModified,
    };

    localStorage.setItem("fileHash", state.fileHash);
    localStorage.setItem("fileInfo", JSON.stringify(state.fileInfo));
  },

  async handleUpload() {
    if (!state.file && state.fileInfo) {
      alert("请重新选择相同的文件以继续上传");
      return;
    }

    DOM.uploadBtn.disabled = true;
    DOM.fileInput.disabled = true;
    DOM.progressContainer.style.display = "block";

    try {
      const shouldUpload = await apiOperations.verifyFile(
        state.file.name,
        state.fileHash
      );
      if (!shouldUpload) {
        DOM.uploadBtn.disabled = false;
        DOM.fileInput.disabled = false;
        return;
      }

      state.chunks = fileOperations.createChunks(state.file);
      let uploadedCount = state.uploadedChunks.length;

      progressManager.update(uploadedCount, state.chunks.length);
      progressManager.save(uploadedCount, state.chunks.length);

      let uploadFailed = false;

      for (let i = 0; i < state.chunks.length; i++) {
        const chunk = state.chunks[i];
        const chunkName = `${state.fileHash}-${chunk.index}`;

        if (state.uploadedChunks.includes(chunkName)) {
          continue;
        }

        try {
          await apiOperations.uploadChunk(
            chunk.file,
            state.fileHash,
            state.file.name,
            chunk.index,
            chunkName
          );

          state.uploadedChunks.push(chunkName);
          localStorage.setItem(
            "uploadedChunks",
            JSON.stringify(state.uploadedChunks)
          );
          uploadedCount++;
          progressManager.update(uploadedCount, state.chunks.length);
          progressManager.save(uploadedCount, state.chunks.length);
        } catch (err) {
          uploadFailed = true;
          console.error("上传失败:", err);
          alert(`上传失败: ${err.message}`);
          break;
        }
      }

      if (!uploadFailed && uploadedCount === state.chunks.length) {
        await eventHandlers.handleMerge();
      } else if (!uploadFailed) {
        alert("还有分片未上传完成，请等待上传完成后再合并");
      }
    } catch (err) {
      console.error("上传过程出错:", err);
      alert(`上传过程出错: ${err.message}`);
    } finally {
      DOM.uploadBtn.disabled = false;
      DOM.fileInput.disabled = false;
    }
  },

  async handleMerge() {
    try {
      const verifyResponse = await apiOperations.verifyFile(
        state.file.name,
        state.fileHash
      );

      if (verifyResponse.shouldUpload) {
        await new Promise((resolve) => setTimeout(resolve, CONFIG.RETRY_DELAY));

        const mergeResponse = await apiOperations.mergeFile(
          state.file.name,
          CONFIG.CHUNK_SIZE,
          state.fileHash,
          state.chunks.length
        );

        if (mergeResponse.success) {
          await eventHandlers.handleUploadComplete();
        } else {
          throw new Error(mergeResponse.error || "合并文件失败");
        }
      } else {
        throw new Error("文件验证失败，请重试");
      }
    } catch (err) {
      console.error("合并文件失败:", err);
      const retry = confirm("合并失败，是否重试？");
      if (retry) {
        await eventHandlers.retryMerge();
      }
    }
  },

  async retryMerge() {
    try {
      const mergeResponse = await apiOperations.mergeFile(
        state.file.name,
        CONFIG.CHUNK_SIZE,
        state.fileHash,
        state.chunks.length
      );

      if (mergeResponse.success) {
        await eventHandlers.handleUploadComplete();
      } else {
        throw new Error(mergeResponse.error || "合并文件失败");
      }
    } catch (err) {
      console.error("重试合并失败:", err);
      alert(`重试合并失败: ${err.message}`);
    }
  },

  async handleUploadComplete() {
    alert("上传完成");
    await storageManager.clear();
  },
};

// 初始化
async function init() {
  await dbOperations.initDB();

  if (state.fileInfo && state.fileHash) {
    try {
      state.file = await dbOperations.getFile(state.fileHash);
      if (state.file) {
        state.uploadedChunks = JSON.parse(
          localStorage.getItem("uploadedChunks") || "[]"
        );
        state.chunks = fileOperations.createChunks(state.file);

        const uploadedCount = state.uploadedChunks.length;
        const percent = Math.floor((uploadedCount / state.chunks.length) * 100);

        let progressMessage = `检测到未完成的文件 "${state.fileInfo.name}"，已上传 ${percent}%`;
        progressMessage += "，是否继续上传？";

        const shouldContinue = confirm(progressMessage);
        if (shouldContinue) {
          DOM.progressContainer.style.display = "block";
          progressManager.update(uploadedCount, state.chunks.length);
          progressManager.save(uploadedCount, state.chunks.length);
          DOM.uploadBtn.click();
          return;
        }
      }
    } catch (error) {
      console.error("恢复文件失败:", error);
    }
    await storageManager.clear();
  }
}

// 事件监听
DOM.fileInput.addEventListener("change", eventHandlers.handleFileSelect);
DOM.uploadBtn.addEventListener("click", eventHandlers.handleUpload);

// 启动应用
init().catch(console.error);

uploadServer.js

const express = require("express");
const multer = require("multer");
const fs = require("fs");
const path = require("path");
const cors = require("cors");
const fsExtra = require("fs-extra");

const app = express();
app.use(
  cors({
    origin: "*",
    methods: ["GET", "POST", "PUT", "DELETE"],
    allowedHeaders: ["Content-Type", "Authorization"],
  })
);
// Add these middleware for body parsing
app.use(express.json());
const PORT = 3001;

// 确保上传目录存在
const UPLOAD_DIR = path.resolve(__dirname, "uploads");
if (!fs.existsSync(UPLOAD_DIR)) {
  fs.mkdirSync(UPLOAD_DIR);
}

// 修改存储配置
const storage = multer.diskStorage({
  destination: (req, file, cb) => {
    // 从文件名中获取 hash（因为文件名包含了 hash）
    const hash = file.originalname.split("-")[0];
    const chunkDir = path.resolve(UPLOAD_DIR, hash);
    if (!fs.existsSync(chunkDir)) {
      fs.mkdirSync(chunkDir);
    }
    cb(null, chunkDir);
  },
  filename: (req, file, cb) => {
    // 直接使用原始文件名
    cb(null, file.originalname);
  },
});

// 添加日志函数
function log(...args) {
  const timestamp = new Date().toISOString();
  const logMessage = `[${timestamp}] ${args.join(" ")}`;
  console.log("\x1b[36m%s\x1b[0m", logMessage); // 使用青色输出
  // 同时写入文件
  fs.appendFileSync("server.log", logMessage + "\n");
}

// 检查文件是否已存在
app.post("/verify", (req, res) => {
  const { filename, hash } = req.body;
  const filePath = path.resolve(UPLOAD_DIR, `${hash}${path.extname(filename)}`);

  if (fs.existsSync(filePath)) {
    return res.json({ shouldUpload: false });
  }

  const chunkDir = path.resolve(UPLOAD_DIR, hash);
  const uploadedList = fs.existsSync(chunkDir) ? fs.readdirSync(chunkDir) : [];

  res.json({ shouldUpload: true, uploadedList });
});

const upload = multer({
  storage, // 使用之前定义的diskStorage
  limits: {
    fileSize: 20 * 1024 * 1024, // 20MB
  },
});

app.post("/upload", upload.single("chunk"), async (req, res) => {
  try {
    const { hash, filename, index } = req.body;
    const chunkName = `${hash}-${index}`;
    const chunkDir = path.resolve(UPLOAD_DIR, hash);
    const chunkPath = path.resolve(chunkDir, chunkName);

    log("上传请求参数:", { hash, filename, index, chunkName });
    log("分片目录:", chunkDir);
    log("分片路径:", chunkPath);

    // 确保分片目录存在
    await fsExtra.ensureDir(chunkDir);
    log("分片目录已创建/确认");

    // 移动上传的分片到目标位置
    await fsExtra.move(req.file.path, chunkPath, { overwrite: true });
    log("分片已保存:", chunkName);

    res.json({
      success: true,
      message: "分片上传成功",
    });
  } catch (error) {
    log("上传过程出错:", error);
    res.json({
      success: false,
      error: error.message,
    });
  }
});

// 添加重试函数
async function retryOperation(operation, maxRetries = 3, delay = 1000) {
  for (let i = 0; i < maxRetries; i++) {
    try {
      return await operation();
    } catch (error) {
      if (i === maxRetries - 1) throw error;
      log(
        `操作失败，${delay}ms 后重试 (${i + 1}/${maxRetries}):`,
        error.message
      );
      await new Promise((resolve) => setTimeout(resolve, delay));
    }
  }
}

// 修改合并分片的函数
async function mergeFileChunks(filePath, chunkDir, chunkSize) {
  return new Promise((resolve, reject) => {
    const writeStream = fs.createWriteStream(filePath);
    const chunks = fs.readdirSync(chunkDir).sort((a, b) => {
      const indexA = parseInt(a.split("-")[1]);
      const indexB = parseInt(b.split("-")[1]);
      return indexA - indexB;
    });

    let currentChunk = 0;
    const totalChunks = chunks.length;

    function writeNextChunk() {
      if (currentChunk >= totalChunks) {
        writeStream.end();
        resolve();
        return;
      }

      const chunkPath = path.join(chunkDir, chunks[currentChunk]);
      const readStream = fs.createReadStream(chunkPath);

      readStream.pipe(writeStream, { end: false });
      readStream.on("end", () => {
        currentChunk++;
        writeNextChunk();
      });
      readStream.on("error", (err) => {
        writeStream.end();
        reject(err);
      });
    }

    writeStream.on("error", (err) => {
      reject(err);
    });

    writeNextChunk();
  });
}

// 修改合并路由
app.post("/merge", async (req, res) => {
  try {
    const { filename, size, hash } = req.body;
    // 使用原始文件名，而不是hash
    const filePath = path.resolve(UPLOAD_DIR, filename);
    const chunkDir = path.resolve(UPLOAD_DIR, hash);

    log("合并请求参数:", { filename, size, hash });
    log("文件路径:", filePath);
    log("分片目录:", chunkDir);

    if (!(await fsExtra.exists(chunkDir))) {
      log("错误: 分片目录不存在:", chunkDir);
      return res.json({
        success: false,
        error: "分片目录不存在，请先上传分片",
      });
    }

    const chunks = await fsExtra.readdir(chunkDir);
    log("找到的分片数量:", chunks.length);
    log("分片列表:", chunks);

    if (chunks.length === 0) {
      log("错误: 没有找到任何分片文件");
      return res.json({
        success: false,
        error: "没有找到任何分片文件",
      });
    }

    // 等待一段时间确保文件写入完成
    await new Promise((resolve) => setTimeout(resolve, 1000));

    // 使用重试机制合并文件
    await retryOperation(async () => {
      await mergeFileChunks(filePath, chunkDir, size);
      log("文件合并成功");
    });

    // 删除切片文件
    try {
      for (const chunk of chunks) {
        const chunkPath = path.join(chunkDir, chunk);
        await fsExtra.remove(chunkPath);
        log("删除切片文件:", chunkPath);
      }
      // 删除切片目录
      await fsExtra.remove(chunkDir);
      log("删除切片目录:", chunkDir);
    } catch (error) {
      log("删除切片文件失败:", error);
      // 即使删除失败也继续返回成功响应，因为文件已经合并成功
    }

    // 只发送一次响应
    return res.json({
      success: true,
      message: "文件合并成功",
    });
  } catch (error) {
    log("合并过程出错:", error);
    return res.json({
      success: false,
      error: error.message,
    });
  }
});

// 修改服务器启动代码
const server = app.listen(PORT, "0.0.0.0", () => {
  console.log(`文件上传服务已启动: http://localhost:${PORT}`);
});

server.on("error", (err) => {
  console.error("服务器启动失败:", err);
  if (err.code === "EADDRINUSE") {
    console.log(`端口 ${PORT} 已被占用，尝试使用其他端口`);
    // 自动切换到备用端口
    app.listen(3002, "0.0.0.0", () => {
      console.log(`服务运行在备用端口: 3002`);
    });
  }
});