项目场景:
客户对上传文件功能提出了一个需求:关闭浏览器再次打开系统时,能够继续上传上次没有上传完的文件,而且希望不需要选择原文件了。
问题描述
之前项目的切片断点续传是要重新选择文件才能继续上传,本次的需求就无法满足,于是通过使用 IndexedDB 的来记录文件信息,以便能够不需要重新选择文件。
一,存储方式
- 文件存在用户本地设备
- 源文件始终存储在用户的本地设备(如电脑、手机)中,前端通过浏览器的 File API 直接读取文件内容。
- 无需将文件存储到服务器或浏览器缓存,仅上传分块数据。
- 前端状态存储
- 文件元数据(如文件名、总大小)和 上传状态(已上传块索引)存储在浏览器的 IndexedDB 中。
- 示例存储结构:
{ "uploadId": "unique-task-id", "fileName": "video.mp4", "fileSize": 10485760, "uploadedChunks": [0, 1, 2] }
二,恢复上传的实现逻辑
- 页面加载时自动检测
- 读取本地存储的 uploadId 和 uploadedChunks,结合文件元数据,计算未上传的块范围。
- 无需用户重新选择文件,直接通过 File API 读取本地文件的剩余块。
- 分块上传机制
- 前端使用 File.slice(start, end) 动态读取文件的未上传部分。
const chunk = file.slice(start, end); // 动态读取文件块
- 服务端按 uploadId 存储临时块文件(如 uploads/{uploadId}/{blockNumber}.part),合并时生成完整文件。
- 前端使用 File.slice(start, end) 动态读取文件的未上传部分。
三,关键优势
- 无需用户干预
- 用户关闭浏览器后,再次打开页面时自动恢复上传,无需重新选择文件。
- 安全与隐私
- 源文件仅存在于用户本地设备,未上传的块数据不会泄露到服务器。
- 高效性
- 仅上传未完成的块,减少网络流量和服务器负载。
解决方案:
项目代码无法拿出来,实现了一个最小可用版本。
需要注意的是如果用户更换了浏览器或者电脑就无法继续上传了
- 先启动node服务,在terminal中输入以下代码,控制台出现
文件上传服务已启动: http://localhost:3001
就是启动成功了。
> cd server
> node uploadServer.js
- 使用浏览器打开index.html,即可查看效果。
项目依赖如下(package.json):
{
"dependencies": {
"axios": "^1.8.4",
"cors": "^2.8.5",
"express": "^5.1.0",
"fs-extra": "^11.3.0",
"multer": "^1.4.5-lts.2",
"spark-md5": "^3.0.2"
}
}
目录结构如下:
源码如下:
index.html
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>大文件上传</title>
<!-- <script src="https://cdn.jsdelivr.net/npm/spark-md5@3.0.2/spark-md5.min.js"></script> -->
<!-- <script src="https://cdn.jsdelivr.net/npm/axios/dist/axios.min.js"></script> -->
<script src="https://cdn.bootcdn.net/ajax/libs/spark-md5/3.0.2/spark-md5.min.js"></script>
<script src="https://cdn.bootcdn.net/ajax/libs/axios/1.8.4/axios.min.js"></script>
<style>
.container {
margin: 20px;
padding: 20px;
border: 1px solid #ddd;
border-radius: 5px;
max-width: 500px;
}
.progress-container {
margin-top: 10px;
}
button:disabled {
opacity: 0.5;
}
</style>
</head>
<body>
<div class="container">
<input type="file" id="fileInput" />
<button id="uploadBtn">上传</button>
<div
class="progress-container"
id="progressContainer"
style="display: none"
>
<progress id="progressBar" value="0" max="100"></progress>
<span id="progressText">0%</span>
</div>
</div>
<script src="src/upload.js"></script>
</body>
</html>
upload.js
// 配置常量
const CONFIG = {
CHUNK_SIZE: 1 * 1024 * 1024, // 1MB
API_BASE_URL: "http://localhost:3001",
DB_NAME: "FileUploadDB",
STORE_NAME: "files",
RETRY_DELAY: 1000, // 1秒
MAX_RETRIES: 3,
};
// DOM 元素
const DOM = {
fileInput: document.getElementById("fileInput"),
uploadBtn: document.getElementById("uploadBtn"),
progressContainer: document.getElementById("progressContainer"),
progressBar: document.getElementById("progressBar"),
progressText: document.getElementById("progressText"),
};
// 状态管理
const state = {
file: null,
uploadedChunks: JSON.parse(localStorage.getItem("uploadedChunks") || "[]"),
fileHash: localStorage.getItem("fileHash") || "",
chunks: [],
fileInfo: JSON.parse(localStorage.getItem("fileInfo") || "null"),
db: null,
};
// IndexedDB 操作
const dbOperations = {
async initDB() {
return new Promise((resolve, reject) => {
const request = indexedDB.open(CONFIG.DB_NAME, 1);
request.onerror = () => reject(request.error);
request.onsuccess = () => {
state.db = request.result;
resolve(state.db);
};
request.onupgradeneeded = (event) => {
const db = event.target.result;
if (!db.objectStoreNames.contains(CONFIG.STORE_NAME)) {
db.createObjectStore(CONFIG.STORE_NAME);
}
};
});
},
async saveFile(file, hash) {
return new Promise((resolve, reject) => {
const transaction = state.db.transaction(
[CONFIG.STORE_NAME],
"readwrite"
);
const store = transaction.objectStore(CONFIG.STORE_NAME);
const request = store.put(file, hash);
request.onsuccess = () => resolve();
request.onerror = () => reject(request.error);
});
},
async getFile(hash) {
return new Promise((resolve, reject) => {
const transaction = state.db.transaction([CONFIG.STORE_NAME], "readonly");
const store = transaction.objectStore(CONFIG.STORE_NAME);
const request = store.get(hash);
request.onsuccess = () => resolve(request.result);
request.onerror = () => reject(request.error);
});
},
async deleteFile(hash) {
return new Promise((resolve, reject) => {
const transaction = state.db.transaction(
[CONFIG.STORE_NAME],
"readwrite"
);
const store = transaction.objectStore(CONFIG.STORE_NAME);
const request = store.delete(hash);
request.onsuccess = () => resolve();
request.onerror = () => reject(request.error);
});
},
};
// 进度管理
const progressManager = {
save(uploadedCount, totalChunks) {
localStorage.setItem(
"uploadProgress",
JSON.stringify({
uploadedCount,
totalChunks,
timestamp: Date.now(),
})
);
},
get() {
const progress = localStorage.getItem("uploadProgress");
return progress ? JSON.parse(progress) : null;
},
clear() {
localStorage.removeItem("uploadProgress");
},
update(uploaded, total) {
const percent = Math.floor((uploaded / total) * 100);
DOM.progressBar.value = percent;
DOM.progressText.textContent = `${percent}% (${uploaded}/${total})`;
},
};
// 存储管理
const storageManager = {
async clear() {
if (state.fileHash) {
await dbOperations.deleteFile(state.fileHash);
}
localStorage.removeItem("uploadedChunks");
localStorage.removeItem("fileHash");
localStorage.removeItem("fileInfo");
progressManager.clear();
state.fileHash = "";
state.fileInfo = null;
state.uploadedChunks = [];
},
};
// 文件操作
const fileOperations = {
createChunks(file) {
const chunks = [];
let current = 0;
while (current < file.size) {
chunks.push({
file: file.slice(current, current + CONFIG.CHUNK_SIZE),
index: chunks.length,
});
current += CONFIG.CHUNK_SIZE;
}
return chunks;
},
async calculateHash(file) {
return new Promise((resolve) => {
const spark = new SparkMD5.ArrayBuffer();
const reader = new FileReader();
reader.onload = (e) => {
spark.append(e.target.result);
resolve(spark.end());
};
reader.readAsArrayBuffer(file);
});
},
};
// API 操作
const apiOperations = {
async verifyFile(filename, hash) {
try {
const { data } = await axios.post(`${CONFIG.API_BASE_URL}/verify`, {
filename,
hash,
});
return data;
} catch (error) {
throw new Error(error.response?.data?.error || error.message);
}
},
async uploadChunk(chunk, hash, filename, index, chunkName) {
const formData = new FormData();
formData.append("chunk", chunk);
formData.append("hash", hash);
formData.append("filename", filename);
formData.append("index", index);
formData.append("chunkName", chunkName);
try {
const response = await axios.post(
`${CONFIG.API_BASE_URL}/upload`,
formData
);
return response.data;
} catch (error) {
throw new Error(error.response?.data?.error || error.message);
}
},
async mergeFile(filename, size, hash, chunks) {
try {
const response = await axios.post(`${CONFIG.API_BASE_URL}/merge`, {
filename,
size,
hash,
chunks,
});
return response.data;
} catch (error) {
throw new Error(error.response?.data?.error || error.message);
}
},
};
// 事件处理
const eventHandlers = {
async handleFileSelect(e) {
state.file = e.target.files[0];
const currentFileHash = await fileOperations.calculateHash(state.file);
await dbOperations.saveFile(state.file, currentFileHash);
if (state.fileInfo && state.fileHash) {
if (currentFileHash === state.fileHash) {
const shouldContinue = confirm(
`检测到未完成的上传,是否继续上传"${state.fileInfo.name}"?`
);
if (shouldContinue) {
DOM.uploadBtn.click();
return;
}
}
await storageManager.clear();
}
state.fileHash = currentFileHash;
state.fileInfo = {
name: state.file.name,
size: state.file.size,
type: state.file.type,
lastModified: state.file.lastModified,
};
localStorage.setItem("fileHash", state.fileHash);
localStorage.setItem("fileInfo", JSON.stringify(state.fileInfo));
},
async handleUpload() {
if (!state.file && state.fileInfo) {
alert("请重新选择相同的文件以继续上传");
return;
}
DOM.uploadBtn.disabled = true;
DOM.fileInput.disabled = true;
DOM.progressContainer.style.display = "block";
try {
const shouldUpload = await apiOperations.verifyFile(
state.file.name,
state.fileHash
);
if (!shouldUpload) {
DOM.uploadBtn.disabled = false;
DOM.fileInput.disabled = false;
return;
}
state.chunks = fileOperations.createChunks(state.file);
let uploadedCount = state.uploadedChunks.length;
progressManager.update(uploadedCount, state.chunks.length);
progressManager.save(uploadedCount, state.chunks.length);
let uploadFailed = false;
for (let i = 0; i < state.chunks.length; i++) {
const chunk = state.chunks[i];
const chunkName = `${state.fileHash}-${chunk.index}`;
if (state.uploadedChunks.includes(chunkName)) {
continue;
}
try {
await apiOperations.uploadChunk(
chunk.file,
state.fileHash,
state.file.name,
chunk.index,
chunkName
);
state.uploadedChunks.push(chunkName);
localStorage.setItem(
"uploadedChunks",
JSON.stringify(state.uploadedChunks)
);
uploadedCount++;
progressManager.update(uploadedCount, state.chunks.length);
progressManager.save(uploadedCount, state.chunks.length);
} catch (err) {
uploadFailed = true;
console.error("上传失败:", err);
alert(`上传失败: ${err.message}`);
break;
}
}
if (!uploadFailed && uploadedCount === state.chunks.length) {
await eventHandlers.handleMerge();
} else if (!uploadFailed) {
alert("还有分片未上传完成,请等待上传完成后再合并");
}
} catch (err) {
console.error("上传过程出错:", err);
alert(`上传过程出错: ${err.message}`);
} finally {
DOM.uploadBtn.disabled = false;
DOM.fileInput.disabled = false;
}
},
async handleMerge() {
try {
const verifyResponse = await apiOperations.verifyFile(
state.file.name,
state.fileHash
);
if (verifyResponse.shouldUpload) {
await new Promise((resolve) => setTimeout(resolve, CONFIG.RETRY_DELAY));
const mergeResponse = await apiOperations.mergeFile(
state.file.name,
CONFIG.CHUNK_SIZE,
state.fileHash,
state.chunks.length
);
if (mergeResponse.success) {
await eventHandlers.handleUploadComplete();
} else {
throw new Error(mergeResponse.error || "合并文件失败");
}
} else {
throw new Error("文件验证失败,请重试");
}
} catch (err) {
console.error("合并文件失败:", err);
const retry = confirm("合并失败,是否重试?");
if (retry) {
await eventHandlers.retryMerge();
}
}
},
async retryMerge() {
try {
const mergeResponse = await apiOperations.mergeFile(
state.file.name,
CONFIG.CHUNK_SIZE,
state.fileHash,
state.chunks.length
);
if (mergeResponse.success) {
await eventHandlers.handleUploadComplete();
} else {
throw new Error(mergeResponse.error || "合并文件失败");
}
} catch (err) {
console.error("重试合并失败:", err);
alert(`重试合并失败: ${err.message}`);
}
},
async handleUploadComplete() {
alert("上传完成");
await storageManager.clear();
},
};
// 初始化
async function init() {
await dbOperations.initDB();
if (state.fileInfo && state.fileHash) {
try {
state.file = await dbOperations.getFile(state.fileHash);
if (state.file) {
state.uploadedChunks = JSON.parse(
localStorage.getItem("uploadedChunks") || "[]"
);
state.chunks = fileOperations.createChunks(state.file);
const uploadedCount = state.uploadedChunks.length;
const percent = Math.floor((uploadedCount / state.chunks.length) * 100);
let progressMessage = `检测到未完成的文件 "${state.fileInfo.name}",已上传 ${percent}%`;
progressMessage += ",是否继续上传?";
const shouldContinue = confirm(progressMessage);
if (shouldContinue) {
DOM.progressContainer.style.display = "block";
progressManager.update(uploadedCount, state.chunks.length);
progressManager.save(uploadedCount, state.chunks.length);
DOM.uploadBtn.click();
return;
}
}
} catch (error) {
console.error("恢复文件失败:", error);
}
await storageManager.clear();
}
}
// 事件监听
DOM.fileInput.addEventListener("change", eventHandlers.handleFileSelect);
DOM.uploadBtn.addEventListener("click", eventHandlers.handleUpload);
// 启动应用
init().catch(console.error);
uploadServer.js
const express = require("express");
const multer = require("multer");
const fs = require("fs");
const path = require("path");
const cors = require("cors");
const fsExtra = require("fs-extra");
const app = express();
app.use(
cors({
origin: "*",
methods: ["GET", "POST", "PUT", "DELETE"],
allowedHeaders: ["Content-Type", "Authorization"],
})
);
// Add these middleware for body parsing
app.use(express.json());
const PORT = 3001;
// 确保上传目录存在
const UPLOAD_DIR = path.resolve(__dirname, "uploads");
if (!fs.existsSync(UPLOAD_DIR)) {
fs.mkdirSync(UPLOAD_DIR);
}
// 修改存储配置
const storage = multer.diskStorage({
destination: (req, file, cb) => {
// 从文件名中获取 hash(因为文件名包含了 hash)
const hash = file.originalname.split("-")[0];
const chunkDir = path.resolve(UPLOAD_DIR, hash);
if (!fs.existsSync(chunkDir)) {
fs.mkdirSync(chunkDir);
}
cb(null, chunkDir);
},
filename: (req, file, cb) => {
// 直接使用原始文件名
cb(null, file.originalname);
},
});
// 添加日志函数
function log(...args) {
const timestamp = new Date().toISOString();
const logMessage = `[${timestamp}] ${args.join(" ")}`;
console.log("\x1b[36m%s\x1b[0m", logMessage); // 使用青色输出
// 同时写入文件
fs.appendFileSync("server.log", logMessage + "\n");
}
// 检查文件是否已存在
app.post("/verify", (req, res) => {
const { filename, hash } = req.body;
const filePath = path.resolve(UPLOAD_DIR, `${hash}${path.extname(filename)}`);
if (fs.existsSync(filePath)) {
return res.json({ shouldUpload: false });
}
const chunkDir = path.resolve(UPLOAD_DIR, hash);
const uploadedList = fs.existsSync(chunkDir) ? fs.readdirSync(chunkDir) : [];
res.json({ shouldUpload: true, uploadedList });
});
const upload = multer({
storage, // 使用之前定义的diskStorage
limits: {
fileSize: 20 * 1024 * 1024, // 20MB
},
});
app.post("/upload", upload.single("chunk"), async (req, res) => {
try {
const { hash, filename, index } = req.body;
const chunkName = `${hash}-${index}`;
const chunkDir = path.resolve(UPLOAD_DIR, hash);
const chunkPath = path.resolve(chunkDir, chunkName);
log("上传请求参数:", { hash, filename, index, chunkName });
log("分片目录:", chunkDir);
log("分片路径:", chunkPath);
// 确保分片目录存在
await fsExtra.ensureDir(chunkDir);
log("分片目录已创建/确认");
// 移动上传的分片到目标位置
await fsExtra.move(req.file.path, chunkPath, { overwrite: true });
log("分片已保存:", chunkName);
res.json({
success: true,
message: "分片上传成功",
});
} catch (error) {
log("上传过程出错:", error);
res.json({
success: false,
error: error.message,
});
}
});
// 添加重试函数
async function retryOperation(operation, maxRetries = 3, delay = 1000) {
for (let i = 0; i < maxRetries; i++) {
try {
return await operation();
} catch (error) {
if (i === maxRetries - 1) throw error;
log(
`操作失败,${delay}ms 后重试 (${i + 1}/${maxRetries}):`,
error.message
);
await new Promise((resolve) => setTimeout(resolve, delay));
}
}
}
// 修改合并分片的函数
async function mergeFileChunks(filePath, chunkDir, chunkSize) {
return new Promise((resolve, reject) => {
const writeStream = fs.createWriteStream(filePath);
const chunks = fs.readdirSync(chunkDir).sort((a, b) => {
const indexA = parseInt(a.split("-")[1]);
const indexB = parseInt(b.split("-")[1]);
return indexA - indexB;
});
let currentChunk = 0;
const totalChunks = chunks.length;
function writeNextChunk() {
if (currentChunk >= totalChunks) {
writeStream.end();
resolve();
return;
}
const chunkPath = path.join(chunkDir, chunks[currentChunk]);
const readStream = fs.createReadStream(chunkPath);
readStream.pipe(writeStream, { end: false });
readStream.on("end", () => {
currentChunk++;
writeNextChunk();
});
readStream.on("error", (err) => {
writeStream.end();
reject(err);
});
}
writeStream.on("error", (err) => {
reject(err);
});
writeNextChunk();
});
}
// 修改合并路由
app.post("/merge", async (req, res) => {
try {
const { filename, size, hash } = req.body;
// 使用原始文件名,而不是hash
const filePath = path.resolve(UPLOAD_DIR, filename);
const chunkDir = path.resolve(UPLOAD_DIR, hash);
log("合并请求参数:", { filename, size, hash });
log("文件路径:", filePath);
log("分片目录:", chunkDir);
if (!(await fsExtra.exists(chunkDir))) {
log("错误: 分片目录不存在:", chunkDir);
return res.json({
success: false,
error: "分片目录不存在,请先上传分片",
});
}
const chunks = await fsExtra.readdir(chunkDir);
log("找到的分片数量:", chunks.length);
log("分片列表:", chunks);
if (chunks.length === 0) {
log("错误: 没有找到任何分片文件");
return res.json({
success: false,
error: "没有找到任何分片文件",
});
}
// 等待一段时间确保文件写入完成
await new Promise((resolve) => setTimeout(resolve, 1000));
// 使用重试机制合并文件
await retryOperation(async () => {
await mergeFileChunks(filePath, chunkDir, size);
log("文件合并成功");
});
// 删除切片文件
try {
for (const chunk of chunks) {
const chunkPath = path.join(chunkDir, chunk);
await fsExtra.remove(chunkPath);
log("删除切片文件:", chunkPath);
}
// 删除切片目录
await fsExtra.remove(chunkDir);
log("删除切片目录:", chunkDir);
} catch (error) {
log("删除切片文件失败:", error);
// 即使删除失败也继续返回成功响应,因为文件已经合并成功
}
// 只发送一次响应
return res.json({
success: true,
message: "文件合并成功",
});
} catch (error) {
log("合并过程出错:", error);
return res.json({
success: false,
error: error.message,
});
}
});
// 修改服务器启动代码
const server = app.listen(PORT, "0.0.0.0", () => {
console.log(`文件上传服务已启动: http://localhost:${PORT}`);
});
server.on("error", (err) => {
console.error("服务器启动失败:", err);
if (err.code === "EADDRINUSE") {
console.log(`端口 ${PORT} 已被占用,尝试使用其他端口`);
// 自动切换到备用端口
app.listen(3002, "0.0.0.0", () => {
console.log(`服务运行在备用端口: 3002`);
});
}
});