往期鸿蒙5.0全套实战文章必看：（文中附带全栈鸿蒙5.0学习资料）
大文件高速并发传输解决方案(上)
1. 概述
针对大文件的高性能传输是应用开发中非常常见的场景，特别是在处理大规模数据传输时，性能优化显得尤为关键。在大文件传输中，最为典型的操作包括文件的上传和下载。这两种操作通常通过HTTP协议实现，并通过开发者根据不同的需求采用特定的技术与策略来确保高效和可靠的数据传输。
然而，当前HarmonyOS接口对这两种操作提供的功能特性有限。对于下载文件场景而言，当前HarmonyOS仅能提供基础的下载功能，在服务端限流或网络条件不佳（如弱网环境）时，可能未能充分利用可用的网络带宽；而当遇到网络连接中断，应用崩溃等异常情况时，已下载的部分文件可能会残留在应用中，并需要手动清理从头开始下载。对于上传文件场景而言，当前HarmonyOS仅能提供基础的上传功能，在遇到网络连接中断，应用崩溃等异常情况时，同样需要从头开始上传文件。这些问题不仅影响用户体验，还可能导致不必要的带宽浪费和失败的重复尝试。
针对上述痛点，本文提供了一个支持大文件多线程并发分块下载、断点续下、分片上传、断点续传、自动重试等多个特性的三方库super_fast_file_trans（SFFT），并基于该三方库提供了一个实际的开发案例，旨在帮助开发人员更好地实现大文件数据传输场景。
2. 功能特性
多线程并发分块下载
多线程并发分块下载是一种利用多线程能力加速下载过程的技术。通过将目标大文件规划为多个块，客户端可以发送多个连接请求来并行下载目标文件的不同数据部分，并同时写入到本地文件中。该技术的目的是充分利用设备的网络带宽，提升下载速率，在网络条件不理想的情况下显著减少下载时间。在网络传输中，超过100MB的文件就可以被视作大文件，这时就可以考虑使用分块下载来提高下载的传输效率（尤其在弱网条件下）。
文件的分块下载通常是基于HTTP协议中的Range请求头实现，Range字段是HTTP的一个请求头部，特别用于在请求中指定目标资源的某个字节范围。通过解析HTTP请求中设置的Range头，服务端可以仅返回文件被指定的字节部分，从而节省传输开销，以支持断点续传和分块下载。
SFFT基于HarmonyOS提供的taskpool和Remote Communication Kit（rcp）两种底座能力实现大文件多线程并行分块下载。对于一个下载任务，SFFT会根据设定参数和目标文件大小将目标文件分成若干个块，并创建多个线程分别请求这些数据块。对于每一个线程，线程内部都会根据被分配的块的范围创建指定Range头的HTTP连接，独立从服务端请求数据。在数据成功响应后，客户端中的多个线程会并行地将各个请求的数据写入到本地指定路径的文件中。
使用多线程分块下载技术的加速效果如下所示：
SFFT实现多线程分块下载技术的关键代码如下：
DownloadCall：
import { WriteOptions } from '@kit.CoreFileKit';
import { taskpool } from '@kit.ArkTS';
import { rcp } from '@kit.RemoteCommunicationKit';
import { emitter } from '@kit.BasicServicesKit';
import { DownloadTaskMetadata } from './DownloadTaskMetadata';
import { ConnectionParams } from './ConnectionParams';
import { DownloadEventConstants } from '../common/DownloadConstants';
import { FdWriteStream } from '../../../framework/file/FdWriteStream';
import { DownloadBlockInfo } from '../../../framework/storage/DownloadTaskInfo';
import { DownloadStoreHelper } from '../../../framework/storage/DownloadStoreHelper';
import { FileUtils } from '../../../common/utils/FileUtils';
import { TimeUtils } from '../../../common/utils/TimeUtils';
export class DownloadCall {
// 下载任务的元信息
private downloadTaskMetadata: DownloadTaskMetadata;
constructor(downloadTaskMetadata: DownloadTaskMetadata) {
this.downloadTaskMetadata = downloadTaskMetadata;
}
public async download() {
// 文件标识符
let fileFd: number;
try {
const filePath = `${this.downloadTaskMetadata.fileDir}/${await FileUtils.getTempFileName(this.downloadTaskMetadata.id)}`;
const file = await FileUtils.openFile(filePath);
fileFd = file.fd;
} catch (err) {
Logger.error(LoggerConstants.CALL, `code: ${err.code}, message: ${err.message}`);
throw new DownloadError('Target file to write does not exist', DownloadErrorType.NO_TARGET_FILE_ERROR);
}
// 从任务元信息中读取所使用的线程数，该线程数与分块的个数对应
const blockInfoCount = this.downloadTaskMetadata.concurrency!;
// 此处从缓存中读取块信息，为分块下载做准备
const blockInfos = DownloadStoreHelper.getBlockInfosFromCacheById(this.downloadTaskMetadata.id);
if (blockInfos.length === 0) {
throw new DownloadError('Task has no matched BlockInfos', DownloadErrorType.NO_BLOCK_INFOS_ERROR);
}
// 当前手动取消状态为false
let isManualCanceled = false;
// 注册监听手动取消指令的监听，以实现退出自动重试
emitter.once(this.downloadTaskMetadata.id + DownloadEventConstants.CANCEL_EVENT, () => {
isManualCanceled = true;
});
for (let i = 0; i < this.downloadTaskMetadata.maxRetries + 1 && !isManualCanceled; i++) {
if (i > 0) {
Logger.error(LoggerConstants.CALL, `Retry download connection, time: ${i}`);
}
const taskGroup = new taskpool.TaskGroup();
// 生成块级别的参数信息
for (let i = 0; i < blockInfoCount; i++) {
// 已经完成下载的块不需要再重复下载
if (blockInfos[i].currentOffset >= blockInfos[i].startOffset + blockInfos[i].contentLength) {
continue;
}
const connectionParams: ConnectionParams = {
url: this.downloadTaskMetadata.url,
fileName: this.downloadTaskMetadata.fileName,
start: blockInfos[i].currentOffset,
end: blockInfos[i].startOffset + blockInfos[i].contentLength - 1,
fd: fileFd,
connectTimeout: this.downloadTaskMetadata.connectTimeout,
transferTimeout: this.downloadTaskMetadata.transferTimeout,
inactivityTimeout: this.downloadTaskMetadata.inactivityTimeout,
requestHeaders: this.downloadTaskMetadata.requestHeaders,
index: i
};
// 新增每个块对应的下载任务
taskGroup.addTask(execute, connectionParams, blockInfos[i]);
}
try {
await taskpool.execute(taskGroup);
// 下载完成，取消监听，
emitter.off(this.downloadTaskMetadata.id + DownloadEventConstants.CANCEL_EVENT);
break;
} catch (err) {
// 达到最大重试次数
if (i === this.downloadTaskMetadata.maxRetries) {
if (i > 0) {
Logger.error(LoggerConstants.CALL, `Retry failed, code: ${err.code}, message: ${err.message}`);
}
throw new DownloadError('Download thread error', DownloadErrorType.THREAD_ERROR);
}
// 重试间隔
await TimeUtils.sleepInterval(this.downloadTaskMetadata.retryInterval, isManualCanceled);
Logger.error(LoggerConstants.CALL, `code: ${err.code}, message: ${err.message}`);
}
}
}
}
@Concurrent
async function execute(params: ConnectionParams, blockInfo: DownloadBlockInfo) {
const configuration: rcp.Configuration = {
transfer: {
timeout: { connectMs: params.connectTimeout, transferMs: params.transferTimeout }
}
}
const transferRange: rcp.TransferRange = { from: params.start, to: params.end };
const req = new rcp.Request(params.url, undefined, params.requestHeaders, undefined, undefined, transferRange, configuration);
const option: WriteOptions = {
offset: params.start,
length: params.end
}
// 设置请求体数据存放地址
req.destination = {
kind: 'stream',
stream: new FdWriteStream(params.fd!, option, blockInfo)
}
const session = rcp.createSession();
// 注册停止任务监听
emitter.once(blockInfo.hostId.toString() + DownloadEventConstants.STOP_EVENT, (): void => {
session.cancel(req);
});
// 注册任务失败监听
emitter.once(blockInfo.hostId.toString() + DownloadEventConstants.FAIL_EVENT, (): void => {
session.cancel(req);
});
try {
await session.fetch(req);
} catch (err) {
// 任务失败时会取消所有的监听，本类中代码不涉及
Logger.warn(LoggerConstants.RCP, `code: ${err.code}, message: HTTP Connection is terminated in download`);
throw new NetworkError('HTTP connection is terminated in download', NetworkErrorType.TERMINATED_ERROR);
} finally {
session.close();
}
}FdWriteFile，FdWriteStream：
import fs from '@ohos.file.fs';
import { rcp } from '@kit.RemoteCommunicationKit';
import { FdWriteFile } from './FdWriteFile';
import { WriteOptions } from '@kit.CoreFileKit';
import { DownloadBlockInfo } from '../storage/DownloadTaskInfo';
export class FdWriteFile implements rcp.WriteFile {
private fd: number;
private option: WriteOptions;
constructor(fd: number, option: WriteOptions) {
this.fd = fd;
this.option = option;
}
// 重写Write方法，计算每次写入后下次的偏移量
async write(buffer: ArrayBuffer): Promise<number> {
this.option.length = buffer.byteLength;
let res = await fs.write(this.fd, buffer, this.option);
this.option.offset! += res;
return res;
}
}
export class FdWriteStream implements rcp.WriteStream {
private file: FdWriteFile;
private blockInfo: DownloadBlockInfo;
constructor(fd: number, option: WriteOptions, blockInfo: DownloadBlockInfo) {
this.file = new FdWriteFile(fd, option);
this.blockInfo = blockInfo;
}
// DownloadInfo为Sendable对象
async write(buffer: ArrayBuffer): Promise<number> {
let writeLen = await this.file.write(buffer);
this.blockInfo.currentOffset += writeLen;
return writeLen;
}
}由于需要使用携带Range头的HTTP请求从远端获取文件，因此SFFT的分块下载特性和断点续下特性对服务端有如下要求：
- 必须支持带有Range请求头的HTTP请求，并能够根据请求的字节范围返回本地文件的部分字节数据。
- 当接收到带有Range请求头的请求时，必须支持Content-Range响应头，并返回响应中文件的字节范围，以及文件的总大小。
- 当客户端需要特定的返回头信息时，返回正确的响应头：如HTTP状态码、Accept-Ranges、Last-Modified、Etag、Content-MD5、Content-SHA256等。
断点续下
断点续下是指在下载过程中，因网络中断、客户端崩溃或其他原因，导致下载未完成时，能够从中断的位置继续下载，而不需要从头开始下载的技术。断点续下的目的是为了避免重复的数据传输，节省下载时间，防止已下载数据丢失。
SFFT基于ohos.data.relationalStore（rdb）和缓存机制实现断点续下。对于每一个新建的下载任务，其下载信息、配置信息、块信息将会被读取到缓存并持久化到数据库中，当下载任务被执行时，块信息将会被定时性地更新，以保证下载进度的可靠性。当下载任务需要从被中断处开始继续下载时，SFFT将会从数据库中读取对应的下载任务信息和块信息，重新发起HTTP请求以继续获取目标文件剩余部分的字节数据。
使用断点续下技术继续下载的效果图如下所示：
SFFT实现断点续下技术的关键代码如下：
DownloadInfoManager：
import { DownloadTaskMetadata } from "./DownloadTaskMetadata";
import { DownloadBlockInfo, DownloadTaskInfo } from "../../../framework/storage/DownloadTaskInfo";
import { DownloadStoreHelper } from "../../../framework/storage/DownloadStoreHelper";
import { MathUtils } from "../../../common/utils/MathUtils";
import { DownloadStatusType } from "../common/DownloadStatusType";
export class DownloadInfoManager {
private static instance = new DownloadInfoManager();
private constructor() {
}
public static getInstance(): DownloadInfoManager {
return DownloadInfoManager.instance;
}
public async setTaskInfoByCache(downloadTask: DownloadTask) {
// 如果下载任务为新任务，直接返回
if (downloadTask.id !== -1) {
return;
}
// 尝试从缓存中匹配下载信息
const taskInfo = await DownloadStoreHelper.getTaskInfoByUrlAndPath(downloadTask.url, downloadTask.fileDir, downloadTask.fileName);
if (taskInfo) {
// 更新相关信息
downloadTask.id = taskInfo.id!;
downloadTask.fileSize = taskInfo.fileSize;
downloadTask.etag = taskInfo.etag;
downloadTask.status = taskInfo.isCompleted ? DownloadStatusType.COMPLETED : downloadTask.status;
}
}
public async checkAndSetTaskInfo(downloadTask: DownloadTask) {
// 尝试从缓存中匹配下载信息
const taskInfo =
await DownloadStoreHelper.getTaskInfoByUrlAndPath(downloadTask.url, downloadTask.fileDir, downloadTask.fileName);
if (!taskInfo) {
throw new ConsistencyError('Breakpoint information lost', ConsistencyErrorType.DOWNLOAD_INFO_LOST_ERROR);
}
// 更新相关信息
downloadTask.id = taskInfo.id!;
downloadTask.fileSize = taskInfo.fileSize;
downloadTask.etag = taskInfo.etag;
downloadTask.status = taskInfo.isCompleted ? DownloadStatusType.COMPLETED : downloadTask.status;
downloadTask.concurrency = taskInfo.chunkCount;
}
public async storeTaskInfo(downloadTask: DownloadTask) {
// 生成任务信息
let taskInfo: DownloadTaskInfo = {
id: undefined,
url: downloadTask.url,
fileName: downloadTask.fileName,
fileDir: downloadTask.fileDir,
fileSize: downloadTask.fileSize,
etag: downloadTask.etag,
chunkCount: downloadTask.concurrency!,
isCompleted: false,
blockInfoList: undefined,
};
// 保存任务信息到数据库，如果成功，返回一个任务id，并存储到缓存中，更新taskInfo的id
try {
const taskId = await DownloadStoreHelper.storeTaskInfo(taskInfo);
downloadTask.id = taskId;
} catch (err) {
throw err as StoreInfoError;
}
}
public async storeBlockInfos(downloadTask: DownloadTask) {
// 生成分片信息
const concurrency = downloadTask.concurrency;
const ranges: number[][] = MathUtils.divideIntoOverlappingRanges(downloadTask.fileSize, concurrency);
let blockInfos: DownloadBlockInfo[] = [];
for (let index = 0; index < ranges.length; index++) {
const start = ranges[index][0];
const end = ranges[index][1];
blockInfos.push(new DownloadBlockInfo(
downloadTask.id,
index,
start,
end - start + 1,
start,
));
}
// 将分片信息存储到数据库以及缓存中
try {
await DownloadStoreHelper.storeBlockInfos(blockInfos);
} catch (err) {
throw err as StoreInfoError;
}
}
public async updateTaskInfo(downloadTask: DownloadTask) {
try {
await DownloadStoreHelper.updateTaskInfoById(downloadTask.id);
} catch (err) {
throw err as StoreInfoError;
}
}
public async updateBlockInfos(downloadTask: DownloadTask) {
try {
await DownloadStoreHelper.updateBlockInfosById(downloadTask.id);
} catch (err) {
throw err as StoreInfoError;
}
}
public async deleteDownloadInfo(downloadTask: DownloadTask) {
try {
await DownloadStoreHelper.deleteDownloadInfoById(downloadTask.id);
} catch (err) {
throw err as StoreInfoError;
}
}
}DownloadController：
import { BusinessError } from '@kit.BasicServicesKit';
import { DownloadTaskMetadata } from './DownloadTaskMetadata'
import { DownloadTrial } from './DownloadTrial';
import { DownloadCall } from './DownloadCall';
import { DownloadStatusManager } from './DownloadStatusManager';
import { DownloadProgressInfo, DownloadProgress } from './DownloadProgress';
import { DownloadInfoManager } from './DownloadInfoManager';
import { DownloadStatusType } from '../common/DownloadStatusType';
import { DownloadUpdateConstants } from '../common/DownloadConstants';-+
export class DownloadController {
private downloadTask: DownloadTask;
private downloadSignal: DownloadSignal;
private downloadTrail: DownloadTrial;
private downloadCall: DownloadCall;
constructor(downloadTask: DownloadTask) {
this.downloadTask = downloadTask;
this.downloadSignal = new DownloadSignal();
this.downloadTrail = new DownloadTrial(downloadTask, this.downloadSignal);
this.downloadCall = new DownloadCall(downloadTask, this.downloadSignal);
}
public async start() {
try {
if (this.downloadTaskMetadata.status === DownloadStatusType.DOWNLOADING) {
return;
}
// 初始化任务状态
DownloadStatusManager.getInstance()
.setStatus(DownloadStatusType.INITIALIZED, this.downloadTaskMetadata);
// 关闭当前已有的下载任务并清除相关信息
await this.cleanDirtyDownload();
// 初始化下载进度 如果有id，显示当前进度，如果id为-1，则为0，有id但是缓存中查不到也是0
this.downloadProgress.initDownloadProgress();
// 状态迁移
DownloadStatusManager.getInstance().setStatus(DownloadStatusType.DOWNLOADING, this.downloadTaskMetadata);
Logger.info(LoggerConstants.DOWNLOAD,
`Start download, url: ${this.downloadTaskMetadata.url}, fileName: ${this.downloadTaskMetadata.fileName}`);
// 试连
await this.downloadTrail.executeTrial();
// 试连成功，根据试连响应检查并更新downloadTask信息
this.downloadTrail.updateTaskInfoWithResponse();
// 回调
this.downloadTaskMetadata.listener?.onStart?.(this.downloadTrail.getResponseHeaders());
// 生成下载任务并存储任务信息和分片信息到数据库与缓存
await DownloadInfoManager.getInstance().storeTaskInfo(this.downloadTaskMetadata);
await DownloadInfoManager.getInstance().storeBlockInfos(this.downloadTaskMetadata);
} catch (err) {
this.dealWithError(err);
}
// 进行文件下载
this.executeDownload();
}
public async pause() {
if (!this.downloadTaskMetadata.isResumable) {
Logger.info(LoggerConstants.DOWNLOAD, `Cannot pause download if isResumable is disabled`);
return;
}
try {
if (this.downloadTaskMetadata.status === DownloadStatusType.PAUSED) {
return;
}
// 状态迁移，修改取消信号为手动设置
DownloadStatusManager.getInstance().setStatus(DownloadStatusType.PAUSED, this.downloadTaskMetadata);
// 更新数据库下载信息
if (this.downloadTaskMetadata.isResumable) {
await DownloadInfoManager.getInstance().updateBlockInfos(this.downloadTaskMetadata);
}
// 初始化下载进度
this.downloadProgress.initDownloadProgress();
const progress = await this.downloadProgress.getDownloadProgressInfo();
this.downloadTaskMetadata.listener?.onPause?.(progress);
Logger.info(LoggerConstants.DOWNLOAD,
`Pause download, url: ${this.downloadTaskMetadata.url}, fileName: ${this.downloadTaskMetadata.fileName}`);
} catch (err) {
this.dealWithError(err);
}
}
public async resume() {
if (!this.downloadTaskMetadata.isResumable) {
Logger.info(LoggerConstants.DOWNLOAD, `Cannot resume download if isResumable is disabled`);
return;
}
try {
if (this.downloadTaskMetadata.status === DownloadStatusType.DOWNLOADING) {
return;
}
// 尝试从缓存中匹配下载信息并写入到downloadTask，不存在则直接退出，无法续下
DownloadInfoManager.getInstance().checkAndSetTaskInfo(this.downloadTaskMetadata);
// 在下载状态无法进行续下操作/同时下载任务数已达上限
DownloadStatusManager.getInstance()
.setStatus(DownloadStatusType.DOWNLOADING, this.downloadTaskMetadata);
// 初始化下载进度
this.downloadProgress.initDownloadProgress();
Logger.info(LoggerConstants.DOWNLOAD,
`Resume download, url: ${this.downloadTaskMetadata.url}, fileName: ${this.downloadTaskMetadata.fileName}`);
// 进行试连
await this.downloadTrail.executeTrial();
// 试连成功，根据试连响应检查并更新downloadTask信息
// 如果downloadTask的部分字段被更新，表明文件发生变化，需调用start重新下载
this.downloadTrail.checkResponseChanged();
// 回调
const progress = await this.downloadProgress.getDownloadProgressInfo();
this.downloadTaskMetadata.listener?.onResume?.(this.downloadTrail.getResponseHeaders(), progress);
} catch (err) {
this.dealWithError(err);
}
// 如果信息匹配，正常续下
this.executeDownload();
}
public async cancel() {
try {
// 状态迁移，修改取消信号为手动设置
DownloadStatusManager.getInstance().setStatus(DownloadStatusType.CANCELED, this.downloadTaskMetadata);
// 清除下载信息以及下载文件
await this.cleanTaskInfoAndFile();
// 初始化下载进度
this.downloadProgress.initDownloadProgress();
this.downloadTaskMetadata.listener?.onCancel?.();
Logger.warn(LoggerConstants.DOWNLOAD,
`Cancel download, url: ${this.downloadTaskMetadata.url}, fileName: ${this.downloadTaskMetadata.fileName}`);
} catch (err) {
this.dealWithError(err);
}
}
public async getProgress(): Promise<DownloadProgressInfo> {
try {
// 尝试从缓存中匹配downloadTask信息
DownloadInfoManager.getInstance().setTaskInfoByCache(this.downloadTaskMetadata);
return await this.downloadProgress.getDownloadProgressInfo();
} catch (err) {
Logger.error(LoggerConstants.DOWNLOAD, `Get progress failed,code: ${err.code}, message: ${err.message}`);
return {
transferredSize: 0,
totalSize: 0,
speed: 0
} as DownloadProgressInfo;
}
}
...
}
分片上传
分片上传是一种适用于大文件的上传机制，使用分片上传的客户端通常将文件分成多个较小的片段（或称为“分片”），然后并行地上传这些分片，以提高上传速率和上传的可靠性。在分片上传中，每一个分片是文件的一部分，通常具有固定的大小，每个分片在上传过程中被独立发送到服务端，当服务端接收到分片时，会根据分片中携带的序号信息保存每个分片，当全部分片上传完成后，服务端需要将分片拼接程完整的文件，并进行完整性校验。
SFFT基于HarmonyOS提供的Remote Communication Kit（rcp）底座能力，结合分片策略实现大文件的分片上传。对于一个上传任务，SFFT会根据设定参数和需要上传的本地文件的大小将文件分成若干个片，并在记录每个片的信息后并发发送多个POST请求，每个请求都会携带一个分片数据。在SFFT中，每一个分片请求的表单都会携带多个表单字段，包含文件的二进制数据、文件总分片数、文件哈希值，以及该片对应的序号信息。
SFFT实现分片上传技术的关键代码如下：
UploadCall：
import fs, { ReadOptions } from '@ohos.file.fs';
import { rcp } from '@kit.RemoteCommunicationKit';
import { UploadTaskMetadata } from './UploadTaskMetadata'
import { UploadParams } from './ConnectionParams';
import { UploadProgress } from './UploadProgress';
import { UploadInfoManager } from './UploadInfoManager';
import { UploadProgressInfo } from '../api/UploadTask';
import { UploadEventManager } from './UploadEventManager';
import { UploadStatusManager } from './UploadStatusManager';
import { UploadStatusType } from '../common/UploadStatusType';
import { UploadConnectionConstants, UploadEventConstants } from '../common/UploadConstants';
export class UploadCall {
private uploadTaskMetadata: UploadTaskMetadata;
public uploadProgress: UploadProgress;
constructor(uploadTaskMetadata: UploadTaskMetadata) {
this.uploadTaskMetadata = uploadTaskMetadata;
this.uploadProgress = new UploadProgress(uploadTaskMetadata);
}
public async getProgress(): Promise<UploadProgressInfo> {
if (this.uploadTaskMetadata.status !== UploadStatusType.UPLOADING) {
this.uploadProgress.updateUploadProgress();
}
return this.uploadProgress.getUploadProgressInfo();
}
public async execute() {
// 设置请求头
const requestHeaders: rcp.RequestHeaders = this.uploadTaskMetadata.requestHeaders ?? {};
requestHeaders['content-type'] = 'multipart/form-data';
// 请求配置
const requestConfig: rcp.Configuration = {
transfer: {
timeout: {
connectMs: this.uploadTaskMetadata.connectTimeout,
transferMs: this.uploadTaskMetadata.transferTimeout,
inactivityMs: this.uploadTaskMetadata.inactivityTimeout
}
}
};
// 发送上传请求
try {
await this.upload(requestConfig, requestHeaders);
} catch (err) {
Logger.error(LoggerConstants.UPLOAD, `Upload fail, code: ${err.code}, message: ${err.message}`);
}
private async upload(requestConfig: rcp.Configuration, requestHeaders: rcp.RequestHeaders) {
// 配置最大连接数
const connectionConfig: rcp.ConnectionConfiguration = {
maxTotalConnections: UploadConnectionConstants.MAX_TOTAL_CONNECTIONS
}
// 获取session
const session = rcp.createSession({ connectionConfiguration: connectionConfig });
let isManualCanceled = false;
UploadEventManager.getInstance().onceEvent(UploadEventConstants.STOP_EVENT, this.uploadTaskMetadata.id, () => {
isManualCanceled = true;
session.close();
})
UploadEventManager.getInstance().onceEvent(UploadEventConstants.CANCEL_EVENT, this.uploadTaskMetadata.id, () => {
isManualCanceled = true;
session.close();
})
const fileStream = await fs.createStream(this.uploadTaskMetadata.filePath, "r");
// 获取文件的大小、片大小、总片数
const fileSize = this.uploadTaskMetadata.fileSize;
const chunkSize = this.uploadTaskMetadata.chunkSize;
const totalChunks = Math.ceil(fileSize / chunkSize);
// 根据重试次数循环发送分片
for (let retryTime = 0; retryTime < this.uploadTaskMetadata.maxRetries && !isManualCanceled; retryTime++) {
if (retryTime > 0) {
Logger.error(LoggerConstants.CALL, `Retry upload connection, time: ${retryTime}`);
}
// 使用Promise.all并行上传所有分片
let uploadPromises: Promise<void>[] = [];
// 循环发送请求
for (let index = 0; index < totalChunks; index++) {
// 根据分片状态判断是否上传分片数据
if (await UploadInfoManager.getInstance().getUnsolvedChunkStatus(this.uploadTaskMetadata, index, false)) {
continue;
}
// 每个分片定义缓存
let buf = new ArrayBuffer(chunkSize);
let readOptions: ReadOptions = {
offset: index * chunkSize,
length: chunkSize
};
let res = await fileStream.read(buf, readOptions);
if (res < chunkSize) {
// 如果最后一块不足chunkSize, 则将长度调整为实际读取的字节数
readOptions.length = res;
}
// 设置每个分片的请求参数
let uploadParams: UploadParams = {
url: this.uploadTaskMetadata.url,
session: session,
requestConfig: requestConfig,
requestHeaders: requestHeaders,
buf: buf,
start: readOptions.offset!,
end: readOptions.offset! + readOptions.length!,
fileName: this.uploadTaskMetadata.uploadFileName,
contentType: this.uploadTaskMetadata.contentType,
totalChunks: this.uploadTaskMetadata.isChunk ? totalChunks : undefined,
chunkIndex: this.uploadTaskMetadata.isChunk ? index : undefined,
};
let uploadPromise = this.executeUpload(uploadParams);
uploadPromises.push(uploadPromise);
}
// 发送多个异步请求
try {
await Promise.allSettled(uploadPromises);
session.close();
if (isManualCanceled) {
break;
}
// 检查上传分片状态
if (await this.uploadProgress.isUploadCompleted()) {
await this.uploadProgress.saveCompletedProgress();
UploadStatusManager.getInstance().setStatus(UploadStatusType.COMPLETED, this.uploadTaskMetadata);
this.uploadProgress.clearUploadSpeed();
this.uploadTaskMetadata.listener?.onSuccess?.(this.uploadTaskMetadata.filePath);
Logger.info(LoggerConstants.UPLOAD, `Chunk upload promises completed`);
break;
}
} catch (err) {
Logger.error(LoggerConstants.UPLOAD,
`Chunk upload promises failed to completed. error code:${err.code}, message: ${err.message}`);
}
// 休眠后重试
await TimeUtils.sleep(this.uploadTaskMetadata.retryInterval);
}
}
private async executeUpload(params: UploadParams) {
// 表单内容
const multiForm = new rcp.MultipartForm({
'file': {
remoteFileName: params.fileName,
contentType: params.contentType,
contentOrPath: { content: params.buf }
}
});
if (params.totalChunks) {
multiForm.fields['totalChunks'] = params.totalChunks;
}
if (params.chunkIndex) {
multiForm.fields['chunkIndex'] = params.chunkIndex;
}
const transferRange: rcp.TransferRange = { from: params.start, to: params.end };
const req = new rcp.Request(params.url, "POST", params.requestHeaders, multiForm, undefined, transferRange,
params.requestConfig);
const session = params.session;
// 订阅手动停止事件
UploadEventManager.getInstance().onceEvent(UploadEventConstants.STOP_EVENT, this.uploadTaskMetadata.id, () => {
session.cancel(req);
this.uploadProgress.clearUploadSpeed();
})
// 订阅任务失败事件
UploadEventManager.getInstance().onceEvent(UploadEventConstants.FAIL_EVENT, this.uploadTaskMetadata.id, () => {
session.cancel(req);
this.uploadProgress.clearUploadSpeed();
})
// 订阅手动取消事件
UploadEventManager.getInstance().onceEvent(UploadEventConstants.CANCEL_EVENT, this.uploadTaskMetadata.id, () => {
session.cancel(req);
this.uploadProgress.clearUploadProgress();
})
try {
await session.fetch(req);
// 更新上传分片的状态
if (this.uploadTaskMetadata.isChunk) {
await UploadInfoManager.getInstance()
.updateUnsolvedChunkList(this.uploadTaskMetadata, params.chunkIndex!, true);
} else {
await UploadInfoManager.getInstance().updateUnsolvedChunkList(this.uploadTaskMetadata, 0, true);
}
await this.uploadProgress.updateUploadProgress();
} catch (err) {
throw new NetworkError('Connection error', NetworkErrorType.TERMINATED_ERROR);
}
}
}由于需要上传多个分片数据到远端，因此SFFT的分片上传特性对服务端有如下要求：
- 必须支持multipart/form-data，服务端允许在一个表单提交中同时携带多个不同类型的数据（如文本字段、文件、二进制数据等），每个数据都有自己的头信息和内容。
- 必须能够接收分片并保存，服务端支持接收不完整的文件，并在接收到分片上传请求时，能够解析请求中的表单数据，并将表单中数据以文件保存到指定位置。
- 服务端能合并分片数据，所有分片上传完成后，服务端需要负责将分片合并为完整的文件。这可由服务端主动合并，也可由客户端发送合并请求被动触发。
断点续传
与断点续下对应，断点续传是指在上传过程中，因网络中断、客户端崩溃或其他原因，导致上传未完成时，任务能够从中断的位置继续上传，而不需要从头开始上传的技术。断点续传同样是为了避免重复的数据传输，节约上传时间。
SFFT基于ohos.data.relationalStore（rdb）和缓存机制实现断点续传。对于每一个新建的上传任务，其上传信息、配置信息、分片信息都会被读取到缓存并持久化到数据库中，当上传任务被执行时，分片信息会根据上传请求的响应结果进行更新，并定期保存到数据库中，以保证上传进度的可靠性。当上传任务需要从被中断处开始继续上传时，SFFT会从数据库中读取还未上传成功的分片，重新发送HTTP请求以上传这些未成功的分片。
使用断点续传技术继续上传的效果图如下所示：
SFFT实现断点续传技术的关键代码如下：
UploadInfoManager：
import { UploadTaskMetadata } from "./UploadTaskMetadata";
import { UploadTaskInfo } from "../../../framework/storage/UploadTaskInfo";
import { UploadStoreHelper } from "../../../framework/storage/UploadStoreHelper";
import { UploadInfoError } from "../../../common/exception/upload/UploadInfoError";
import { ConsistencyError, ConsistencyErrorType } from "../../../common/exception/ConsistencyError";
import { UploadStatusType } from "../../upload/common/UploadStatusType";
import { UploadTaskConstants } from "../common/UploadConstants";
export class UploadInfoManager {
private static instance = new UploadInfoManager();
private constructor() {
}
public static getInstance(): UploadInfoManager {
return UploadInfoManager.instance;
}
public async setTaskInfoByCache(uploadTaskMetadata: UploadTaskMetadata) {
// 如果上传任务为新任务，直接返回
if (uploadTaskMetadata.id !== UploadTaskConstants.DEFAULT_ID) {
return;
}
// 尝试从缓存中匹配上传信息
const taskInfo = UploadStoreHelper.getTaskInfoFromCacheByFileInfo(
uploadTaskMetadata.url, uploadTaskMetadata.filePath, uploadTaskMetadata.uploadFileName);
if (!taskInfo) {
return;
}
// 更新相关信息
uploadTaskMetadata.id = taskInfo.id!;
uploadTaskMetadata.chunkSize = taskInfo.chunkSize;
uploadTaskMetadata.fileSize = await FileUtils.getFileSize(uploadTaskMetadata.filePath);
uploadTaskMetadata.contentType = taskInfo.contentType;
uploadTaskMetadata.status = taskInfo.isCompleted ? UploadStatusType.COMPLETED : uploadTaskMetadata.status;
}
public async checkAndSetTaskInfo(uploadTaskMetadata: UploadTaskMetadata) {
if (uploadTaskMetadata.id !== UploadTaskConstants.DEFAULT_ID) {
return;
}
// 尝试从缓存中匹配上传信息
const taskInfo = UploadStoreHelper.getTaskInfoFromCacheByFileInfo(
uploadTaskMetadata.url, uploadTaskMetadata.filePath, uploadTaskMetadata.uploadFileName);
// 没有匹配到上传信息，抛出断点信息丢失异常
if (!taskInfo) {
throw new ConsistencyError('Breakpoint information lost', ConsistencyErrorType.INFO_LOST_ERROR);
}
const fileHash = await FileUtils.getFileHash(uploadTaskMetadata.filePath);
if (fileHash !== taskInfo.fileHash) {
throw new ConsistencyError('File hash changed', ConsistencyErrorType.LOCAL_FILE_CHANGED_ERROR);
}
// 更新一次相关信息
uploadTaskMetadata.id = taskInfo.id!;
uploadTaskMetadata.chunkSize = taskInfo.chunkSize;
uploadTaskMetadata.fileSize = await FileUtils.getFileSize(uploadTaskMetadata.filePath);
uploadTaskMetadata.contentType = taskInfo.contentType;
uploadTaskMetadata.status = taskInfo.isCompleted ? UploadStatusType.COMPLETED : uploadTaskMetadata.status;
}
public async storeTaskInfo(uploadTaskMetadata: UploadTaskMetadata) {
const fileHash = await FileUtils.getFileHash(uploadTaskMetadata.filePath);
const fileSize = await FileUtils.getFileSize(uploadTaskMetadata.filePath);
if (uploadTaskMetadata.chunkSize === UploadTaskConstants.ZERO_CHUNK_SIZE) {
uploadTaskMetadata.chunkSize = uploadTaskMetadata.isChunk ? this.getChunkSize(fileSize) : fileSize;
}
const totalChunks = Math.ceil(fileSize / uploadTaskMetadata.chunkSize);
// 生成任务信息与初始分片信息
const uploadTaskInfo: UploadTaskInfo = {
id: undefined,
url: uploadTaskMetadata.url,
filePath: uploadTaskMetadata.filePath,
uploadFileName: uploadTaskMetadata.uploadFileName,
contentType: uploadTaskMetadata.contentType ?? '',
fileHash: fileHash,
chunkSize: uploadTaskMetadata.chunkSize,
isCompleted: false,
unsolvedChunkList: new Set([...Array(totalChunks)].map((_: number, i) => i)),
}
try {
// 保存任务信息到数据库，如果成功，返回一个任务id，并存储到缓存中，更新taskInfo相关信息
const taskId = await UploadStoreHelper.storeTaskInfo(uploadTaskInfo);
uploadTaskMetadata.id = taskId;
uploadTaskMetadata.fileSize = fileSize;
} catch (err) {
throw err as UploadInfoError;
}
}
public async getUnsolvedChunkStatus(uploadTaskMetadata: UploadTaskMetadata, index: number,
isFromDb: boolean): Promise<boolean | undefined> {
try {
return (await UploadStoreHelper.getChunkStatusByIdAndIndex(uploadTaskMetadata.id, index, isFromDb));
} catch (err) {
throw err as UploadInfoError;
}
}
public async getUnsolvedChunkList(uploadTaskMetadata: UploadTaskMetadata,
isFromDb: boolean): Promise<Set<number> | undefined> {
try {
return (await UploadStoreHelper.getUnsolvedChunkListById(uploadTaskMetadata.id, isFromDb));
} catch (err) {
throw err as UploadInfoError;
}
}
public async updateUnsolvedChunkList(uploadTaskMetadata: UploadTaskMetadata, index: number, isSolved: boolean) {
try {
await UploadStoreHelper.updateUnsolvedChunkListById(uploadTaskMetadata.id, index, isSolved);
} catch (err) {
throw err as UploadInfoError;
}
}
public async updateTaskInfo(uploadTaskMetadata: UploadTaskMetadata) {
try {
await UploadStoreHelper.updateTaskInfoById(uploadTaskMetadata.id);
} catch (err) {
throw err as UploadInfoError;
}
}
public async deleteUploadInfo(uploadTaskMetadata: UploadTaskMetadata) {
try {
const taskInfo = UploadStoreHelper.getTaskInfoFromCacheByFileInfo(
uploadTaskMetadata.url, uploadTaskMetadata.filePath, uploadTaskMetadata.uploadFileName);
if (!taskInfo) {
return;
}
await UploadStoreHelper.deleteUploadInfoById(taskInfo.id!);
} catch (err) {
throw err as UploadInfoError;
}
}
private getChunkSize(fileSize: number): number {
if (MathUtils.isInRange(fileSize, 0, UploadTaskConstants.DEFAULT_BIG_FILE_SIZE)) {
return UploadTaskConstants.DEFAULT_CHUNK_SIZE;
} else {
return Math.ceil(fileSize / 1000);
}
}
}UploadController：
import { BusinessError } from '@kit.BasicServicesKit';
import { UploadCall } from './UploadCall';
import { UploadTaskMetadata } from './UploadTaskMetadata';
import { UploadInfoManager } from './UploadInfoManager';
import { UploadStatusManager } from './UploadStatusManager';
import { UploadProgressInfo } from '../api/UploadTask';
import { UploadStatusType } from '../common/UploadStatusType';
export class UploadController {
private uploadTaskMetadata: UploadTaskMetadata;
private uploadCall: UploadCall;
constructor(uploadTask: UploadTaskMetadata) {
this.uploadTaskMetadata = uploadTask;
this.uploadCall = new UploadCall(uploadTask);
}
public async start() {
try {
if (this.uploadTaskMetadata.status === UploadStatusType.UPLOADING) {
return;
}
this.uploadCall.uploadProgress.clearUploadProgress();
// 清除相关上传信息
await UploadInfoManager.getInstance().deleteUploadInfo(this.uploadTaskMetadata);
// 生成新任务并存储任务信息和分片信息到数据库与缓存
await UploadInfoManager.getInstance().storeTaskInfo(this.uploadTaskMetadata);
// 进入上传状态
UploadStatusManager.getInstance().setStatus(UploadStatusType.UPLOADING, this.uploadTaskMetadata);
// 开始上传回调
this.uploadTaskMetadata.listener?.onStart?.();
Logger.info(LoggerConstants.UPLOAD, `Start upload, url: ${this.uploadTaskMetadata.url}, fileName: ${this.uploadTaskMetadata.uploadFileName}`);
// 进行文件上传
await this.uploadCall.execute();
} catch (err) {
this.dealWithError(err);
}
}
public async pause() {
// 不启用断点续传（分片）的情况下，暂停不生效
if (!this.uploadTaskMetadata.isChunk) {
Logger.info(LoggerConstants.UPLOAD, `Cannot pause upload if isChunk is disabled`);
return;
}
if (this.uploadTaskMetadata.status === UploadStatusType.PAUSED) {
return;
}
try {
// 状态迁移，修改取消信号为手动设置
UploadStatusManager.getInstance().setStatus(UploadStatusType.PAUSED, this.uploadTaskMetadata);
// 更新数据库上传信息
if (this.uploadTaskMetadata.isChunk) {
await UploadInfoManager.getInstance().updateTaskInfo(this.uploadTaskMetadata);
}
const uploadProgressInfo = await this.getProgress();
this.uploadTaskMetadata.listener?.onPause?.(uploadProgressInfo);
Logger.info(LoggerConstants.UPLOAD, `Pause upload, url: ${this.uploadTaskMetadata.url}, fileName: ${this.uploadTaskMetadata.uploadFileName}`);
} catch (err) {
this.dealWithError(err);
}
}
public async resume() {
// 不启用断点续传（分片）的情况下，续下不生效
if (!this.uploadTaskMetadata.isChunk) {
Logger.info(LoggerConstants.UPLOAD, `Cannot resume upload if isResumable is disabled`);
return;
}
if (this.uploadTaskMetadata.status === UploadStatusType.UPLOADING) {
return;
}
try {
// 尝试从缓存中匹配上传信息并写入到uploadTaskMetadata，不存在/不匹配则无法上传
UploadInfoManager.getInstance().checkAndSetTaskInfo(this.uploadTaskMetadata);
// 状态迁移
UploadStatusManager.getInstance().setStatus(UploadStatusType.UPLOADING, this.uploadTaskMetadata);
// 暂停上传回调
const uploadProgressInfo = await this.getProgress();
this.uploadTaskMetadata.listener?.onResume?.(uploadProgressInfo);
Logger.info(LoggerConstants.UPLOAD,
`Resume upload, url: ${this.uploadTaskMetadata.url}, fileName: ${this.uploadTaskMetadata.uploadFileName}`);
// 进行文件上传
await this.uploadCall.execute();
} catch (err) {
this.dealWithError(err);
}
}
public async cancel() {
try {
this.uploadCall.uploadProgress.clearUploadProgress();
// 状态迁移
UploadStatusManager.getInstance().setStatus(UploadStatusType.CANCELED, this.uploadTaskMetadata);
// 清除上传记录
await UploadInfoManager.getInstance().deleteUploadInfo(this.uploadTaskMetadata);
// 取消上传回调
this.uploadTaskMetadata.listener?.onCancel?.();
Logger.warn(LoggerConstants.UPLOAD,
`Cancel upload, url: ${this.uploadTaskMetadata.url}, fileName: ${this.uploadTaskMetadata.uploadFileName}`);
} catch (err) {
this.dealWithError(err);
}
}
public async getProgress(): Promise<UploadProgressInfo> {
try {
// 尝试从缓存中匹配downloadTask信息
await UploadInfoManager.getInstance().setTaskInfoByCache(this.uploadTaskMetadata);
// 获取当前下载进度
return await this.uploadCall.getProgress();
} catch (err) {
Logger.error(LoggerConstants.UPLOAD, `Get progress failed,code: ${err.code}, message: ${err.message}`);
return {
transferredSize: 0,
totalSize: 0,
speed: 0,
}
}
}
private dealWithError(err: BusinessError) {
if (err instanceof UploadStatusError) {
return;
}
UploadStatusManager.getInstance().setStatus(UploadStatusType.FAILED, this.uploadTaskMetadata);
this.uploadTaskMetadata.listener?.onFail?.(err);
throw new UploadTaskError(err.message, err.code);
}
}
扫一扫
983
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
