本文介绍了如何通过浏览器实现实现超大文件上传和断点续传，特别是针对高清视频、4K视频和大数据应用。文章提到了基于脚本语言的上传性能不足，转而采用C++实现的高性能上传服务器，该服务器支持4GB以上文件、断点续传，并且兼容多种浏览器。此外，还分享了一个开源项目Hyper Upload Server的详细信息及其主要特性。
随着视频网站和大数据应用的兴起，特别是高清视频、4K视频、全景VR和大数据应用的到来，超大文件上传已经成为了日常的应用需求。
基于云计算的架构，我们需要将各地的大数据上传到云平台进行存储和处理，那么，动辄几个GB或者TB的大数据文件如何上传？在网络抖动的情况下如何保障断点续传？如何通过多终端快速上传？这是现实中常常遇到的问题。
为了通用性更强，采用安装特定的浏览器上传插件方式并不可取，最好的实现方式就是通过浏览器直接上传，当前已经有很多基于PHP或者Java程序来实现的上传服务器，然而在实际使用中你会会发现，这些基于脚本语言实现的上传功能模块性能很弱，一是不支持2GB以上的文件上传；二是无法支持断点续传；三是效率极低，单台服务器最多支持几十个并发上传连接。
因此，我们为了解决这个问题，花费了很大精力自主用C++语言实现了这一高性能上传服务器，在客户端只要调用一段 js 代码即可，这是一个通用的上传服务模块，可以快速集成到第三方应用平台中。
项目地址：
https://github.com/liufeihong/Hyper-Upload-Server
Hyper Upload Server 超级上传服务器简介：
这是一款超级文件上传服务器，采用异步I/O架构，采用C++语言编码实现。它支持4GB以上超大文件上传和断点续传，支持Windows和Linux服务器平台，支持任意格式的文件上传，尤其适合大的视频网站应用。单台服务器支持1000并发上传进程，支持PC端和智能手机端主流的浏览器。
主要特性
1. 服务器端采用异步I/O架设设计，具有高性能I/O处理能力，尤其适用于超大文件上传；
2. 服务器端采用高效内存分配技术确保在运行过程中服务器的内存开销最小化；
3. 完全采用标准协议实现，因此兼容几乎所有的PC端和移动端浏览器；
4. 服务器端采用C++语言自主实现，对上传文件的尺寸无限制，天生支持超大文件上传。
而基于PHP、JAVA等技术实现的文件上传服务天生无法支持超大文件上传，无法逾越2GB的最大文件尺寸瓶颈；
5. 服务器端采用无缓冲即时写入方式，上传数据写入一步到位。不同于PHP、JAVA等技术实现方式需要两步写入；
6. 服务器端可跨平台编译运行，支持Windows和Linux平台；
7. 高性能，单台服务器支持1000个并发上传进程；
8. 支持4GB以上超大文件上传，文件大小不受限制；
9. 客户端支持采用HTTP标准协议上传；
10.支持断点续传，断网、关机重启均不受影响；
11.支持HTML5浏览器上传进度实时显示；
12.支持IE8及以上浏览器上传进度显示；
13.支持查看客户端在线连接， 查看方法： http://ip:port/lists
14.多浏览器兼容，包括Chrome，Firefox，Safari，IE，Opera,Edge；
上传代码实现
;
/*--jquery.md5.js begin--*/
(function($){
var rotateLeft = function(lValue, iShiftBits) {
return (lValue << iShiftBits) | (lValue >>> (32 - iShiftBits));
}
var addUnsigned = function(lX, lY) {
var lX4, lY4, lX8, lY8, lResult;
lX8 = (lX & 0x80000000);
lY8 = (lY & 0x80000000);
lX4 = (lX & 0x40000000);
lY4 = (lY & 0x40000000);
lResult = (lX & 0x3FFFFFFF) + (lY & 0x3FFFFFFF);
if (lX4 & lY4) return (lResult ^ 0x80000000 ^ lX8 ^ lY8);
if (lX4 | lY4) {
if (lResult & 0x40000000) return (lResult ^ 0xC0000000 ^ lX8 ^ lY8);
else return (lResult ^ 0x40000000 ^ lX8 ^ lY8);
} else {
return (lResult ^ lX8 ^ lY8);
}
}
var F = function(x, y, z) {
return (x & y) | ((~ x) & z);
}
var G = function(x, y, z) {
return (x & z) | (y & (~ z));
}
var H = function(x, y, z) {
return (x ^ y ^ z);
}
var I = function(x, y, z) {
return (y ^ (x | (~ z)));
}
var FF = function(a, b, c, d, x, s, ac) {
a = addUnsigned(a, addUnsigned(addUnsigned(F(b, c, d), x), ac));
return addUnsigned(rotateLeft(a, s), b);
};
var GG = function(a, b, c, d, x, s, ac) {
a = addUnsigned(a, addUnsigned(addUnsigned(G(b, c, d), x), ac));
return addUnsigned(rotateLeft(a, s), b);
};
var HH = function(a, b, c, d, x, s, ac) {
a = addUnsigned(a, addUnsigned(addUnsigned(H(b, c, d), x), ac));
return addUnsigned(rotateLeft(a, s), b);
};
var II = function(a, b, c, d, x, s, ac) {
a = addUnsigned(a, addUnsigned(addUnsigned(I(b, c, d), x), ac));
return addUnsigned(rotateLeft(a, s), b);
};
var convertToWordArray = function(string) {
var lWordCount;
var lMessageLength = string.length;
var lNumberOfWordsTempOne = lMessageLength + 8;
var lNumberOfWordsTempTwo = (lNumberOfWordsTempOne - (lNumberOfWordsTempOne % 64)) / 64;
var lNumberOfWords = (lNumberOfWordsTempTwo + 1) * 16;
var lWordArray = Array(lNumberOfWords - 1);
var lBytePosition = 0;
var lByteCount = 0;
while (lByteCount < lMessageLength) {
lWordCount = (lByteCount - (lByteCount % 4)) / 4;
lBytePosition = (lByteCount % 4) * 8;
lWordArray[lWordCount] = (lWordArray[lWordCount] | (string.charCodeAt(lByteCount) << lBytePosition));
lByteCount++;
}
lWordCount = (lByteCount - (lByteCount % 4)) / 4;
lBytePosition = (lByteCount % 4) * 8;
lWordArray[lWordCount] = lWordArray[lWordCount] | (0x80 << lBytePosition);
lWordArray[lNumberOfWords - 2] = lMessageLength << 3;
lWordArray[lNumberOfWords - 1] = lMessageLength >>> 29;
return lWordArray;
};
var wordToHex = function(lValue) {
var WordToHexValue = "", WordToHexValueTemp = "", lByte, lCount;
for (lCount = 0; lCount <= 3; lCount++) {
lByte = (lValue >>> (lCount * 8)) & 255;
WordToHexValueTemp = "0" + lByte.toString(16);
WordToHexValue = WordToHexValue + WordToHexValueTemp.substr(WordToHexValueTemp.length - 2, 2);
}
return WordToHexValue;
};
var uTF8Encode = function(string) {
string = string.replace(/\x0d\x0a/g, "\x0a");
var output = "";
for (var n = 0; n < string.length; n++) {
var c = string.charCodeAt(n);
if (c < 128) {
output += String.fromCharCode(c);
} else if ((c > 127) && (c < 2048)) {
output += String.fromCharCode((c >> 6) | 192);
output += String.fromCharCode((c & 63) | 128);
} else {
output += String.fromCharCode((c >> 12) | 224);
output += String.fromCharCode(((c >> 6) & 63) | 128);
output += String.fromCharCode((c & 63) | 128);
}
}
return output;
};
$.extend({
md5: function(string) {
var x = Array();
var k, AA, BB, CC, DD, a, b, c, d;
var S11=7, S12=12, S13=17, S14=22;
var S21=5, S22=9 , S23=14, S24=20;
var S31=4, S32=11, S33=16, S34=23;
var S41=6, S42=10, S43=15, S44=21;
string = uTF8Encode(string);
x = convertToWordArray(string);
a = 0x67452301; b = 0xEFCDAB89; c = 0x98BADCFE; d = 0x10325476;
for (k = 0; k < x.length; k += 16) {
AA = a; BB = b; CC = c; DD = d;
a = FF(a, b, c, d, x[k+0], S11, 0xD76AA478);
d = FF(d, a, b, c, x[k+1], S12, 0xE8C7B756);
c = FF(c, d, a, b, x[k+2], S13, 0x242070DB);
b = FF(b, c, d, a, x[k+3], S14, 0xC1BDCEEE);
a = FF(a, b, c, d, x[k+4], S11, 0xF57C0FAF);
d = FF(d, a, b, c, x[k+5], S12, 0x4787C62A);
c = FF(c, d, a, b, x[k+6], S13, 0xA8304613);
b = FF(b, c, d, a, x[k+7], S14, 0xFD469501);
a = FF(a, b, c, d, x[k+8], S11, 0x698098D8);
d = FF(d, a, b, c, x[k+9], S12, 0x8B44F7AF);
c = FF(c, d, a, b, x[k+10], S13, 0xFFFF5BB1);
b = FF(b, c, d, a, x[k+11], S14, 0x895CD7BE);
a = FF(a, b, c, d, x[k+12], S11, 0x6B901122);
d = FF(d, a, b, c, x[k+13], S12, 0xFD987193);
c = FF(c, d, a, b, x[k+14], S13, 0xA679438E);
b = FF(b, c, d, a, x[k+15], S14, 0x49B40821);
a = GG(a, b, c, d, x[k+1], S21, 0xF61E2562);
d = GG(d, a, b, c, x[k+6], S22, 0xC040B340);
c = GG(c, d, a, b, x[k+11], S23, 0x265E5A51);
b = GG(b, c, d, a, x[k+0], S24, 0xE9B6C7AA);
a = GG(a, b, c, d, x[k+5], S21, 0xD62F105D);
d = GG(d, a, b, c, x[k+10], S22, 0x2441453);
c = GG(c, d, a, b, x[k+15], S23, 0xD8A1E681);
b = GG(b, c, d, a, x[k+4], S24, 0xE7D3FBC8);
a = GG(a, b, c, d, x[k+9], S21, 0x21E1CDE6);
d = GG(d, a, b, c, x[k+14], S22, 0xC33707D6);
c = GG(c, d, a, b, x[k+3], S23, 0xF4D50D87);
b = GG(b, c, d, a, x[k+8], S24, 0x455A14ED);
a = GG(a, b, c, d, x[k+13], S21, 0xA9E3E905);
d = GG(d, a, b, c, x[k+2], S22, 0xFCEFA3F8);
c = GG(c, d, a, b, x[k+7], S23, 0x676F02D9);
b = GG(b, c, d, a, x[k+12], S24, 0x8D2A4C8A);
a = HH(a, b, c, d, x[k+5], S31, 0xFFFA3942);
d = HH(d, a, b, c, x[k+8], S32, 0x8771F681);
c = HH(c, d, a, b, x[k+11], S33, 0x6D9D6122);
b = HH(b, c, d, a, x[k+14], S34, 0xFDE5380C);
a = HH(a, b, c, d, x[k+1], S31, 0xA4BEEA44);
d = HH(d, a, b, c, x[k+4], S32, 0x4BDECFA9);
c = HH(c, d, a, b, x[k+7], S33, 0xF6BB4B60);
b = HH(b, c, d, a, x[k+10], S34, 0xBEBFBC70);
a = HH(a, b, c, d, x[k+13], S31, 0x289B7EC6);
d = HH(d, a, b, c, x[k+0], S32, 0xEAA127FA);
c = HH(c, d, a, b, x[k+3], S33, 0xD4EF3085);
b = HH(b, c, d, a, x[k+6], S34, 0x4881D05);
a = HH(a, b, c, d, x[k+9], S31, 0xD9D4D039);
d = HH(d, a, b, c, x[k+12], S32, 0xE6DB99E5);
c = HH(c, d, a, b, x[k+15], S33, 0x1FA27CF8);
b = HH(b, c, d, a, x[k+2], S34, 0xC4AC5665);
a = II(a, b, c, d, x[k+0], S41, 0xF4292244);
d = II(d, a, b, c, x[k+7], S42, 0x432AFF97);
c = II(c, d, a, b, x[k+14], S43, 0xAB9423A7);
b = II(b, c, d, a, x[k+5], S44, 0xFC93A039);
a = II(a, b, c, d, x[k+12], S41, 0x655B59C3);
d = II(d, a, b, c, x[k+3], S42, 0x8F0CCC92);
c = II(c, d, a, b, x[k+10], S43, 0xFFEFF47D);
b = II(b, c, d, a, x[k+1], S44, 0x85845DD1);
a = II(a, b, c, d, x[k+8], S41, 0x6FA87E4F);
d = II(d, a, b, c, x[k+15], S42, 0xFE2CE6E0);
c = II(c, d, a, b, x[k+6], S43, 0xA3014314);
b = II(b, c, d, a, x[k+13], S44, 0x4E0811A1);
a = II(a, b, c, d, x[k+4], S41, 0xF7537E82);
d = II(d, a, b, c, x[k+11], S42, 0xBD3AF235);
c = II(c, d, a, b, x[k+2], S43, 0x2AD7D2BB);
b = II(b, c, d, a, x[k+9], S44, 0xEB86D391);
a = addUnsigned(a, AA);
b = addUnsigned(b, BB);
c = addUnsigned(c, CC);
d = addUnsigned(d, DD);
}
var tempValue = wordToHex(a) + wordToHex(b) + wordToHex(c) + wordToHex(d);
return tempValue.toLowerCase();
}
});
})(jQuery);
/*--jquery.md5.js end--*/
/*--- HYUpload.js-begin--*/
function toast(msg,duration)
{
var toastDiv = document.getElementById('hyupload-toast');
if (msg) {
toastDiv.style.display = 'block';
toastDiv.innerHTML = msg;
duration = parseInt(duration);
if (!duration) duration = 3000;
if (!window.uploadtoast) {
window.clearTimeout(window.uploadtoast);
}
window.uploadtoast = window.setTimeout(function() {
toastDiv.style.display = 'none';
delete window.uploadtoast;
},duration);
}
else {
toastDiv.style.display = 'none';
toastDiv.innerHTML = '';
}
}
Array.prototype.in_array=function(e){
var S=String.fromCharCode(2);
var r=new RegExp(S+e+S);
return (r.test(S+this.join(S)+S));
};
var HYFileUploader = function (options) {
var _options = {
filelist:'hyupload-filelist',
notice_url:'',
savepaths:{'video':'.mp4,.mkv,.avi,.wmv','document':'.zip,.rar','image':'.jpg,.jpeg,.png,.gif','audio':'.aac,.mp3'},
ontaskfinish:function(taskobj) {
},
ontaskprogress:function(taskobj) {
},
ontaskprogress:function(progress) {
},
ontaskstart:function(taskobj) {
return true;
}
};
$.extend(_options, options);
//简单的Cookie帮助函数
var setCookie = function(cname,cvalue,exdays) {
var d = new Date();
d.setTime(d.getTime()+(exdays*24*60*60*1000));
var expires = "expires="+d.toGMTString();
document.cookie = cname + "=" + cvalue + "; " + expires;
};
var getCookie = function(cname) {
var name = cname + "=";
var ca = document.cookie.split(';');
for(var i=0; i<ca.length; i++)
{
var c = ca[i].trim();
if (c.indexOf(name)==0) return c.substring(name.length,c.length);
}
return "";
};
//
//简单的文件HASH值计算，如果您不是十分考究，应该可以用于产品。
//由于计算文件HASH值用到了多种数据，因此在HYFileUploader系统范围内发生HASH冲突的可能性应该非常小，应该可以放心使用。
//获取文件的ID可以用任何算法来实现，只要保证做到同一文件的ID是相同的即可，获取的ID长度不要超过32字节
//
var getFileId = function (file) {
//给浏览器授予一个唯一的ID用于区分不同的浏览器实例
var clientid = getCookie("HUAYIUPLOAD");
if (clientid == "") {
//用一个随机值来做浏览器的ID，将作为文件HASH值的一部分
var rand = parseInt(Math.random() * 1000);
var t = (new Date()).getTime();
clientid =rand+'T'+t;
setCookie("HUAYIUPLOAD",clientid,365);
}
var info = clientid;
if (file.lastModified)
info += file.lastModified;
if (file.name)
info += file.name;
if (file.size)
info += file.size;
//https://cdn.bootcss.com/blueimp-md5/2.10.0/js/md5.min.js
var fileid = $.md5(info);
return fileid;
};
var get_file_mimetype = function(filename) {
var mimeType;
var extPos = filename.lastIndexOf('.');
if (extPos != -1) {
var extName = filename.substr(extPos).toLowerCase();
if (extName == '.rm' || extName == '.rmvb') {
mimeType = 'video/realvideo';
}
else if (extName == '.dat') {
mimeType = 'video/vcd';
}
else if (extName == '.ra') {
mimeType = 'audio/realaudio';
}
}
if (!mimeType) mimeType = 'bin/unknown';
return mimeType;
};
//savepaths:{'video':'.mp4,*.mkv,.avi,.wmv','document':'.zip,*.rar','image':'.jpg,.jpeg,.png,.gif','audio':'.aac,.mp3'}
var get_save_path = function(filename,savepaths) {
var extPos = filename.lastIndexOf('.');
if (extPos != -1) {
var extName = filename.substr(extPos).toLowerCase();
for (path in savepaths) {
var extList = savepaths[path].split(',');
if (extList.indexOf(extName) != -1) {
return path;
}
}
}
return '';
}
var bytesToSize = function (bytes) {
if (bytes === 0) return '0 B';
if (bytes < 1024) return (bytes + 'B');
var k = 1024,sizes = ['B','KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
var i = Math.floor(Math.log(bytes) / Math.log(k));
return (bytes / Math.pow(k, i)).toFixed(2) + ' ' + sizes[i];
}
var _addFileToList = function (listViewer,fileobj) {
var fileSources = '';
var fileid = '';
var filename = '',filesize = 0;
var fileitemId = '';
var $listViewer = $(listViewer);
if (fileobj instanceof Array) {
fileSources = '<filesources>';
for(var i = 0; i < fileobj.length; i ++) {
if (filename != '') filename += '|';
var file = fileobj[i];
filename += file.name;
filesize += file.size;
fileid = getFileId(file);
&nb
最低0.47元/天 解锁文章
扫一扫
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
