Django大文件分块上传和分块下载

下面是简单的demo，可能有Bug，需要的看官拿去自己拓展吧。

views.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import re
import stat
import base64
import shutil
import pickle
import rehash
import mimetypes
import posixpath
from django.conf import settings
from urllib.parse import unquote
from django.core.cache import caches
from django.utils._os import safe_join
from django.utils.http import http_date
from django.views.static import was_modified_since
from django.contrib.auth.decorators import login_required
from django.http import JsonResponse, Http404, FileResponse, HttpResponseNotModified


def remove_file(file_path):
    if not os.path.isfile(file_path):
        return
    os.remove(file_path)


def write_file(file_path_temp, file):
    if not os.path.isdir(os.path.dirname(file_path_temp)):
        os.makedirs(os.path.dirname(file_path_temp))
    try:
        with open(file_path_temp, 'ab') as destination:
            destination.write(file['code'])
    except OSError as exc:
        return exc.errno


def check_file(file, file_path, file_path_temp):
    # is_end表示整个文件都传输完毕了

    if file['percent'][0] == '0' and os.path.exists(file_path_temp):
        remove_file(file_path_temp)  # 开始上传时删除已有的暂存文件
    elif file['percent'][0] != '0' and not os.path.exists(file_path_temp):
        # 上传过程中暂存文件丢失
        # firefox浏览器调用接口删除文件并取消传输后还会上传一个文件块，这里做兼容
        return {'code': '1', 'msg': '暂存文件丢失，上传失败', 'is_end': True}

    # 校验文件内容及大小
    if not file['code']:
        return {'code': '1', 'msg': '不能上传空文件'}
    if not 0 < file['size'] <= settings.max_size:
        return {'code': '1', 'msg': '文件大小不符合要求'}

    # 获得已传输的文件大小
    file_size = len(file['code'])
    if os.path.exists(file_path_temp):
        file_size += os.path.getsize(file_path_temp)

    # 校验文件大小
    if not 0 < file_size <= settings.max_size:
        return {'code': '1', 'msg': '文件大小不符合要求'}
    if (file['percent'][1] == '100' and file_size != file['size']) or file_size > file['size']:
        return {'code': '1', 'msg': '文件大小校验失败'}

    # 生成已传输的文件块的md5值
    if file['percent'][0] == '0':
        file_hash = rehash.md5()
    else:
        file_hash = caches['default'].get(file_path_temp)
        if not file_hash:
            return {'code': '1', 'msg': '文件校验值过期'}
        file_hash = pickle.loads(base64.b64decode(file_hash.encode()))
    file_hash.update(file['code'])

    # 校验md5值，保证内容一致性
    if file['md5'] != file_hash.hexdigest():
        return {'code': '1', 'msg': '文件校验失败'}

    if file['percent'][1] == '100' or file_size == file['size']:  # 传输完毕的状态
        if write_file(file_path_temp, file) == 36:
            return {'code': '1', 'msg': '文件名过长'}
        shutil.move(file_path_temp, file_path)
        return {'code': '0', 'msg': '上传并提交成功', 'is_end': True}
    else:
        if write_file(file_path_temp, file) == 36:
            return {'code': '1', 'msg': '文件名过长'}

        # 暂存md5的hash值
        caches['default'].set(file_path_temp, base64.b64encode(pickle.dumps(file_hash)).decode(), 60)
        return {'code': '0', 'msg': '上传并提交文件块成功', 'is_end': False}


@login_required
def upload_file(request):
    file = {
        'name': unquote(request.META['HTTP_FILE_UPLOAD_NAME']),
        'percent': request.META['HTTP_FILE_UPLOAD_PERCENT'].replace('NaN-Infinity', '0-100').split('-'),
        'size': int(request.META['HTTP_FILE_UPLOAD_SIZE']),
        'md5': request.META['HTTP_FILE_UPLOAD_MD5'],
        'code': request.body
    }  # 文件分为不大于100KB的小块传输

    # 传输过程中直接跳到其他页面
    if file['percent'][1].startswith('http'):
        return JsonResponse({'code': '-4', 'msg': '传输已终止', 'is_end': True})

    file_path = os.path.join(settings.BASE_DIR, 'upload_files/%s' % file['name'])
    file_path_temp = file_path + '.temp'

    if file['percent'][0] == '0' and caches['default'].get(file_path):  # 同名文件上传冲突，不进行其它操作
        return JsonResponse({'code': '1', 'msg': '同名文件正在上传'})
    else:  # 标记文件正在上传
        caches['default'].set(file_path, 'uploading', 2)

    response = check_file(file, file_path, file_path_temp)
    # 文件上传失败或者上传完毕后，清理暂存文件和缓存
    if response['code'] != '0' or response['is_end']:
        remove_file(file_path_temp)
        caches['default'].delete_pattern(file_path + '*')
    return JsonResponse(response)


@login_required
def download_file(request, path, document_root=None):
    # 防止目录遍历漏洞
    path = posixpath.normpath(path).lstrip('/')
    fullpath = safe_join(document_root, path)
    if os.path.isdir(fullpath):
        raise Http404('Directory indexes are not allowed here.')
    if not os.path.exists(fullpath):
        raise Http404('"%(path)s" does not exist' % {'path': fullpath})

    statobj = os.stat(fullpath)

    # 判断下载过程中文件是否被修改过
    if not was_modified_since(request.META.get('HTTP_IF_MODIFIED_SINCE'),
                              statobj.st_mtime, statobj.st_size):
        return HttpResponseNotModified()

    # 获取文件的content_type
    content_type, encoding = mimetypes.guess_type(fullpath)
    content_type = content_type or 'application/octet-stream'

    # 计算读取文件的起始位置
    start_bytes = re.search(r'bytes=(\d+)-', request.META.get('HTTP_RANGE', ''), re.S)
    start_bytes = int(start_bytes.group(1)) if start_bytes else 0

    # 打开文件并移动下标到起始位置，客户端点击继续下载时，从上次断开的点继续读取
    the_file = open(fullpath, 'rb')
    the_file.seek(start_bytes, os.SEEK_SET)

    # status=200表示下载开始，status=206表示下载暂停后继续，为了兼容火狐浏览器而区分两种状态
    # 关于django的response对象，参考：https://www.cnblogs.com/scolia/p/5635546.html
    # 关于response的状态码，参考：https://www.cnblogs.com/DeasonGuan/articles/Hanami.html
    # FileResponse默认block_size = 4096，因此迭代器每次读取4KB数据
    response = FileResponse(the_file, content_type=content_type, status=206 if start_bytes > 0 else 200)

    # 'Last-Modified'表示文件修改时间，与'HTTP_IF_MODIFIED_SINCE'对应使用，参考：https://www.jianshu.com/p/b4ecca41bbff
    response['Last-Modified'] = http_date(statobj.st_mtime)

    # 这里'Content-Length'表示剩余待传输的文件字节长度
    if stat.S_ISREG(statobj.st_mode):
        response['Content-Length'] = statobj.st_size - start_bytes
    if encoding:
        response['Content-Encoding'] = encoding

    # 'Content-Range'的'/'之前描述响应覆盖的文件字节范围，起始下标为0，'/'之后描述整个文件长度，与'HTTP_RANGE'对应使用
    # 参考：http://liqwei.com/network/protocol/2011/886.shtml
    response['Content-Range'] = 'bytes %s-%s/%s' % (start_bytes, statobj.st_size - 1, statobj.st_size)

    # 'Cache-Control'控制浏览器缓存行为，此处禁止浏览器缓存，参考：https://blog.youkuaiyun.com/cominglately/article/details/77685214
    response['Cache-Control'] = 'no-cache, no-store, must-revalidate'
    return response

upload.js

import SparkMD5 from 'spark-md5'
import ajax_fetch from '@/config/ajax'
import { MessageBox, Message } from 'element-ui'

function onUpload(file, size, name, start, end, md5) {
    return new Promise((resolve, reject) => {
        ajax_fetch('/upload/', file, 'POST', {
            'file-upload-name': encodeURIComponent(name),
            'file-upload-percent': start + '-' + end,
            'file-upload-size': size,
            'file-upload-md5': md5
            'x-csrftoken': window.localStorage.getItem('csrfmiddlewaretoken'),
        }).then((res) => {
            resolve(res);
        }).catch(err => {
            reject(err)
        });
    })
}

async function uploadFile() {
    if(!document.getElementsByName('uploadForm')[0].files[0]){
        return;
    }
    return new Promise(function (resolve, reject) {
        var blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice,  // 兼容切片方法
            file = document.getElementsByName('uploadForm')[0].files[0],  // 文件对象
            chunkSize = 2 * 1024 * 1024,                             // 每块文件2MB
            chunks = Math.ceil(file.size / chunkSize),               // 文件分多少块
            currentChunk = 0,                                        // 已经传输了多少块
            md5 = '',
            spark = new SparkMD5.ArrayBuffer(),
            fileReader = new FileReader();
        fileReader.onload = async function (e) {
            spark.append(e.target.result);
            md5 = spark.digest();
            var start_p = (currentChunk / chunks) * 100,
                end_p = ((currentChunk + 1) / chunks) * 100;
                let response = await onUpload(e.target.result, file.size, file.name, start_p, end_p, md5);
            if (response.code == 0 && response.is_end) {
                // 传输文件结束
                resolve(response);
                Message.success(file.name + '文件上传成功！');
            } else if (response.code == 0 && !response.is_end) {
                if (/upload-file/.test(window.location.href)) {  //文件块传输成功，传输下一块
                    currentChunk++;
                    loadNext();
                } else {  // 传输过程中跳转到其他页面了
                    onUpload(e.target.result, file.size, file.name, 1, window.location.href, md5);
                }
            } else if (response.code == -4) {
                spark.end();  // 释放内存
            } else {
                spark.end();  // 释放内存
                Message.error(response.msg);
            }
        };
        function loadNext() {
            var start = currentChunk * chunkSize,
                end = ((start + chunkSize) >= file.size) ? file.size : start + chunkSize;
            fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));  // 这里会触发onload事件
        }
        loadNext();
    }).catch((err)=>{
        console.log('文件传输失败')
    })
}

export default uploadFile

urls.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
from site import views
from django.urls import re_path

urlpatterns = [
    re_path(r'^/upload/?$', views.upload_file),
    re_path(r'^/download/(?P<path>.*)$', views.download_file, {'document_root': settings.MEDIA_ROOT}),
]