Python全栈项目--云存储系统开发

原创已于 2025-09-29 15:06:18 修改 · 428 阅读

9 ·

CC 4.0 BY-SA版权

文章标签：

#python #开发语言

于 2025-09-29 15:05:38 首次发布

Python项目实战专栏收录该内容

50 篇文章

订阅专栏

项目概述

云存储系统是一种基于云计算技术的数据存储解决方案，允许用户通过网络上传、下载、管理和共享文件。本文将详细介绍如何使用Python全栈技术开发一个功能完整的云存储系统。

技术栈选择

后端技术

框架: Flask/Django
数据库: PostgreSQL/MySQL + Redis
存储: 本地文件系统或对象存储(MinIO/AWS S3)
认证: JWT Token
异步任务: Celery

前端技术

框架: Vue.js/React
UI组件库: Element UI/Ant Design
HTTP客户端: Axios
状态管理: Vuex/Redux

核心功能模块

1. 用户认证模块

实现用户注册、登录、权限管理等功能。

from flask import Flask, request, jsonify
from flask_jwt_extended import JWTManager, create_access_token
from werkzeug.security import generate_password_hash, check_password_hash

app = Flask(__name__)
app.config['JWT_SECRET_KEY'] = 'your-secret-key'
jwt = JWTManager(app)

@app.route('/register', methods=['POST'])
def register():
    data = request.get_json()
    username = data.get('username')
    password = data.get('password')
    
    # 密码加密
    hashed_password = generate_password_hash(password)
    
    # 存储用户信息到数据库
    # user = User(username=username, password=hashed_password)
    # db.session.add(user)
    # db.session.commit()
    
    return jsonify({'message': '注册成功'}), 201

@app.route('/login', methods=['POST'])
def login():
    data = request.get_json()
    username = data.get('username')
    password = data.get('password')
    
    # 验证用户
    # user = User.query.filter_by(username=username).first()
    # if user and check_password_hash(user.password, password):
    access_token = create_access_token(identity=username)
    return jsonify({'access_token': access_token}), 200

2. 文件上传模块

支持单文件和多文件上传，实现分片上传功能。

import os
from flask import request
from werkzeug.utils import secure_filename

UPLOAD_FOLDER = '/path/to/uploads'
ALLOWED_EXTENSIONS = {'txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif', 'doc', 'docx'}

def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

@app.route('/upload', methods=['POST'])
def upload_file():
    if 'file' not in request.files:
        return jsonify({'error': '没有文件'}), 400
    
    file = request.files['file']
    
    if file.filename == '':
        return jsonify({'error': '文件名为空'}), 400
    
    if file and allowed_file(file.filename):
        filename = secure_filename(file.filename)
        filepath = os.path.join(UPLOAD_FOLDER, filename)
        file.save(filepath)
        
        # 保存文件信息到数据库
        # file_record = File(
        #     filename=filename,
        #     filepath=filepath,
        #     size=os.path.getsize(filepath),
        #     user_id=current_user.id
        # )
        # db.session.add(file_record)
        # db.session.commit()
        
        return jsonify({
            'message': '文件上传成功',
            'filename': filename
        }), 200

3. 分片上传实现

对于大文件，实现分片上传可以提高上传稳定性和速度。

import hashlib

@app.route('/upload/chunk', methods=['POST'])
def upload_chunk():
    chunk = request.files['chunk']
    chunk_number = request.form.get('chunkNumber')
    total_chunks = request.form.get('totalChunks')
    file_id = request.form.get('fileId')
    
    # 创建临时目录存储分片
    chunk_dir = os.path.join(UPLOAD_FOLDER, 'chunks', file_id)
    os.makedirs(chunk_dir, exist_ok=True)
    
    # 保存分片
    chunk_path = os.path.join(chunk_dir, f'chunk_{chunk_number}')
    chunk.save(chunk_path)
    
    # 检查是否所有分片都已上传
    uploaded_chunks = len(os.listdir(chunk_dir))
    
    if uploaded_chunks == int(total_chunks):
        # 合并分片
        merge_chunks(file_id, total_chunks)
        return jsonify({'message': '文件上传完成'}), 200
    
    return jsonify({'message': f'分片 {chunk_number} 上传成功'}), 200

def merge_chunks(file_id, total_chunks):
    chunk_dir = os.path.join(UPLOAD_FOLDER, 'chunks', file_id)
    output_file = os.path.join(UPLOAD_FOLDER, f'{file_id}.file')
    
    with open(output_file, 'wb') as outfile:
        for i in range(int(total_chunks)):
            chunk_path = os.path.join(chunk_dir, f'chunk_{i}')
            with open(chunk_path, 'rb') as infile:
                outfile.write(infile.read())
    
    # 清理临时分片文件
    import shutil
    shutil.rmtree(chunk_dir)

4. 文件下载模块

实现文件下载和断点续传功能。

from flask import send_file, make_response

@app.route('/download/<file_id>', methods=['GET'])
def download_file(file_id):
    # 从数据库获取文件信息
    # file_record = File.query.get(file_id)
    # if not file_record:
    #     return jsonify({'error': '文件不存在'}), 404
    
    filepath = f'/path/to/file/{file_id}'
    
    return send_file(
        filepath,
        as_attachment=True,
        download_name='filename.ext'
    )

@app.route('/download/range/<file_id>', methods=['GET'])
def download_file_range(file_id):
    """支持断点续传的下载"""
    filepath = f'/path/to/file/{file_id}'
    file_size = os.path.getsize(filepath)
    
    range_header = request.headers.get('Range', None)
    
    if range_header:
        byte_range = range_header.replace('bytes=', '').split('-')
        start = int(byte_range[0])
        end = int(byte_range[1]) if byte_range[1] else file_size - 1
        
        with open(filepath, 'rb') as f:
            f.seek(start)
            data = f.read(end - start + 1)
        
        response = make_response(data)
        response.headers['Content-Range'] = f'bytes {start}-{end}/{file_size}'
        response.headers['Content-Length'] = end - start + 1
        response.status_code = 206
        
        return response
    
    return send_file(filepath)

5. 文件管理模块

实现文件列表、搜索、删除、重命名等功能。

@app.route('/files', methods=['GET'])
def list_files():
    # user_id = get_current_user_id()
    # files = File.query.filter_by(user_id=user_id).all()
    
    files_data = [
        {
            'id': 'file.id',
            'filename': 'file.filename',
            'size': 'file.size',
            'upload_time': 'file.upload_time',
            'type': 'file.file_type'
        }
        # for file in files
    ]
    
    return jsonify({'files': files_data}), 200

@app.route('/files/search', methods=['GET'])
def search_files():
    keyword = request.args.get('keyword', '')
    # files = File.query.filter(
    #     File.filename.like(f'%{keyword}%')
    # ).all()
    
    return jsonify({'files': []}), 200

@app.route('/files/<file_id>', methods=['DELETE'])
def delete_file(file_id):
    # file = File.query.get(file_id)
    # if file:
    #     os.remove(file.filepath)
    #     db.session.delete(file)
    #     db.session.commit()
    
    return jsonify({'message': '文件已删除'}), 200

@app.route('/files/<file_id>/rename', methods=['PUT'])
def rename_file(file_id):
    new_name = request.json.get('new_name')
    # file = File.query.get(file_id)
    # if file:
    #     file.filename = new_name
    #     db.session.commit()
    
    return jsonify({'message': '文件已重命名'}), 200

6. 文件共享模块

实现文件分享链接生成和权限控制。

import secrets
from datetime import datetime, timedelta

@app.route('/share/<file_id>', methods=['POST'])
def create_share_link(file_id):
    expire_days = request.json.get('expire_days', 7)
    password = request.json.get('password', None)
    
    # 生成唯一的分享码
    share_code = secrets.token_urlsafe(16)
    
    # 创建分享记录
    # share = FileShare(
    #     file_id=file_id,
    #     share_code=share_code,
    #     password=password,
    #     expire_time=datetime.now() + timedelta(days=expire_days)
    # )
    # db.session.add(share)
    # db.session.commit()
    
    share_url = f'http://your-domain.com/s/{share_code}'
    
    return jsonify({
        'share_url': share_url,
        'expire_time': 'expire_time'
    }), 200

@app.route('/s/<share_code>', methods=['GET'])
def access_shared_file(share_code):
    # share = FileShare.query.filter_by(share_code=share_code).first()
    # 
    # if not share or share.expire_time < datetime.now():
    #     return jsonify({'error': '分享链接已失效'}), 404
    # 
    # if share.password:
    #     # 需要密码验证
    #     password = request.args.get('password')
    #     if password != share.password:
    #         return jsonify({'error': '密码错误'}), 403
    
    # 返回文件信息或直接下载
    return jsonify({'file_info': {}}), 200

数据库设计

用户表 (users)

CREATE TABLE users (
    id SERIAL PRIMARY KEY,
    username VARCHAR(50) UNIQUE NOT NULL,
    email VARCHAR(100) UNIQUE NOT NULL,
    password VARCHAR(255) NOT NULL,
    storage_quota BIGINT DEFAULT 10737418240, -- 10GB
    used_storage BIGINT DEFAULT 0,
    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);

文件表 (files)

CREATE TABLE files (
    id SERIAL PRIMARY KEY,
    user_id INTEGER REFERENCES users(id),
    filename VARCHAR(255) NOT NULL,
    filepath VARCHAR(500) NOT NULL,
    file_size BIGINT NOT NULL,
    file_type VARCHAR(50),
    md5_hash VARCHAR(32),
    upload_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    is_deleted BOOLEAN DEFAULT FALSE
);

文件分享表 (file_shares)

CREATE TABLE file_shares (
    id SERIAL PRIMARY KEY,
    file_id INTEGER REFERENCES files(id),
    share_code VARCHAR(50) UNIQUE NOT NULL,
    password VARCHAR(50),
    expire_time TIMESTAMP,
    view_count INTEGER DEFAULT 0,
    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);

前端实现要点

文件上传组件示例 (Vue.js)

<template>
  <div class="upload-container">
    <input type="file" @change="handleFileSelect" multiple />
    <div v-for="file in uploadFiles" :key="file.name">
      <span>{{ file.name }}</span>
      <progress :value="file.progress" max="100"></progress>
      <span>{{ file.progress }}%</span>
    </div>
  </div>
</template>

<script>
export default {
  data() {
    return {
      uploadFiles: []
    }
  },
  methods: {
    handleFileSelect(event) {
      const files = Array.from(event.target.files)
      files.forEach(file => {
        this.uploadFile(file)
      })
    },
    
    async uploadFile(file) {
      const fileObj = {
        name: file.name,
        progress: 0
      }
      this.uploadFiles.push(fileObj)
      
      const formData = new FormData()
      formData.append('file', file)
      
      try {
        await axios.post('/upload', formData, {
          headers: {
            'Content-Type': 'multipart/form-data'
          },
          onUploadProgress: (progressEvent) => {
            fileObj.progress = Math.round(
              (progressEvent.loaded * 100) / progressEvent.total
            )
          }
        })
        
        this.$message.success('上传成功')
      } catch (error) {
        this.$message.error('上传失败')
      }
    }
  }
}
</script>

性能优化策略

1. 文件去重

使用MD5哈希值实现文件去重，节省存储空间。

import hashlib

def calculate_md5(filepath):
    hash_md5 = hashlib.md5()
    with open(filepath, "rb") as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hash_md5.update(chunk)
    return hash_md5.hexdigest()

@app.route('/upload/dedup', methods=['POST'])
def upload_with_dedup():
    file = request.files['file']
    
    # 计算文件MD5
    temp_path = '/tmp/' + secure_filename(file.filename)
    file.save(temp_path)
    md5_hash = calculate_md5(temp_path)
    
    # 检查是否已存在相同文件
    # existing_file = File.query.filter_by(md5_hash=md5_hash).first()
    # if existing_file:
    #     # 创建引用而不是重复存储
    #     os.remove(temp_path)
    #     return jsonify({'message': '文件已存在', 'file_id': existing_file.id})
    
    # 移动文件到正式存储位置
    # final_path = os.path.join(UPLOAD_FOLDER, md5_hash)
    # os.rename(temp_path, final_path)
    
    return jsonify({'message': '上传成功'}), 200

2. 缓存策略

使用Redis缓存文件元数据和热门文件。

import redis
import json

redis_client = redis.Redis(host='localhost', port=6379, db=0)

def get_file_metadata(file_id):
    # 先从缓存获取
    cache_key = f'file:{file_id}'
    cached_data = redis_client.get(cache_key)
    
    if cached_data:
        return json.loads(cached_data)
    
    # 缓存未命中，从数据库获取
    # file = File.query.get(file_id)
    # file_data = {
    #     'id': file.id,
    #     'filename': file.filename,
    #     'size': file.size
    # }
    
    # 存入缓存，设置过期时间
    # redis_client.setex(cache_key, 3600, json.dumps(file_data))
    
    return {}

3. 异步任务处理

使用Celery处理耗时操作。

from celery import Celery

celery = Celery('tasks', broker='redis://localhost:6379/0')

@celery.task
def compress_file(file_id):
    """异步压缩文件"""
    # file = File.query.get(file_id)
    # 执行压缩操作
    pass

@celery.task
def generate_thumbnail(file_id):
    """异步生成缩略图"""
    # 为图片文件生成缩略图
    pass

安全性考虑

1. 文件类型验证

import magic

def validate_file_type(filepath):
    mime = magic.from_file(filepath, mime=True)
    allowed_mimes = [
        'image/jpeg', 'image/png', 'application/pdf',
        'text/plain', 'application/zip'
    ]
    return mime in allowed_mimes

2. 访问权限控制

from functools import wraps
from flask_jwt_extended import get_jwt_identity

def check_file_permission(f):
    @wraps(f)
    def decorated_function(*args, **kwargs):
        file_id = kwargs.get('file_id')
        current_user_id = get_jwt_identity()
        
        # file = File.query.get(file_id)
        # if file.user_id != current_user_id:
        #     return jsonify({'error': '无权访问'}), 403
        
        return f(*args, **kwargs)
    return decorated_function

3. 存储配额管理

def check_storage_quota(user_id, file_size):
    # user = User.query.get(user_id)
    # if user.used_storage + file_size > user.storage_quota:
    #     return False
    return True

部署建议

Docker容器化部署

FROM python:3.9-slim

WORKDIR /app

COPY requirements.txt .
RUN pip install -r requirements.txt

COPY . .

EXPOSE 5000

CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5000", "app:app"]

Nginx配置示例

server {
    listen 80;
    server_name your-domain.com;
    
    client_max_body_size 1024M;
    
    location / {
        proxy_pass http://localhost:5000;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
    }
    
    location /uploads/ {
        alias /path/to/uploads/;
        expires 30d;
    }
}