Ubuntu 22.04 下 CURL（C++）文件分块上传/分块下载源码

原创已于 2024-08-16 13:39:07 修改 · 802 阅读

6 ·

CC 4.0 BY-SA版权

文章标签：

#ubuntu #curl #c++ #文件上传下载

于 2024-06-12 08:24:41 首次发布

C++ 同时被 2 个专栏收录

20 篇文章

订阅专栏

编程语言

4 篇文章

订阅专栏

该博客介绍了如何在Ubuntu 22.04上使用C++和CURL库实现文件的分块上传功能。通过模拟Python requests库的分块上传过程，展示C++代码实现从读取文件、设置HTTP头部到使用CURL进行POST请求的完整步骤。

为了帮助大家理解代码，首先介绍文件上传/下载流程：

上传文件流程说明：首先向服务器 restful api 接口 /common发送 Post 请求，创建一个新的项目，服务器返回 project guid。创建成功后，读取本地视频文件，按照给定 chunk_size（例如 10240 byte），不断循环向服务器 restful api 接口 /upload_chunk 发送数据，直到文件传输完成。

视频上传后，在服务器端做 3D 建模，耗时1-2小时，故拆分步骤，首先向服务器发送请求生成 project, “刷新”按钮不停获取服务器状态。这些步骤不是必须，使用者按照自己项目需求取舍。发送请求的 header, body_json 等格式，同样按照自己的项目需求设计即可。

创建 project 的接口说明如下：

url	http://192.168.1.0:8000/common
type	POST
param	{ "model_type":1, "project_type":0, "object":"plants" }
return	{ "data": "85c9c7cc-18d6-11ef-818a-7d68b3e42070", "errcode": 0 }

下载文件：同样采取分块下载。

CMakeList.txt:

# 查找libcurl组件
find_package(CURL REQUIRED)

include_directories(
include
${PYTHON_INCLUDE_DIRS}
${CURL_INCLUDE_DIRS}
)

target_link_libraries(${PROJECT_NAME} PRIVATE "${CURL_LIBRARY}")
target_include_directories(${PROJECT_NAME} PRIVATE "${CURL_INCLUDE_DIR}")

upload调用代码：

void test_restapi_upload_chunk(){
    std::string project_id = "bee8400e-1f13-11ef-818a-7d68b3e42070";
    std::string file_path = "/home/coco/Documents/IMG_02391.MOV";
    std::string url = "http://192.168.1.1:8100/upload_chunk";

    try {
        upload_chunk(url, project_id, file_path);
        std::cout << "Chunk upload completed successfully." << std::endl;
    } catch (const std::exception& e) {
        std::cerr << "An error occurred: " << e.what() << std::endl;
    }
}

upload_chunk() 函数实现代码：

void upload_chunk(const std::string& url, const std::string& project_id, const std::string& file_path, size_t chunkSize) {
    // curl初始化
    CURL* curl = curl_easy_init();
    if (!curl) {
        std::cerr << "CURL initialization failed." << std::endl;
        return;
    }

    // 初始化表单和头部
    curl_httppost* formpost = NULL;
    curl_httppost* lastptr = NULL;
    struct curl_slist* headers = NULL;

    // 设置 URL 和 HTTP POST 选项
    std::cout << "url.c_str(): " + url << std::endl;
    curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
    curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);

    // 打开文件和读取逻辑
    std::ifstream file(file_path, std::ios::binary);
    if (!file.is_open()) {
        std::cerr << "Failed to open file: " << file_path << std::endl;
        return;
    }
    std::filesystem::path path(file_path);
    std::string file_name = path.filename();

    size_t totalSize = std::filesystem::file_size(file_path);
    size_t offset = 0;
    size_t start = 0;
    std::cout << "totalSize: " + std::to_string(totalSize) << std::endl;

    // 循环读取文件并发送
    std::vector<char> buffer(chunkSize);
    while (!file.eof()) {
        // 读取文件内容到buffer
        file.read(buffer.data(), chunkSize);
        size_t bytesRead = file.gcount();

        if (bytesRead == 0) {
            break; // 没有更多数据可读，退出循环
        }

        // 构建project表单数据
        CURLFORMcode form_result = curl_formadd(
            &formpost,
            &lastptr,
            CURLFORM_COPYNAME, "project", // 表单字段名称
            CURLFORM_COPYCONTENTS, project_id.c_str(), // 表单字段值
            CURLFORM_END);

        // 为每个文件块创建一个新的表单
        form_result = curl_formadd(
            &formpost,
            &lastptr,
            CURLFORM_COPYNAME, "file", // 表单字段名称
            CURLFORM_BUFFERPTR, buffer.data(), // 缓冲区指针
            CURLFORM_BUFFERLENGTH, static_cast<long>(bytesRead), // 缓冲区长度
            CURLFORM_FILENAME, file_name.c_str(), // 文件名
            CURLFORM_CONTENTTYPE, "application/octet-stream", // 内容类型
            CURLFORM_END);

        if (form_result != CURL_FORMADD_OK) {
            std::cerr << "Error building file form." << std::endl;
            break;
        }

        // 设置请求头和POST数据
        std::string contentRange = std::to_string(start) + "-" + std::to_string(start + bytesRead - 1) + "/" + std::to_string(totalSize);
        std::cout << "contentRange: " + contentRange << std::endl;
        headers = curl_slist_append(headers, ("Content-Range: " + contentRange).c_str());
        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
        curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);

        // 设置回调函数和数据
        std::string responseContent;
        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, UploadWriteCallback);
        curl_easy_setopt(curl, CURLOPT_WRITEDATA, &responseContent);

        // 执行请求
        CURLcode res = curl_easy_perform(curl);
        if (res != CURLE_OK) {
            std::cerr << "CURL error: " << curl_easy_strerror(res) << std::endl;
            break;
        }

        // 更新下一个块的起始位置
        start += bytesRead;

        // 处理响应数据
        if (!responseContent.empty()) {
            std::cout << responseContent << std::endl;
        }

        // 清理headers和表单
        curl_slist_free_all(headers);
        headers = nullptr;
        curl_formfree(formpost);
        formpost = NULL;
        lastptr = NULL;
    }

    // 清理资源
    curl_formfree(formpost);
    curl_easy_cleanup(curl);
    file.close();
}

download 调用代码：

void test_restapi_download_chunk(){
    char cwd[PATH_MAX];
    if (getcwd(cwd, sizeof(cwd)) != NULL) {
        std::cout << "getcwd(): " << cwd << std::endl;
    } else {
        perror("getcwd() error");
    }

    QString currentPath = QDir::currentPath();
    std::cout << "Current working directory: " << currentPath.toStdString() << std::endl;

    std::string url = "http://192.168.1.87:8200/download_chunk"; // 替换为实际的Flask服务器URL
    std::string project = "85c9c7cc-18d6-11ef-818a-7d68b3e42070"; // 替换为实际项目名称
    std::string output_path = "/home/coco/Downloads/point_cloud.ply"; // 替换为实际保存文件的路径
    size_t start = 0; // 替换为实际起始字节
    size_t end = 1024; // 替换为实际结束字节，这里假设DOWNLOAD_RANGE的值为102400

    try {
        if (FileSystemUtils::exists(output_path)) {
            std::cout << output_path + "already exists." << std::endl;
            return ;
        }
        download_chunk(url, project, output_path, start, end);
        std::cout << "Chunk download completed successfully." << std::endl;
    } catch (const std::exception& e) {
        std::cerr << "An error occurred: " << e.what() << std::endl;
    }
}

download_chunk()函数代码实现：

int download_chunk(const std::string& url, const std::string& project_id, const std::string& output_path, size_t start, size_t end=10240) {
    // 创建 curl 句柄
    CURL *curl = curl_easy_init();
    if (!curl) {
        std::cerr << "CURL initialization failed." << std::endl;
        return 1;
    }

    // 保存文件，确保以追加模式打开
    std::ofstream outfile(output_path, std::ios::binary | std::ios::app);
    if (!outfile.is_open()) {
        std::cerr << "Cannot open file: " << output_path << std::endl;
        return 1;
    }

    // 设置请求的URL
    curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
    curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "GET");
    //curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    //curl_easy_setopt(curl, CURLOPT_HTTPHEADER, nullptr);

    // 设置请求body: project_id
    std::string json_data = nlohmann::json{{"project", project_id}}.dump();
    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, json_data.c_str());

    CURLcode res;
    size_t total_size = -1;
    size_t content_length;
    bool is_total_size_calculated = false;
    std::string response_content;

    std::string response_headers;
    std::string range_header = "Range: " + std::to_string(start) + "-" + std::to_string(end);
    std::cout << "first range_header: " << range_header << std::endl << std::endl;

    int i = 1;
    struct curl_slist *headers = nullptr;
    try {
        do {
            std::cout << "第" << std::to_string(i)  <<"次循环：" << std::endl;
            i = i + 1;

            // 设置请求头, 包括Range和Content-Type
            headers = curl_slist_append(headers, "Content-Type: application/json");
            headers = curl_slist_append(headers, range_header.c_str());
            curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);

            // 设置响应头回调函数
            curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, HeaderCallback);
            curl_easy_setopt(curl, CURLOPT_HEADERDATA, &response_headers);


            // 设置写入回调函数
            curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, DownloadWriteCallback);
            curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_content);

            // 执行请求
            res = curl_easy_perform(curl);

            if (res != CURLE_OK) {
                std::cerr << "Download error: " << curl_easy_strerror(res) << std::endl;
                curl_slist_free_all(headers);
                curl_easy_cleanup(curl);
                //curl_global_cleanup();

                if (outfile.is_open()) {
                    outfile.close(); // 关闭文件
                }

                return 1;
            } else {
                // 响应头存储在response_header中,解析response_header来获取需要的信息
                std::cout << "Response headers: "<< std::endl << response_headers << std::endl;

                // 解析响应头, Content-Range, Content-Length
                std::istringstream header_stream(response_headers);
                std::string header_line;
                while (getline(header_stream, header_line)) {
                    if (header_line.find("Content-Range:") == 0) {
                        std::cout << "header_line: " << header_line << std::endl;
                        if (!is_total_size_calculated ) {
                            size_t pos = header_line.find('/');
                            std::string total_size_str = header_line.substr(pos + 1);
                            total_size = std::stoull(total_size_str);
                            std::cout << "total_size: " << total_size << std::endl;

                            is_total_size_calculated = true;
                        }
                    } else if (header_line.find("Content-Length:") == 0) {
                        size_t pos = header_line.find(':');
                        content_length = std::stoul(header_line.substr(pos + 2));
                        std::cout << "content_length: " << content_length << std::endl;
                        // 更新下一次请求的范围
                        start += content_length;
                        end = std::min(end + content_length, total_size);

                        // 更新range_header
                        range_header = "Range: bytes=" + std::to_string(start) + "-" + std::to_string(end);
                        std::cout << "request range_header: " << range_header << std::endl;
                        std::cout << "start: " << start << std::endl;
                        std::cout << "end: " << end << std::endl << std::endl;
                    }
                }

                // 将 responseContent 写入到文件
                outfile.write(response_content.c_str(), response_content.length());

                curl_slist_free_all(headers);
                headers = nullptr;
                response_headers.clear();
                response_content.clear();
            }
        } while (start < total_size);
    }catch (const std::bad_alloc& e) {
        std::cerr << "Memory allocation failed: " << e.what() << std::endl;
        if (headers) {
            curl_slist_free_all(headers);
        }
        curl_easy_cleanup(curl);
        if (outfile.is_open()) {
            outfile.close();
        }
        return 1;
    }

    curl_easy_cleanup(curl);
    if (outfile.is_open()) {
        outfile.close(); // 关闭文件
    }

    return 0;
}