java大文件读取拆分（30G以上）

java大文件读取拆分，本次示例解析拆分40G大小的SQL文件，解析过程50万行数据一个文件，具体逻辑直接上代码 test1 是单文件读取过滤，test2 是文件拆分

代码可以直接运行，只需修改：fileName(需要操作的目标文件路径)，outDir(拆分后文件输出目录)，outFile（单文件过滤后的输出路径）

代码如下：

package org.example;

import org.junit.jupiter.api.Test;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;

/**
 * @description: java读取拆分大文件
 * @author: w*****
 * @date: 2021/4/8 18:51
 */
public class FileSplit {
    private static String fileName = "D://备份//历史待恢复数据//sql//2.sql";
    private static String outFile = "D://备份//历史待恢复数据//sql//sql//3.sql";
    private static String outDir = "D://备份//历史待恢复数据//sql//2sql//";


    /**
     * 2G以上大文件读取
     *
     * @param inputFile, outputFile
     * @return void
     * @author wang****
     * @date: 2021/4/7 16:16
     */
    void largeFileIO(String inputFile, String outputFile) {
        try {
            BufferedInputStream bis = new BufferedInputStream(new FileInputStream(new File(inputFile)));
            // 10M缓存
            BufferedReader in = new BufferedReader(new InputStreamReader(bis, "utf-8"), 10 * 1024 * 1024);
            FileWriter fw = new FileWriter(outputFile);
            while (in.ready()) {
                String line = in.readLine();
                if (line.startsWith("INSERT")) {
                    fw.append(line + "\r");
                }
            }
            in.close();
            fw.flush();
            fw.close();
        } catch (IOException ex) {
            ex.printStackTrace();
        }
    }


    /**
     * 大文件拆分
     *
     * @param inputFile, outputFile
     * @return void
     * @author wang****
     * @date: 2021/4/8 16:19
     */
    void largeFileSplit(String inputFile, String outputDir) {
        int j = 0;
        try {
            File out = new File(outputDir);
            if (!out.exists()) {
                out.mkdirs();
            }
            BufferedInputStream bis = new BufferedInputStream(new FileInputStream(new File(inputFile)));
            // 20M缓存
            BufferedReader in = new BufferedReader(new InputStreamReader(bis, "utf-8"), 20 * 1024 * 1024);
            int i = 0;
            while (in.ready()) {
                long startTime = System.currentTimeMillis();
                ++i;
                FileWriter fw = new FileWriter(outputDir + i + ".sql");
                String line = null;
                //50万行一个文件拆分
                for (long lineCounter = 0; lineCounter < 500000 && (line = in.readLine()) != null; ++lineCounter) {
                    if (line.startsWith("INSERT INTO `cur_rec` VALUES")) {
                        fw.append(line + "\r");
                        ++j;
                    }
                }
                fw.flush();
                fw.close();
                long endTime = System.currentTimeMillis();
                System.out.println("第" + i + "个文件拆分成功，耗时：" + (float) (endTime - startTime) / 1000 + "秒");
            }
            in.close();
        } catch (IOException ex) {
            System.out.println("第" + j + "行数据拆分异常\r" + ex);
            ex.printStackTrace();
        }
    }

    /**
     * 读取测试
     */
    @Test
    public void test1() {
        largeFileIO(fileName, outFile);
    }

    /**
     * 大文件拆分
     */
    @Test
    public void test2() {
        long startTime = System.currentTimeMillis();
        largeFileSplit(fileName, outDir);
        long endTime = System.currentTimeMillis();
        System.out.println("拆文件成功，共耗时：" + (float) (endTime - startTime) / 1000 + "秒");
    }
}