Java按行读取正在被动态写入的大文件实例--使用BufferedInputStream(2)

[like Sunday like rain]

1.适用场景

适用于正在被动态按行写入的大文件的读取和处理。

2.BufferedInputStream类使用原因

之前使用的RandomAccessFile类速度太慢，影响了整个流程的吞吐量。

使用了BufferedInputStream的skip方法，跳过已经读过的部分。

3.code

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.text.SimpleDateFormat;
import java.util.*;

/**
 * Created by maixiaohai on 16/7/4.
 */
public class FisFileTest {
    private static int SLEEP_TIME = 1000; // 1 seconds
    private static Scanner sc;
    private static BufferedInputStream fis;
    private static boolean isRun = true;
    private static boolean readStop = false;

    public static void main(String[] args) throws Exception{
        String logPath = args[0];
        long breakpoint = 0;
        if (args.length == 2) {
            breakpoint = Long.parseLong(args[1]);
        }
        if (!logPath.endsWith("/")) {
            System.out.println("path should end with /");
        } else {
            readFile(logPath, breakpoint);
        }
    }

    public static void readFile(String logPath, long breakpoint) {
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        SimpleDateFormat sdf2 = new SimpleDateFormat("yyyyMMdd");
        String filePath = logPath + "test.log";

        long startIndex = 0;
        boolean segmentFlag = true;
        boolean initialFlag = true;
        Runtime.getRuntime().addShutdownHook(new Thread() {
            @Override
            public void run() {
                isRun = false;
                System.out.println("Inside Add Shutdown Hook");
                while (!readStop) {
                    try {
                        sleep(1 * 1000);
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                }
            }
        });
        while (true) {
            fis = null;
            sc = null;
            try {
                File file = new File(filePath);
                try {
                    fis = new BufferedInputStream(new FileInputStream(file));
                } catch (Exception e) {
                    e.printStackTrace();
                    //切换时，旧文件得命名，新文件还没有创建，一秒钟后重试一次
                    System.out.println(sdf.format(new Date(System.currentTimeMillis()))
                            + " 该文件有问题，sleep 1 seconds");
                    Thread.sleep(1 * 1000);
                    continue;
                }

                long fileLength = file.length();
                System.out.println("fileLength " + fileLength);
                if ( initialFlag && startIndex == 0 ) {
                    System.out.println("==========first start fileLength :" + fileLength  + "=========" );
                    System.out.println("==========first start breakpoint :" + breakpoint  + "=========" );
                    startIndex = breakpoint;
                    initialFlag = false;
                } else if (startIndex > fileLength){
                    //0点文件切换时发生,此时需要把昨天的日志继续读完
                    if (segmentFlag) {
                        String date = sdf2.format(new Date().getTime()-24*60*60*1000);
                        String path = logPath + "test_" + date;
                        File file2 = new File(path);
                        fis = new BufferedInputStream(new FileInputStream(file2));
                        segmentFlag = false;
                    } else {
                        startIndex = 0;
                        segmentFlag = true;
                    }
                }

                <span style="color:#ff0000;">fis.skip(startIndex)</span>;
                sc = new Scanner(fis, "UTF-8").useDelimiter(System.getProperty("line.separator"));
                long startTime = System.currentTimeMillis();
                String line = null;

                int onceCount = 0;
                while ( isRun && sc.hasNext() ) {
                    line = sc.next();
                    onceCount++;
                    //记录读取位置
                    <span style="color:#ff0000;">startIndex += line.length() + 1;</span>
                    // do sth
                }

                long endTime = System.currentTimeMillis();
                System.out.println(sdf.format(new Date(System.currentTimeMillis())) +
                        " once read " + onceCount + " records");
                System.out.println("once read spend " + (endTime - startTime) + "ms");
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                try {
                    System.out.println("offset : " + startIndex);
                    fis.close();
                    // 全部处理完成后，readStop置位，1s后退出
                    if ( !isRun ) {
                        readStop = true;
                    }
                    Thread.sleep(SLEEP_TIME);
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
    }
}

4.需要注意的地方

 本例仅适用于按行写入的文件