2017.09.22 从hdfs 目录递归读取文件

做poorstudentstag项目。3个mr，要从另外两个mr输出文件中递归

@Override
protected void setup(Context context) throws IOException, InterruptedException {

    FSDataInputStream fsr = null;
    StringBuffer buffer = new StringBuffer();
    BufferedReader bufferedReader = null;

    Configuration conf = context.getConfiguration();
    String threshold_output_path = conf.get(Config.threshold_output_path);

    String path = conf.get(Config.DataDimensionList_output_path);


    try {

        FileSystem fs = FileSystem.get(conf);

        FileStatus[] status = fs.listStatus(new Path(threshold_output_path));

        for (FileStatus file : status){
            if (!file.getPath().getName().startsWith("part-")){
                continue;
            }

            fsr = fs.open(file.getPath());
            bufferedReader = new BufferedReader(new InputStreamReader(fsr));

            while ((lineTxt = bufferedReader.readLine()) != null){

                if (lineTxt.split("\t")[0].equals(Config.consumptionState)) {

                consumptionState_twentypercentvalue = Integer.parseInt(lineTxt.split("\t")[1]);
                consumptionState_fiftypercentvalue = Integer.parseInt(lineTxt.split("\t")[2]);
            } else if (lineTxt.split("\t")[0].equals(Config.shopState)) {
                shopState_twentypercentvalue = Integer.parseInt(lineTxt.split("\t")[1]);
                shopState_fiftypercentvalue = Integer.parseInt(lineTxt.split("\t")[2]);
            } else if (lineTxt.split("\t")[0].equals(Config.orderMealAPPState)) {
                orderMealAPPState_twentypercentvalue = Integer.parseInt(lineTxt.split("\t")[1]);
                orderMealAPPState_fiftypercentvalue = Integer.parseInt(lineTxt.split("\t")[2]);
            } else if (lineTxt.split("\t")[0].equals(Config.consumptionAPPState)) {
                consumptionAPPState_twentypercentvalue = Integer.parseInt(lineTxt.split("\t")[1]);
                consumptionAPPState_fiftypercentvalue = Integer.parseInt(lineTxt.split("\t")[2]);
            }

        }


        FileSystem fs1 = FileSystem.get(conf);

            FileStatus[] status1 = fs1.listStatus(new Path(path));

            for (FileStatus file1 : status1){
                if (!file.getPath().getName().startsWith("part-")){
                    continue;
                }

                fsr = fs1.open(file1.getPath());
                bufferedReader = new BufferedReader(new InputStreamReader(fsr));
                while ((lineTxt = bufferedReader.readLine()) != null) {
                    if (lineTxt.split("\t")[0].equals(Config.consumptionState)) {
                        consumptionStateSource = lineTxt.split("\t")[1];
                    } else if (lineTxt.split("\t")[0].equals(Config.shopState)) {
                        shopStateSource = lineTxt.split("\t")[1];

                    } else if (lineTxt.split("\t")[0].equals(Config.orderMealAPPState)) {
                        orderMealAPPStateSource = lineTxt.split("\t")[1];

                    } else if (lineTxt.split("\t")[0].equals(Config.consumptionAPPState)) {
                        consumptionAPPStateSource = lineTxt.split("\t")[1];
                    }
                }
            }
        }

    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        if (bufferedReader != null) {
            try {
                bufferedReader.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }