MapReduce Job Control

最新推荐文章于 2021-01-17 06:05:59 发布

caiandyong

最新推荐文章于 2021-01-17 06:05:59 发布

阅读量887

点赞数

分类专栏： hadoop 文章标签： MapReduce Job Contro JobControl

本文链接：https://blog.youkuaiyun.com/caiandyong/article/details/45130515

版权

hadoop 专栏收录该内容

33 篇文章

订阅专栏

本文介绍了一个基于Hadoop MapReduce的作业流程示例，该流程包括三个阶段的MapReduce任务：数据标准化、属性数量统计及属性最小值计算。通过使用JobControl实现了作业之间的依赖关系，确保前一阶段完成后下一阶段才开始执行。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

package org.cy.pack1;

import java.net.URI;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob;
import org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.cy.pack1.MR1_Normalization.Map_Normalization;
import org.cy.pack1.MR1_Normalization.Reduce_Normalization;
import org.cy.pack1.MR2_Attnum.MyMap_Att;
import org.cy.pack1.MR2_Attnum.MyReduce_Att;
import org.cy.pack1.MR3_Attmin.MyMap_atm;
import org.cy.pack1.MR3_Attmin.MyReduce_atm;

public class Main extends PublicInfo{


   public static void main(String[] args) throws Exception {

       //MR1
       Configuration conf = new Configuration();
       FileSystem fs = FileSystem.get(new URI(input.toString()), conf);

        if (fs.exists(outMR1)) {
            fs.delete(outMR1, true);
            logger.info("outMR1文件夹存在，已执行删除操作！");
          }

        //job1 config
       Job job1 = new Job(conf,"MR1_");
        ......

        //MR2

        if (fs.exists(outMR2)) {
            fs.delete(outMR2, true);
            logger.info("outMR2文件夹存在，已执行删除操作！");
          }

        //job config
       Job job2 = new Job(conf,"MR2_Att");
        ......


       //MR3

        if (fs.exists(outMR3)) {
            fs.delete(outMR3, true);
            logger.info("outMR3文件夹存在，已执行删除操作！");
          }

        //job3 config
       Job job3 = new Job(conf,"MR3");
        ......


       ControlledJob jobx = new ControlledJob(conf);
       ControlledJob joby = new ControlledJob(conf);
       ControlledJob jobz = new ControlledJob(conf);

       jobx.setJob(job1);
       joby.setJob(job2);
       jobz.setJob(job3);

       joby.addDependingJob(jobx);//joby   and jobz   depending jobx
       jobz.addDependingJob(jobx);
       JobControl jc = new JobControl("xyzJob");

       jc.addJob(jobx);
       jc.addJob(joby);
       jc.addJob(jobz);

       Thread th = new Thread(jc);
       th.start();

       while(true){
           if(jc.allFinished()){
               jc.stop();
               break;
           }
       }




       System.out.println("Ok!");

       System.exit(0);

   }

}