LazyMapReduce

最新推荐文章于 2025-09-13 12:54:57 发布

转载最新推荐文章于 2025-09-13 12:54:57 发布 · 174 阅读

0 ·

CC 4.0 BY-SA版权

原文链接：http://www.cnblogs.com/goongqk/archive/2013/06/06/lazuMapReduce.html

文章标签：

#大数据

本文对比了两种LazyMapReduce的实现方式，一种是通过直接创建Job实例并配置输入输出路径，另一种则是通过继承Configured类并实现Tool接口来完成。这两种方法都实现了基本的MapReduce流程，并详细展示了具体的代码实现。

两种LazyMapReduce实现方法对比：

 1 import org.apache.hadoop.conf.Configuration;
 2 import org.apache.hadoop.util.GenericOptionsParser;
 3 import org.apache.hadoop.mapreduce.Job;
 4 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 5 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 6 import org.apache.hadoop.fs.Path;
 7 public class LazyMR {
 8     public static void main(String[] args) throws Exception{
 9         Configuration conf =new Configuration();
10         
11         String[] otherArgs = new GenericOptionsParser(conf,args).getRemainingArgs();
12         if(otherArgs.length!=2){
13             System.err.println("Usage:<in><out>");
14             System.exit(2);
15         }
16         Job job = new Job(conf,"LazyMR");
17         
18         FileInputFormat.addInputPath(job, new Path(args[0]));
19         FileOutputFormat.setOutputPath(job, new Path(args[1]));
20         
21         System.exit(job.waitForCompletion(true)? 0:1);
22             
23     }
24 
25 }

 1 import org.apache.hadoop.conf.Configuration;
 2 import org.apache.hadoop.util.ToolRunner;
 3 import org.apache.hadoop.mapreduce.Job;
 4 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 5 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 6 import org.apache.hadoop.fs.Path;
 7 import org.apache.hadoop.util.Tool;
 8 import org.apache.hadoop.conf.Configured;
 9 
10 public class MyJob extends Configured implements Tool{
11 
12     @Override
13     public int run (String[] args) throws Exception{
14         Configuration conf = getConf();
15         
16         Job job = new Job(conf,"MyJob");
17         job.setJarByClass(MyJob.class);
18         
19         FileInputFormat.addInputPath(job, new Path(args[0]));
20         FileOutputFormat.setOutputPath(job, new Path(args[1]));
21         
22         System.exit(job.waitForCompletion(true)? 0:1);
23     
24         return 0;
25     }
26     
27     public static void main(String[] args) throws Exception{
28         int res = ToolRunner.run(new Configuration(), new MyJob(), args);
29         
30         System.exit(res);
31     }
32     
33 }