Hadoop任务配置与执行-优快云博客

本文链接：https://blog.youkuaiyun.com/csdncjh/article/details/52446721

本文详细介绍了一个Hadoop任务的具体配置过程及执行方法，包括如何设置队列、选择输入输出路径、指定Map和Reduce类以及优化配置等。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

---------------------------------------------------job

public class Count1Task extends Configuration implements Tool{  
    private Configuration conf=new Configuration();  
    @Override  
    public void setConf(Configuration conf) {  
          
    }  
  
    @Override  
    public Configuration getConf() {  
        return conf;  
    }  
  
    @Override  
    public int run(String[] args) throws Exception {  
          
        //job 参数  
        Configuration conf=new Configuration();  
        conf.set("mapreduce.job.queuename", "vendor.ven3"); //设置队列  
        FileSystem fs=FileSystem.get(conf);  
        Job job=Job.getInstance(conf);  
          
        //入口  
        job.setJarByClass(Count1Task.class);  
      
        //map  
        job.setMapperClass(CountMapper.class);  
      
        //reduce  
        job.setReducerClass(CountReducer.class);  
        job.setNumReduceTasks(10);  
          
        //输入 输出路径  
        Path in=new Path(args[0]);  
        Path out=new Path("/user/vendorwins/chenjinghui/domainCount2/count"); 
	SequenceFileInputFormat.setInputPaths(job,in); 		
        FileOutputFormat.setOutputPath(job, out);  
        //输入 输出格式  
        job.setInputFormatClass(SequenceFileInputFormat.class);  
        job.setOutputFormatClass(SequenceFileOutputFormat.class);  
        job.setOutputKeyClass(Text.class);  
        job.setOutputValueClass(LongWritable.class);  
          
        //保护代码   
        if(fs.exists(out)){  
            fs.delete(out, true);  
        }  
          
        //优化配置  
//      job.setCombinerClass(CountReducer.class);  
//      job.setPartitionerClass(MyPartition.class);  
          
        //执行  
        int flag=  job.waitForCompletion(true)?0:1;  
          
        return flag;  
    }  
}

----------------------------------------------执行

int res = ToolRunner.run(new Count1Task(), args);

mr的Demo