hadoop--mapreduce代码之排序

最新推荐文章于 2024-05-12 21:25:50 发布

原创最新推荐文章于 2024-05-12 21:25:50 发布 · 194 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#java

hadoop 专栏收录该内容

10 篇文章

订阅专栏

本文介绍了一个基于Hadoop的MapReduce示例程序，该程序用于演示如何通过MapReduce进行排序操作。具体来说，Map阶段将输入文本行转换为整数并设置为输出键，而Reduce阶段则根据输入值的数量确定键的输出次数。

package com.hadoop.sample;

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;

public class Sort {
	//map将输入中的value化成IntWritable类型，作为输出的key
	public static class Map extends Mapper<Object,Text,IntWritable,IntWritable>{
		private static IntWritable data = new IntWritable();
		public void map(Object key,Text value,Context context) throws IOException,InterruptedException{
			 String line = value.toString();
			 data.set(Integer.parseInt(line));
			 context.write(data, new IntWritable(1));
		}
	}
	//reduce将输入中的key复制到输出的value上，然后根据输入的
	//value-list中的元素的个数决定key的输出次数
	//用全局的linenum来代表key的位次
	public static class Reduce extends Reducer<IntWritable,IntWritable,IntWritable,IntWritable>{
		private static IntWritable linenum = new IntWritable(1);
		public void reduce(IntWritable key,Iterable<IntWritable> values,Context context) throws IOException,InterruptedException{
			for(IntWritable val:values){
				context.write(linenum, key);
				linenum = new IntWritable(linenum.get()+1);
			}

		}
	}
	/**
	 * @param args
	 */
	public static void main(String[] args) throws Exception{
		// TODO Auto-generated method stub
		Configuration conf = new Configuration();
		String[] otherArgs = new GenericOptionsParser(conf,args).getRemainingArgs();
		if(otherArgs.length != 2){
			System.err.println("Usage WordCount <int> <out>");
			System.exit(2);
		}
		Job job = new Job(conf,"Sort");
		job.setJarByClass(Sort.class);
		job.setMapperClass(Map.class);
		job.setCombinerClass(Reduce.class);
		job.setReducerClass(Reduce.class);
		job.setOutputKeyClass(IntWritable.class);
		job.setOutputValueClass(IntWritable.class);
		FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
		FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
		System.exit(job.waitForCompletion(true) ? 0 : 1);
	}

}