Hadoop-使用mapreduce对数字进行排序

最新推荐文章于 2025-04-16 10:15:30 发布

原创最新推荐文章于 2025-04-16 10:15:30 发布

· 2.1k 阅读

4 ·

版权

我写了三个类

创建一个Mapper类

import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class PaixuMapper extends Mapper<LongWritable,Text,IntWritable,NullWritable>{
	IntWritable key2=new IntWritable();
	NullWritable value2=NullWritable.get();
	@Override
	protected void map(LongWritable key, Text value, Context context)
			throws IOException, InterruptedException {
		
		key2.set(Integer.parseInt(value.toString()));
		context.write(key2,value2);
	}

}

创建一个Reducer类

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class PaixuReducer extends Reducer<IntWritable,NullWritable,IntWritable,NullWritable>{
	@Override
	protected void reduce(IntWritable arg0, Iterable<NullWritable> arg1,
			Reducer<IntWritable, NullWritable, IntWritable, NullWritable>.Context arg2)
			throws IOException, InterruptedException {
		// TODO Auto-generated method stub
		super.reduce(arg0, arg1, arg2);
	}
}

写main方法

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class PaixuDriver {
	public static void main(String[] args) throws Exception {
		//加载配置文件
				Configuration conf = new Configuration();
				//创建mt任务
			    Job job = Job.getInstance(conf, "paixu");
			    job.setJarByClass(PaixuDriver.class);
			    //设置map
			    job.setMapperClass(PaixuMapper.class);
			    //设置输出格式
			    job.setOutputKeyClass(IntWritable.class);
			    job.setOutputValueClass(NullWritable.class);
			    //设置输入路径
			    FileInputFormat.addInputPath(job, new Path("d:/wc/suminput"));
			    //设置输出路径
			    FileOutputFormat.setOutputPath(job, new Path("d:/wc/paixuoutput"));
			    //等待程序执行完毕退出
			    System.exit(job.waitForCompletion(true) ? 0 : 1);
	}
}

d:/wc/suminput这个目录下放了一个txt文件，里面是要排序的数字。