mapreduce代码整理之sort

本编文章主要运用mapreduce中的机制进行排序

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.lib.HashPartitioner;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Partitioner;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
 
public class Sort {
 
    //map将输入中的value化成IntWritable类型，作为输出的key
    public static class Map extends Mapper<Object,Text,IntWritable,IntWritable>{
        private static IntWritable data=new IntWritable();
       
        //实现map函数
        public void map(Object key,Text value,Context context)
                throws IOException,InterruptedException{
            String line=value.toString();
            data.set(Integer.parseInt(line));
            context.write(data, new IntWritable(1));
        }
       
    }
   
    //reduce将输入中的key复制到输出数据的key上，
    //然后根据输入的value-list中元素的个数决定key的输出次数
    //用全局linenum来代表key的位次
    public static class Reduce extends
            Reducer<IntWritable,IntWritable,IntWritable,IntWritable>{
       
        private static IntWritable linenum = new IntWritable(1);
       
        //实现reduce函数
        public void reduce(IntWritable key,Iterable<IntWritable> values,Context context)
                throws IOException,InterruptedException{
            for(IntWritable val:values){
                context.write(linenum, key);
                linenum = new IntWritable(linenum.get()+1);
            }
        }
    }
  public static class MyPartition extends Partitioner<IntWritable,IntWritable>{
	   @Override
	   public int getPartition(IntWritable key,IntWritable value,int numPartitions){
		   if(key.get()>=1&&key.get()<=11){
			   return 1;
		   }
		   if(key.get()>=12&&key.get()<=21){
			   return 2;
		   }
		   if(key.get()>=22&&key.get()<=31){
			   return 3;
		   }
		   if(key.get()>=32&&key.get()<=41){
			   return 4;
		   }
		   if(key.get()>=42&&key.get()<=51){
			   return 5;
		   }
		   if(key.get()>=52&&key.get()<=61){
			   return 6;
		   }
		   return 0;
		/*System.out.print(numPartitions);
		int MaxNumber = 100;
		int bound = MaxNumber / numPartitions +1;
		int keynumber = key.get();
		for(int i = 0;i<numPartitions;i++){
			if(keynumber<bound*i&&keynumber>=bound*(i-1))
				return  i-1;
		}
		return 0;*/
	  }
   }
    public static void main(String[] args) throws Exception{
        Configuration conf = new Configuration();
        //conf.setBoolean("mapred.compress.map.output",true);
      //conf.set("mapred.compress.map.output", "true");//代码实现map输出压缩减少网络传输压力
      //conf.set("mapred.map.output.compression.codec","org.apache.hadoop.io.compress.DefaultCodec");
     Job job = new Job(conf, "Data Sort");
     job.setJarByClass(Sort.class);
     
     //设置Map和Reduce处理类
     job.setMapperClass(Map.class);
     job.setReducerClass(Reduce.class);
    // job.setPartitionerClass(MyPartition.class);
    // job.setNumReduceTasks(10);
     //设置输出类型
     job.setOutputKeyClass(IntWritable.class);
     job.setOutputValueClass(IntWritable.class);
     
     //设置输入和输出目录
     FileInputFormat.addInputPath(job, new Path(args[0]));
     FileOutputFormat.setOutputPath(job, new Path(args[1]));
   //FileOutputFormat.setCompressOutput(job,true);//代码实现输出压缩
  //FileOutputFormat.setOutputCompressorClass(job, GzipCodec.class);
     System.exit(job.waitForCompletion(true) ? 0 : 1);
     }
}