mapreduce编程模型之WritableComparator

WritableComparator是一个类 这个类是用于mapreduce编程模型中的比较 排序 

mapreduce中有两次排序 一次是 在环形缓冲区域之中进行分区 排序

还有一次是数据在reduce端获取文件之后进行分组

现在我讲的是后面那个

 //Define the comparator that controls which keys are grouped together or a single call to Reducer#reduce
job.setGroupingComparatorClass(MyComparator.class);

上面是我们在定义job时候进行的配置 配置如何进行分组

setGroupingComparatorClass内部的参数是RawComparator

而WritableComparator是实现RawComparator

所以我们直接继承WritableComparator类就可以自己定义一个MyComparator

public static class MyComparator extends WritableComparator {
        public MyComparator() {
            super(Text.class,true);
        }

        @Override
        public int compare(WritableComparable a, WritableComparable b) {
            Text a1 = (Text) a;
            Text b1 = (Text) b;
            if (a1.toString().equals("hello") && b1.toString().equals("hello")) {
                return -1;
            } else {
                return 0;
            }

        }
    }

上面这段代码我必须说一个坑 坑了我好几个小时 最后在statckvoerflow网站上才找到的提示

就是那个无参构造子 必须调用父类的构造子 不然会报空指针 未初始化 buffer

通过查找源码 也确实发现了这个问题

  protected WritableComparator(Class<? extends WritableComparable> keyClass,
                               Configuration conf,
                               boolean createInstances) {
    this.keyClass = keyClass;
    this.conf = (conf != null) ? conf : new Configuration();
    if (createInstances) {
      key1 = newKey();
      key2 = newKey();
      buffer = new DataInputBuffer();
    } else {
      key1 = key2 = null;
      buffer = null;
    }
  }

因为从报错的空指针来说 是buffer为空 整个类也就只有这人对buffer进行了初始化

最后来看一下结果

hdfs@yksp005206:/home/jumpserver$ hadoop fs -cat /test/wc/output/part-r-00000
hello value[] hello,
hello value[] hello,
world value[] hello,hellp,hive,kylin,spark,world,

从结果上来看 确实是hello没有被分到一个reduce中 而其他的所有单词都被分配到了同一个reducer中

package com.hit.ee;

/**
 * Created by zh on 2017/9/28.
 */

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Partitioner;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;
import java.util.StringTokenizer;

public class WorldCount2 {

    public static class TokenizerMapper
            extends Mapper<Object, Text, Text, Text> {

        private Text word = new Text();

        public void map(Object key, Text value, Context context
        ) throws IOException, InterruptedException {
            StringTokenizer itr = new StringTokenizer(value.toString());
            while (itr.hasMoreTokens()) {
                word.set(itr.nextToken());
                context.write(word, word);
            }
        }
    }

    public static class IntSumReducer
            extends Reducer<Text, Text, Text, Text> {

        public void reduce(Text key, Iterable<Text> values,
                           Context context
        ) throws IOException, InterruptedException {
            Text text = new Text();
            StringBuffer sb = new StringBuffer("value[] ");
            for (Text value : values) {
                sb.append(value).append(",");
            }
            text.set(sb.toString());
            context.write(key, text);
        }
    }

    public static class MyComparator2 implements RawComparator<Text>{

        DataInputBuffer buffer = new DataInputBuffer();

        @Override
        public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {

            try {
                Text a = new Text();
                buffer.reset(b1,s1,l1);
                a.readFields(buffer);

                Text b = new Text();
                buffer.reset(b2,s2,l2);
                b.readFields(buffer);
                return compare(a,b);

            } catch (IOException e) {
                e.printStackTrace();
            }

            return -1;
        }


        @Override
        public int compare(Text a1, Text b1) {
            if (a1.toString().equals("hello") && b1.toString().equals("hello")) {
                return -1;
            } else {
                return 0;
            }
        }
    }

    public static class MyComparator extends WritableComparator {
        public MyComparator() {
            super(Text.class,true);
        }

        @Override
        public int compare(WritableComparable a, WritableComparable b) {
            Text a1 = (Text) a;
            Text b1 = (Text) b;
            if (a1.toString().equals("hello") && b1.toString().equals("hello")) {
                return -1;
            } else {
                return 0;
            }

        }
    }

    public static class MyPartitioner extends Partitioner<Text,Text>{

        @Override
        public int getPartition(Text key, Text value, int numPartitions) {
            if (key.toString().equals("hello"))
                return 0;
            else
                return 1;
        }
    }

    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        FileSystem.get(conf).deleteOnExit(new Path(args[1]));

        Job job = Job.getInstance(conf, "word count");
        job.setJarByClass(WorldCount2.class);
        job.setMapperClass(TokenizerMapper.class);
        job.setReducerClass(IntSumReducer.class);

//        job.setNumReduceTasks(2);
//        job.setPartitionerClass(MyPartitioner.class);
        //Define the comparator that controls how the keys are sorted before they are passed to the reducer
        //job.setSortComparatorClass(MyComparator.class);
        //Define the comparator that controls which keys are grouped together or a single call to Reducer#reduce
        job.setGroupingComparatorClass(MyComparator.class);


        job.setMapOutputValueClass(Text.class);
        job.setMapOutputKeyClass(Text.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);

        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));


        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }
}