十二、用MapReduce完成类似倒排索引的功能



1）理解【倒排索引】功能
2）熟悉 MapReduce 中的 Combiner 功能
3）依据需求编码实现【倒排索引】功能，旨在对 MapReduce理解。
数据：

结果：

代码：

package com.hyhc.mr;
 
import java.io.IOException;
 
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
importorg.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
 
 
publicclass InvertedIndexMapReduce extends Configured implements Tool{
//url:key1->10
//url:key2->12
//url1:key2->3
    publicstaticclass IndexMapper extends
       Mapper<LongWritable,Text,Text,Text>{
       private Text mapOutputKey = new Text();
       private Text mapOutputValue = new Text("1");
       @Override
       publicvoid map(LongWritable key, Text value, Context context)
              throws IOException, InterruptedException {
           String lineValue=value.toString();
           String strs[]=lineValue.split("##");
           String url = strs[0] ;
           String title = strs[1] ;
           String content = strs[2] ;
           String[] tstrs = title.split(" ") ;
           for(String ts : tstrs){
              mapOutputKey.set(ts+","+url);
              context.write(mapOutputKey, mapOutputValue);
           }
           String[] cstrs = content.split(" ") ;
           for(String cs : cstrs){
              mapOutputKey.set(cs+","+url);
              context.write(mapOutputKey, mapOutputValue);
           }
        }
    }
   
    publicstaticclass  IndexCombiner extends
    Reducer<Text,Text,Text,Text>{
    private Text combinerOutoutKey = new Text() ;
    private Text combinerOutputValue = new Text();
   
    @Override
    publicvoid reduce(Text key, Iterable<Text> values,
           Context context)
           throws IOException, InterruptedException {
      
       String keys[]=key.toString().split(",");
       combinerOutoutKey.set(keys[0]);
       intsum = 0 ;
       for(Text value : values){
           sum += Integer.parseInt(value.toString());
       }
       combinerOutputValue.set(keys[1]+"->"+sum);
       context.write(combinerOutoutKey, combinerOutputValue);
    }
}
    publicstaticclass IndexReducer extends
       Reducer<Text,Text,Text,Text>{
       private Text outputKey = new Text() ;
       private Text splitline = new Text("----------------");
       private Text splitline1 = new Text("----------------------------------------");
       @Override
       publicvoid reduce(Text key, Iterable<Text> values,
              Context context)
              throws IOException, InterruptedException {
           outputKey.set("key:"+key);
           context.write(outputKey, null);
           context.write(splitline, null);
           for(Text value : values){
              context.write(null, value);
           }
           context.write(splitline1, null);
       }
    }
    publicint run(String[] args) throws Exception {
       Configuration configuration = super.getConf() ;
       Job job = Job.getInstance(
           configuration,
           this.getClass().getSimpleName()
       );
       job.setJarByClass(this.getClass());
       Path inPath = new Path(args[0]) ;
       FileInputFormat.addInputPath(job, inPath);
       job.setMapperClass(IndexMapper.class);
       job.setMapOutputKeyClass(Text.class);
       job.setMapOutputValueClass(Text.class);
       job.setCombinerClass(IndexCombiner.class);
       job.setReducerClass(IndexReducer.class);
       job.setOutputKeyClass(Text.class);
       job.setOutputValueClass(Text.class);
       Path outPath = new Path(args[1]);
       FileOutputFormat.setOutputPath(job, outPath);
       booleanisSuccess = job.waitForCompletion(true);
       returnisSuccess ? 0 : 1 ;
    }
   
    publicstaticvoid main(String[] args) throws Exception {
       Configuration configuration = new Configuration();
       intstatus = ToolRunner.run(//
           configuration, //
           new InvertedIndexMapReduce(), //
           args
       ) ;
       System.exit(status);
    }
}