Mapreduce《案例之倒排索引》

最新推荐文章于 2019-09-05 11:20:09 发布

原创最新推荐文章于 2019-09-05 11:20:09 发布 · 221 阅读

1 ·

CC 4.0 BY-SA版权

文章标签：

#Mapreduce #倒排索引

分布式专栏收录该内容

7 篇文章

订阅专栏

这篇博客通过一个案例展示了如何使用MapReduce处理数据，构建倒排索引。源数据包括三个文件，内容涉及MapReduce的相关描述。Map阶段将单词和文件路径作为键值对输出，Combiner和Reduce阶段进一步聚合计数。最终输出展示了每个单词在各文件中出现的次数。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

Mapreduce《案例之倒排索引》

源数据：

1）file1：

MapReduce is simple

2）file2：

MapReduce is powerful is simple

3）file3：

Hello MapReduce bye MapReduce

要实现的结果：

样例输出如下所示。

MapReduce      file1.txt:1;file2.txt:1;file3.txt:2;

is        　　　　file1.txt:1;file2.txt:2;

simple        　 file1.txt:1;file2.txt:1;

powerful   　　 file2.txt:1;

Hello       　　 file3.txt:1;

bye       　　   file3.txt:1;

=========================JAVA CODE======================

package gq;

import java.io.IOException;

import java.util.Iterator;

import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.input.FileSplit;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

/**

* Class Description:分词小demo测试类

* Author:gaoqi

* Date:2015年6月5日下午2:03:08

public class FenCi {

public static class Map extends Mapper<LongWritable, Text, Text, Text>{

private FileSplit fileSplit;

public void map(LongWritable key,Text value,Context context) throws IOException, InterruptedException{

fileSplit = (FileSplit) context.getInputSplit();

StringTokenizer stk = new StringTokenizer(value.toString().trim());

while(stk.hasMoreElements()){

String v = stk.nextToken();

String path = fileSplit.getPath().toString();

String filename = path.substring(path.indexOf("file"));

context.write(new Text(v+":"+filename), new Text("1"));

}

public static class Combiner extends Reducer<Text, Text, Text, Text>{

public void reduce(Text key,Iterable<Text> values,Context context) throws IOException, InterruptedException{

int sum = 0;

for(Text t :values){

sum += Integer.parseInt(t.toString());

}

String v = key.toString();

context.write(new Text(v.substring(0,v.indexOf(":"))), new Text(v.substring(v.indexOf(":")+1)+":"+sum));

}

public static class Reduce extends Reducer<Text, Text, Text, Text>{

public void reduce(Text key,Iterable<Text> values,Context context) throws IOException, InterruptedException{

Iterator<Text> its = values.iterator();

String value = "";

while(its.hasNext()){

value+=its.next()+";";

}

context.write(key, new Text(value));

}

public static void main(String[] args) throws Exception{

Configuration conf = new Configuration();

Job job = new Job(conf,"FenCi");

job.setJarByClass(FenCi.class);

job.setMapperClass(Map.class);

job.setCombinerClass(Combiner.class);

job.setReducerClass(Reduce.class);

job.setMapOutputKeyClass(Text.class);

job.setMapOutputValueClass(Text.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(Text.class);

FileInputFormat.addInputPath(job, new Path("hdfs://h0:9000/user/tallqi/in/inputInvertedIndex"));

FileOutputFormat.setOutputPath(job, new Path("hdfs://h0:9000/user/tallqi/in/outputFenci"));

System.exit(job.waitForCompletion(true)?0:1);

}