Hadoop基础认识之WordCount

Hadoop WordCount 实现

最新推荐文章于 2022-09-23 18:57:28 发布

原创最新推荐文章于 2022-09-23 18:57:28 发布 · 376 阅读

0 ·

CC 4.0 BY-SA版权

大数据开发专栏收录该内容

11 篇文章

订阅专栏

本文介绍了使用Hadoop实现WordCount的基本原理与步骤，包括Mapper和Reducer的具体实现，以及如何设置Job并运行。

大数据的核心是分布式存储HDFS和分布式计算MapReduce！

其中分布式计算MapReduce最基础实例Wordcount如下所示：

依赖jar包
   $HADOOP_HOME/share/hadoop/common
   $HADOOP_HOME/share/hadoop/common/lib
   $HADOOP_HOME/share/hadoop/mapreduce
   $HADOOP_HOME/share/hadoop/mapreducel/lib

1.WordCountMapper.java代码段：

package wc;

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

//                                      泛型                 k1       v1    k2       v2
public class WordCountMapper extends Mapper<LongWritable, Text, Text, IntWritable>{

	@Override
	protected void map(LongWritable key1, Text value1, Context context)
			throws IOException, InterruptedException {
		/*
		 * context 表示Mapper的上下文
		 * 上文：HDFS
		 * 下文：Mapper
		 */
		//数据： I love Beijing
		String data = value1.toString();
		
		//分词
		String[] words = data.split(" ");
		
		//输出 k2    v2
		for(String w:words){
			context.write(new Text(w), new IntWritable(1));
		}
	}

}

2.WordCountReducer.java代码段：

package wc;

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

//                                             k3      v3         k4       v4
public class WordCountReducer extends Reducer<Text, IntWritable, Text, IntWritable> {

	@Override
	protected void reduce(Text k3, Iterable<IntWritable> v3,Context context) throws IOException, InterruptedException {
		/*
		 * context是reduce的上下文
		 * 上文
		 * 下文 
		 */
		//对v3求和
		int total = 0;
		for(IntWritable v:v3){
			total += v.get();
		}
		
		//输出   k4 单词   v4  频率
		context.write(k3, new IntWritable(total));
	}

}

3.WordCountMain.java代码段：

package wc;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class WordCountMain {

	public static void main(String[] args) throws Exception {
		// 创建一个job和任务入口
		Job job = Job.getInstance(new Configuration());
		job.setJarByClass(WordCountMain.class);  //main方法所在的class
		
		//指定job的mapper和输出的类型<k2 v2>
		job.setMapperClass(WordCountMapper.class);
		job.setMapOutputKeyClass(Text.class);    //k2的类型
		job.setMapOutputValueClass(IntWritable.class);  //v2的类型
		
		//指定job的reducer和输出的类型<k4  v4>
		job.setReducerClass(WordCountReducer.class);
		job.setOutputKeyClass(Text.class);  //k4的类型
		job.setOutputValueClass(IntWritable.class);  //v4的类型
		
		//指定job的输入和输出
		FileInputFormat.setInputPaths(job, new Path(args[0]));
		FileOutputFormat.setOutputPath(job, new Path(args[1]));
		
		//执行job
		job.waitForCompletion(true);
	}

}