Hadoop之WordCount计数器程序编写并打包

最新推荐文章于 2024-11-01 21:06:18 发布

UckyK

最新推荐文章于 2024-11-01 21:06:18 发布

阅读量858

点赞数

CC 4.0 BY-SA版权

本文链接：https://blog.youkuaiyun.com/UckyK/article/details/49785883

本文详细介绍了如何使用Hadoop实现WordCount计数器程序，包括导入相关包、编写Mapper和Reducer类，以及如何配置Job进行数据处理。通过实例演示了如何将文本数据映射为键值对，并在Reducer中进行聚合计算，最终输出各单词出现的次数。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

Hadoop之WordCount计数器程序编写并运行

导入包：

package com.ucky.mapReduce;

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;



public class MyWordCount {

	static class MyMapper extends Mapper<LongWritable, Text, Text, IntWritable>{

		private Text word = new Text();
		private final static IntWritable one = new IntWritable(1);
		
		@Override
		protected void map(LongWritable key, Text value, Context context)
				throws IOException, InterruptedException {
			
			//获取每行的数据值
			String lineValue = value.toString();
			
			//进行字符串分割
			StringTokenizer stringTokenizer = new StringTokenizer(lineValue);
			
			//遍历
			while(stringTokenizer.hasMoreTokens()){
				
				//获取每个值
				String wordValue = stringTokenizer.nextToken();
				
				//设置map输出的key
				word.set(wordValue);
				context.write(word, one);
			}
			
		}
	}
	
	static class MyReduce extends Reducer< Text, IntWritable, Text, IntWritable>{

		private IntWritable result = new IntWritable();
		@Override
		protected void reduce(Text key, Iterable<IntWritable> values,Context context)
				throws IOException, InterruptedException {
			
			//用于累加
			int sum = 0;
			
			//遍历Iterable
			for(IntWritable value :values){
				//累加
				sum += value.get();
			}
			
			//设置总次数
			result.set(sum);
			context.write(key, result);
		}
	}
	
	
	public static void main(String[] args) throws Exception  {
		
		//获取配置信息
		Configuration configuration = new Configuration();
		
		//创建job，设置配置和job名称
		Job job = new Job(configuration,"wordCount");
		
		//1、设置job运行类
		job.setJarByClass(MyWordCount.class);
		
		//2、设置mapper类和reducer类
		job.setMapperClass(MyMapper.class);
		job.setReducerClass(MyReduce.class);
		
		//3、设置输入文件的目录和输出目录
		FileInputFormat.addInputPath(job, new Path(args[0]));
		FileOutputFormat.setOutputPath(job, new Path(args[1]));
		
		//4、输出结果的KEY和VALUE的类型
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(IntWritable.class);
		
		//5、提交job，等待运行结果，并在客户端显示运行信息
		job.waitForCompletion(true);
		
		//结束程序
		System.exit(0);
			
	}
}

代码编写成功右键项目 --> EXPORT