MapReduce 简单实现统计

最新推荐文章于 2024-12-22 11:24:55 发布

原创最新推荐文章于 2024-12-22 11:24:55 发布 · 778 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#java #MapReduce #大数据 #Hadoop

java 同时被 2 个专栏收录

9 篇文章

订阅专栏

hadoop

3 篇文章

订阅专栏

本文介绍了一个使用Hadoop实现的最大温度查找程序。该程序通过Mapper将数据格式化并发送到Reducer，Reducer则找出每个站点记录中的最高温度。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

reduce，对于许多的key我们的分布式进行分发，然后根据我们设计的计算方式得出结果package com.tem;
import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;

import com.exercise.hadoop.command.HadoopManipulate;


public class MaxTemperature {
	public static Configuration config;

	static{
		config=new Configuration();
	}
	public static void main(String[] args) throws IOException {
//		if (args.length !=2) {
//			System.err.println("Usage: Maxtemperature <input path> <output path>");
//			System.exit(-1);
//		}
		JobConf conf = new JobConf(MaxTemperature.class);
		conf.setJobName("max temperature");
		FileInputFormat.addInputPath(conf, new Path("/t1/test"));
		FileOutputFormat.setOutputPath(conf, new Path("/z2"));
		conf.setMapperClass(MaxTemperatureMapper.class);
		conf.setReducerClass(MaxTemperatureReducer.class);
		conf.setOutputKeyClass(Text.class);
		conf.setOutputValueClass(Text.class);
		
		JobClient.runJob(conf);
		HadoopManipulate hm = new HadoopManipulate();
		hm.readFile("/z2/part-00000");
	}
}

package com.tem;
import java.io.IOException;
import java.util.Iterator;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;

public class MaxTemperatureReducer extends MapReduceBase implements
		Reducer<Text, Text, Text, Text> {
	
	public void reduce(Text key, Iterator<Text> values,
			OutputCollector<Text, Text> output, Reporter reporter)
			throws IOException {
		int maxValue = Integer.MIN_VALUE;
		while (values.hasNext()) {
			maxValue = Math.max(maxValue, Integer.parseInt(values.next().toString()));
		}
		output.collect(key, new Text(String.valueOf(maxValue)));
	}


}

Mapper，作为hadoop强大的统计工具，讲我们大数据在文件中按照我们规定的格式写出后，对key进行查重合并传给reduce

package com.tem;
import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;

public class MaxTemperatureMapper extends MapReduceBase implements
		Mapper<LongWritable, Text, Text, Text> {


	/**
	 * @param args
	 */

	public void map(LongWritable key, Text value,
			OutputCollector<Text, Text> output, Reporter reporter)
			throws IOException {
		String line =value.toString();
		if(line.split(" ").length==2)
		{
		String first = line.split(" ")[0];
		String second=line.split(" ")[1];
		output.collect(new Text(first), new Text(second));
		}
	}
}