hadoop----MapReduce简单案例（续）

找出气象数据集中的最高气温

map函数

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

/**
 * 
 * 根据全球气象温度数据：统计每年的最高气温
 * 原始数据：
 * 0029029070999991901010106004+64333+023450FM-12+000599999V0202701N015919999999N0000001N9-00781+99999102001ADDGF108991999999999999999999
 * 0029029070999991901010113004+64333+023450FM-12+000599999V0202901N008219999999N0000001N9-00721+99999102001ADDGF104991999999999999999999
 * 0029029070999991901010120004+64333+023450FM-12+000599999V0209991C000019999999N0000001N9-00941+99999102001ADDGF108991999999999999999999
 *      
 *      KEYIN:   LongWritable
 *      VALUEIN: Text
 *      KEYOUT:  Text
 *      VALUEOUT: IntWritable
 *             map阶段
 *     
 *     k1 v1 
 *     0  0029029070999991901010106004+64333+023450FM-12+000599999V0202701N015919999999N0000001N9-00781+99999102001ADDGF108991999999999999999999
 *     133 0029029070999991901010113004+64333+023450FM-12+000599999V0202901N008219999999N0000001N9-00721+99999102001ADDGF104991999999999999999999
 *     266 0029029070999991901010120004+64333+023450FM-12+000599999V0209991C000019999999N0000001N9-00941+99999102001ADDGF108991999999999999999999
 * 
 *         经过map函数  k2 v2
 *           ("1901",-78)
 *           ("1901",-72)
 *           ("1901",-94)
 * 
 * 		   map函数的编写：
 * 
 * 
 */
public class MyMapper extends Mapper<LongWritable, Text, Text, IntWritable>{

	@Override
	protected void map(LongWritable key, Text value, Context context)
			throws IOException, InterruptedException {
		//将value转成字符串类型，进行截取年份和温度
		String line = value.toString();
		//获取年份
		String year = line.substring(15, 19);
		//定义温度变量
		int temp;
		if(line.charAt(87)=='+'){
			temp = Integer.parseInt(line.substring(88, 92));
		}else{
			temp = Integer.parseInt(line.substring(87, 92));
		}
		String code = line.substring(92, 93);
		if(temp!=9999&&code.matches("[01459]")){
			//转换年份的类型为输出类型
			Text yearT = new Text(year);
			//转换温度的类型为输出类型
			IntWritable temperature = new IntWritable(temp);
			//将数据写出
			context.write(yearT, temperature);
			//context.write(new Text(year), new IntWritable(temp));
			
		}
		
	}
	
}

reducer函数

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

/**
 * Reduce阶段：
  *   接收的数据       ("1901",-78)
  *            ("1901",-72)
  *            ("1901",-94)
 * 
 * KEYIN:   Text
 * VALUEIN: IntWritable
 * KEYOUT: Text
 * VALUEOUT:IntWritable
 * 
 *         reduce函数接收的数据：("1901",[-78,-72,-94]) 
 *         reduce函数输出的数据：("1901",maxTemperature)
 * 
 *
 */
public class MyReducer extends Reducer<Text, IntWritable, Text, IntWritable>{

	@Override
	protected void reduce(Text year, Iterable<IntWritable> temps,
			Context context) throws IOException, InterruptedException {
		//定义一个变量
		int maxTemperature = Integer.MIN_VALUE;
		//遍历所有的温度
		for(IntWritable i : temps){
			//当前温度和变量所指的温度比较，找出较大值，赋值给当前变量
			maxTemperature = Math.max(i.get(), maxTemperature);
		}
		//循环结束后，当前变量就是最高温度，写出去
		context.write(year, new IntWritable(maxTemperature));
	}
}

驱动类

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class MyMaxTemperature {

	public static void main(String[] args) throws Exception, IOException {
		// TODO Auto-generated method stub
		//获取一个Job对象
				Configuration conf = new Configuration();
				Job job = Job.getInstance(conf,"maxtemp");
				//设置主类
				job.setJarByClass(MyMaxTemperature.class);
				//设置map函数的类型
				job.setMapperClass(MyMapper.class);
				//设置reduce函数的类型
				job.setReducerClass(MyReducer.class);
				//设置输出键值对的类型
				job.setOutputKeyClass(Text.class);
				job.setOutputValueClass(IntWritable.class);
				//设置输入输出路径
				FileInputFormat.addInputPath(job, new Path(args[0]));
				FileOutputFormat.setOutputPath(job, new Path(args[1]));
				//查看状态，推出程序
				System.exit(job.waitForCompletion(true)?0:1);
	}
}

测试

查看结果

在IDE中运行程序

在驱动类中，右键选择Debug AS---->Debug Configurations

运行

查看结果

注意：

IDE中运行的程序，在http://192.168.172.150:8088/中无法查看信息

若出现错误：org.apache.hadoop.io.nativeio.NativeIO$Windows.access0(Ljava/lang/String;I)Z
可查看网址：https://blog.youkuaiyun.com/congcong68/article/details/42043093
进行对应程序调试