Hadoop的MapReduce计算天气数据的高低温值

最新推荐文章于 2024-01-16 01:00:00 发布

翻译最新推荐文章于 2024-01-16 01:00:00 发布 · 1k 阅读

Hadoop学习专栏收录该内容

1 篇文章

订阅专栏

本文介绍了一种在Ubuntu环境下使用Hadoop MapReduce框架对历史温度数据进行分析的方法，包括提取年度最高、最低及平均温度的过程。通过自定义Mapper和Reducer类，实现了对大量气象数据的高效处理。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

/*
Ubuntu下使用Mapreduce对当年的温度数据进行分析提取
*/
才开始学习Hadoop，这项实验也是查阅了许多资料

package com.bipt.model.wether;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

import java.io.IOException;
 
public class TemperatureComputation {
    public static class TemperatureMapper extends Mapper<LongWritable, Text,Text, IntWritable>{
    	private static final Integer ERROR_TEMPER = 9999;
        @Override
        protected void map(LongWritable key, Text value, Context context)
                throws IOException, InterruptedException {
            String content = value.toString();        //值以字符串形式存入
            String year = content.substring(15,19);    //获取年份
            String month=content.substring(19,21);//获取月份

        Integer temperature  = null;    //获取温度
        if(content.charAt(87)=='+'  ) {
            temperature = Integer.parseInt(content.substring(88, 92));
            
        } else {
 			temperature = Integer.parseInt(content.substring(87, 92));
        }
        if(temperature <= ERROR_TEMPER && content.substring(92, 93).matches("[01459]")) {
            context.write(new Text(year+"年"+month+"月"), new IntWritable(temperature));
        }
    }
}

public static class TemperatureReduce extends Reducer<Text,IntWritable,Text,IntWritable>{
 @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context)
            throws IOException, InterruptedException {

        int maxTemperature = Integer.MIN_VALUE;  //定义最高温度
        int minTemperature = Integer.MAX_VALUE;  //定义最低温度
        int sum = 0;
        int count = 0;

        for(IntWritable value : values) {
            maxTemperature = Math.max(maxTemperature, value.get());  //获取最高温度
            minTemperature = Math.min(minTemperature, value.get());   //获取最低温度
 			sum += value.get();  //计算温度总和
            count++;
        }
        int average = (int) sum/count;  //计算平均温度

        context.write(new Text(key+"   max"), new IntWritable(maxTemperature));   //温度输出格式
        context.write(new Text(key+"   min"), new IntWritable(minTemperature));
        context.write(new Text(key+"   ave "), new IntWritable(average));
    }

}


public static void main(String[] args) throws Exception{
	 Configuration conf = new Configuration();
     conf.set("fs.defaultFS", "hdfs://localhost:9000");
     String[] otherArgs = new String[]{"input path", "output path"};
    if (otherArgs.length != 2){
        System.err.println("Usage: TemperatureComputation<input path> <output path>");
        System.exit(-1);
    }
    Job job = Job.getInstance(conf, "temperature");  //设置一个用户定义的job名称
    job.setJarByClass(TemperatureComputation.class);
    job.setMapperClass(TemperatureMapper.class);  //为job设置M a p p e r类
    job.setMapOutputKeyClass(Text.class);  
    job.setMapOutputValueClass(IntWritable.class);  //为job输出设置value类

    job.setReducerClass(TemperatureReduce.class);  //为job设置Reduce类
    job.setOutputKeyClass(Text.class);  //为job输出数据设置Key类
    job.setOutputValueClass(IntWritable.class);

    FileInputFormat.addInputPath(job, new Path(otherArgs [0]));  //添加输入路径
    FileOutputFormat.setOutputPath(job,new Path(otherArgs [1]));

    System.exit(job.waitForCompletion(true)?1:0);
}

}