MapReduce之求每年最高气温(练习)

最新推荐文章于 2022-04-23 21:14:48 发布

原创最新推荐文章于 2022-04-23 21:14:48 发布 · 728 阅读

3 ·

CC 4.0 BY-SA版权

本文介绍了一个基于Hadoop MapReduce的温度数据处理系统，该系统能够从大量气象数据中提取年份信息，并通过MapReduce框架计算每年的最高温度。Mapper阶段解析数据并按年份分组，Reducer阶段找出每组的最大温度。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

static {
System.setProperty(“hadoop.home.dir”,“E:/x3/hadoop-2.9.2”);
}

public static class MyMapper extends Mapper<LongWritable,Text,Text,Temp>{
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        //将value转换成字符串形式
        String valueLine = value.toString();
        //根据制表符分割
        String[] split = valueLine.split("\t");
        //截取年份
        String year = split[0].substring(0, 4);
        context.write(new Text(year),new Temp(split[0],Double.parseDouble(split[1])));
    }
}

public static class MyReduce extends Reducer<Text,Temp,Text,Temp>{
    @Override
    protected void reduce(Text key, Iterable<Temp> values, Context context) throws IOException, InterruptedException {
        //年    年月日 + 气温
        Double tempMax = Double.MIN_VALUE;
        String year = "";
        for (Temp temp : values){

            if(temp.getTemp() > tempMax){
                tempMax =temp.getTemp();
                year = temp.getYear();
            }

        }
        context.write(key,new Temp(year,tempMax));
    }
}

public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {

    //job
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "temp-max");
    //输入文件
    FileInputFormat.addInputPath(job,new Path(args[0]));
    //map计算
    job.setMapperClass(MyMapper.class);
    job.setMapOutputValueClass(Text.class);
    job.setMapOutputKeyClass(Temp.class);
    //reduce计算
    job.setReducerClass(MyReduce.class);
   /* job.setMapOutputValueClass(Text.class);
    job.setMapOutputKeyClass(Text.class);*/
    //使用文件系统 判断文件是否存在
    FileSystem fs = FileSystem.get(conf);
    if(fs.exists(new Path(args[1]))){
        //如果存在则删除
        fs.delete(new Path(args[1]),true);
    }
    //输出文件
    FileOutputFormat.setOutputPath(job,new Path(args[1]));
    //提交
    boolean result = job.waitForCompletion(true);
    System.out.println(result);

}

实体类
public class Temp implements WritableComparable{

private String year;
private Double temp;

public String getYear() {
    return year;
}

public Double getTemp() {
    return temp;
}

public Temp() {
}

public Temp(String year, Double temp) {
    this.year = year;
    this.temp = temp;
}

public void setYear(String year) {
    this.year = year;
}

public void setTemp(Double temp) {
    this.temp = temp;
}


@Override
public String toString() {
    return "Temp{" +
            "year='" + year + '\'' +
            ", temp='" + temp + '\'' +
            '}';
}

//比较
@Override
public int compareTo(Temp o) {

    return this.compareTo(o);
}
//写
@Override
public void write(DataOutput out) throws IOException {
    out.writeUTF(year);
    out.writeDouble(temp);
}
//读
@Override
public void readFields(DataInput in) throws IOException {
    this.year = in.readUTF();
    this.temp = in.readDouble();
}

}