两表MapReducer

最新推荐文章于 2021-01-19 02:13:39 发布

原创最新推荐文章于 2021-01-19 02:13:39 发布 · 287 阅读

0 ·

CC 4.0 BY-SA版权

mapreducer 专栏收录该内容

2 篇文章

订阅专栏

本文介绍了一个使用Hadoop进行学生分数处理的MapReduce程序。该程序能够从不同文件中读取学生的姓名和成绩数据，并通过MapReduce框架进行处理，最终输出每个学生的各科成绩。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

不多说直接上代码

输出结果如下:

zhangsan   math=80
zhangsan   computer=56
zhangsan   english=78
lisi   english=69
lisi   math=76
lisi   computer=77
wangwu   english=88
wangwu   computer=84
wangwu   math=90
zhaoliu   math=67
zhaoliu   computer=92
zhaoliu   english=98
jack   english=56
jack   math=78
jack   computer=55

其代码如下:

import java.io.IOException;
import java.util.ArrayList;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class score {
   public static void main(String[] args) throws IllegalArgumentException, IOException, ClassNotFoundException, InterruptedException {
       Configuration conf = new Configuration();

       conf.set("fs.defaultFS", "hdfs://192.168.0.100:9000");
       conf.set("yarn.resourcemanager.hostname", "192.168.0.100");

       Job job=Job.getInstance(conf);
       job.setMapperClass(mymapper.class);
       job.setMapOutputKeyClass(Text.class);
       job.setMapOutputValueClass(Text.class);

       job.setReducerClass(myreducer.class);
       job.setOutputKeyClass(Text.class);
       job.setOutputValueClass(Text.class);

       FileInputFormat.addInputPath(job, new Path("/input/student/"));
       Path path = new Path("/stu1/");
       path.getFileSystem(conf).delete(path, true);
       FileOutputFormat.setOutputPath(job, path);
       boolean waitForCompletion = job.waitForCompletion(true);
       if(waitForCompletion){
           System.out.println("成功啦");
       }
   }
   public static class mymapper extends Mapper<LongWritable, Text, Text, Text>{
       Text k=new Text();
       Text v=new Text();
       @Override
       protected void map(LongWritable key, Text value, Context context)
               throws IOException, InterruptedException {
           String[] split = value.toString().split("\t");
           FileSplit fs=(FileSplit) context.getInputSplit();
           String name = fs.getPath().getName();
               if(name.endsWith("student.txt")){
                   k.set(split[0]);
                   v.set(split[1]+"_a");
               }else{
                   k.set(split[0]);
                   v.set(split[1]+"="+split[2]+"_b");
               }
           context.write(k,v);
       }
   }

   public static class myreducer extends Reducer<Text, Text, Text, Text>{
       Text k=new Text();
       @Override
       protected void reduce(Text key, Iterable<Text> value, Context context)
               throws IOException, InterruptedException {
           String stu=null;
           ArrayList<Text> list = new ArrayList<>();
           for (Text t : value) {
               String[] s = t.toString().split("_");
               if("a".equals(s[1])){
                   stu=s[0];
               }else{
                   list.add(new Text(s[0]));
               }
           }
           if(stu!=null&&list.size()>0){
               k.set(stu);
               for (Text test : list) {
                   context.write(k, test);
               }
           }
       }
   }

}