mapreduce之数据排序_mapreduce对数据排序-优快云博客

本文链接：https://blog.youkuaiyun.com/weixin_55885866/article/details/130524409

该代码示例展示了使用HadoopMapReduce框架进行数据排序的过程。Mapper类将输入文本中的数字转换为键值对，Reducer类则进行聚合计数。主函数配置了作业并指定输入输出路径。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

1.mapp

ackage hzy.com.WordSort;

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class WordSortMapper extends Mapper<LongWritable, Text, IntWritable, IntWritable>{
private IntWritable data = new IntWritable();

   @Override
   protected void map(LongWritable key, Text value,
           Mapper<LongWritable, Text, IntWritable, IntWritable>.Context context)
           throws IOException, InterruptedException {

       //从小到大输出每个数字
   String   a = value.toString();
   data.set(Integer.parseInt(a));
   //每行 <成绩，1>
   context.write(data, new IntWritable('1'));

   }

}

2.reduce

package hzy.com.WordSort;

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Reducer;

public class WordSortReduce extends Reducer<IntWritable, IntWritable, IntWritable, IntWritable>{
private IntWritable count = new IntWritable(1);

   @Override
   protected void reduce(
           IntWritable arg0,
           Iterable<IntWritable> arg1,
           Reducer<IntWritable, IntWritable, IntWritable, IntWritable>.Context arg2)
           throws IOException, InterruptedException {

       arg2.write(arg0, count);
       count.set(count.get()+1);

   }

}

3.main

ic void main(String[] args) throws Exception{
          //1.连接hadoop
       Configuration cf=new Configuration();
          cf.set("fs.defaultFS","hdfs://hadoop0:9000/");
          //2.创建Job，设置入口
       Job job = Job.getInstance(cf);
       job.setJarByClass(WordSortApp.class);
       //3.读取hdfs的数

       FileInputFormat.addInputPath(job, new Path("/data/sore3.txt"));
       FileInputFormat.addInputPath(job, new Path("/data/sore1.txt"));

       //4.进行mapper计算
       job.setMapperClass(WordSortMapper.class);
       //map输出 key
       job.setMapOutputKeyClass(IntWritable.class);
       //map输出value
       job.setMapOutputValueClass(IntWritable.class);


       // job.setCombinerClass(WordSortReduce.class);


       //5.进行reducer计
       job.setReducerClass(WordSortReduce.class);
       //reduce输出key
           job.setOutputKeyClass(IntWritable.class);
           //reduce输出value
           job.setOutputValueClass(IntWritable.class);
           //6.写出结果到hdfs
           FileOutputFormat.setOutputPath(job, new Path("/data/xt97"));//写入的目录应该是空的，否则报错
           //7.提交任
           job.waitForCompletion(true);



   }

}