Hadoop——测试代码

最新推荐文章于 2025-07-10 13:00:23 发布

原创最新推荐文章于 2025-07-10 13:00:23 发布 · 993 阅读

0 ·

CC 4.0 BY-SA版权

Hadoop 专栏收录该内容

22 篇文章

订阅专栏

该博客介绍了如何在本地和服务器环境下进行Hadoop MapReduce的测试，包括WCMap和WCReduce类的实现，用于单词计数。在本地测试中设置了配置参数，而在服务器测试中未展示具体配置。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

1、本地测试代码

class WCMap extends Mapper<LongWritable, Text, Text, IntWritable>{
/**
* key: 偏移量
* value:每行文本内容
* context：输出内容上下文
*/
@Override
protected void map(LongWritable key, Text value,Context context)
throws IOException, InterruptedException {
IntWritable num = new IntWritable(1);
String str = value.toString();
String[] words = str.split(" ");
for (String word : words) {
context.write(new Text(word), num);
}
}
}
class WCReduce extends Reducer<Text, IntWritable, Text, IntWritable>{
/**
* v1:map输出的key，即词内容
* itr：对应每个词的值集合
* context:输出内容上下文
*/
@Override
protected void reduce(Text v1, Iterable<IntWritable> itr,Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable i : itr) {
sum += i.get();
}
context.write(v1, new IntWritable(sum));
}
}
public class WCJob {
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
//默认加载src目录下的配置文件（本地测试时，src下不能添加MR的配置文件）
Configuration conf = new Configuration();
//本地测试环境
conf.set("fs.defaultFS", "hdfs://slave1:8020");
conf.set("yarn.resourcemanager.hostname", "slave3");

Job job = Job.getInstance(conf);//实例化job

job.setJarByClass(WCJob.class);

job.setMapperClass(WCMap.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);

job.setReducerClass(WCReduce.class);

Path inPath = new Path("/wc/input/wc.txt");//设置输入文件
FileInputFormat.addInputPath(job, inPath);

Path outPath = new Path("/wc/output");//设置结果保存路径
FileSystem fs = FileSystem.get(conf);
if (fs.exists(outPath)) {
fs.delete(outPath,true);
}
FileOutputFormat.setOutputPath(job, outPath);

boolean flag = job.waitForCompletion(true);
if (flag) {
System.out.println("Job Success!");
}
}
}

2、服务器测试代码

Configuration conf = new Configuration();

Job job = Job.getInstance(conf);//实例化job

job.setJarByClass(WCJob.class);

job.setMapperClass(WCMap.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);

job.setReducerClass(WCReduce.class);

Path inPath = new Path("/wc/input/wc.txt");//设置输入文件
FileInputFormat.addInputPath(job, inPath);

Path outPath = new Path("/wc/output");//设置结果保存路径
FileSystem fs = FileSystem.get(conf);
if (fs.exists(outPath)) {
fs.delete(outPath,true);
}
FileOutputFormat.setOutputPath(job, outPath);

boolean flag = job.waitForCompletion(true);
if (flag) {
System.out.println("Job Success!");
}
}
}