hadoop中mapreduce提交作业到集群运行报错ClassNotFoundException

最新推荐文章于 2024-05-12 21:25:50 发布

原创最新推荐文章于 2024-05-12 21:25:50 发布 · 1.7k 阅读

3 ·

CC 4.0 BY-SA版权

文章标签：

#hadoop #mapreduce

hadoop 专栏收录该内容

4 篇文章

订阅专栏

本文详细解析了Hadoop WordCount任务中的常见错误——未设置job jar文件，提供了具体的解决方法，并附上了官方demo中WordCount类的完整代码，帮助读者理解如何正确配置Hadoop任务。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

错误信息：

解决方法：

在代码中添加：

job.setJarByClass(MyWordCount.class);

如果未指定，打印信息中会提示：

No job jar file set. User classes may not be found. See Job or Job#setJar(String).

这里贴上官方demo中WordCount类的反编译代码：

package org.apache.hadoop.examples;

import java.io.IOException;
import java.io.PrintStream;
import java.util.Iterator;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Mapper.Context;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.Reducer.Context;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;

public class WordCount
{
  public static void main(String[] args)
    throws Exception
  {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
      System.err.println("Usage: wordcount <in> [<in>...] <out>");
      System.exit(2);
    }
    Job job = new Job(conf, "word count");
    job.setJarByClass(WordCount.class);
    job.setMapperClass(TokenizerMapper.class);
    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    for (int i = 0; i < otherArgs.length - 1; ++i)
      FileInputFormat.addInputPath(job, new Path(otherArgs[i]));

    FileOutputFormat.setOutputPath(job, new Path(otherArgs[(otherArgs.length - 1)]));

    System.exit(1);
  }

  public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable>
  {
    private IntWritable result;

    public IntSumReducer()
    {
      this.result = new IntWritable();
    }

    public void reduce(Text key, Iterable<IntWritable> values, Reducer<Text, IntWritable, Text, IntWritable>.Context context) throws IOException, InterruptedException
    {
      int sum = 0;
      for (Iterator i$ = values.iterator(); i$.hasNext(); ) { IntWritable val = (IntWritable)i$.next();
        sum += val.get();
      }
      this.result.set(sum);
      context.write(key, this.result);
    }
  }

  public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable>
  {
    private static final IntWritable one = new IntWritable(1);
    private Text word;

    public TokenizerMapper()
    {
      this.word = new Text();
    }

    public void map(Object key, Text value, Mapper<Object, Text, Text, IntWritable>.Context context) throws IOException, InterruptedException {
      StringTokenizer itr = new StringTokenizer(value.toString());
      while (itr.hasMoreTokens()) {
        this.word.set(itr.nextToken());
        context.write(this.word, one);
      }
    }
  }
}