helloworld问题

我是单节点模拟并发模式

下面把整了一下午WordCount的问题总结一下，我是自己实现了一个。

将源码打成jar包

问题1：

命令：xxx@xxx-ubuntu:~/Hadoop/hadoop-0.20.2$ bin/hadoop jar wordcount.jar WordCount input output

报错：

11/12/21 15:07:06 WARN mapred.JobClient: Use GenericOptionsParser for parsing the arguments. Applications should implement Tool for the same.

11/12/21 15:07:06 WARN mapred.JobClient: No job jar file set.  User classes may not be found. See JobConf(Class) or JobConf#setJar(String).

Exception in thread "main" org.apache.hadoop.mapred.FileAlreadyExistsException: Output directory input already exists

解决方案：这个主要是WordCount类没有找到，要加上包名，eg：bin/hadoop jar wordcount.jar org.mypackage.WordCount input output

问题2：（同样的命令，加上包名）

INFO mapred.JobClient: Task Id : attempt_201112211459_0003_m_000000_0, Status : FAILED

java.lang.RuntimeException: java.lang.ClassNotFoundException : cn.edu.fudan.util.WordCount$Map

解决方案：需要在源码中加入一句话——job.setJarByClass(WordCount.class);

问题3：输出的文件夹不能存在，否则报错

Exception in thread "main" org.apache.hadoop.mapred.FileAlreadyExistsException: Output directory output already exists

解决方案：命令——bin/hadoop fs -rmr output

上源码，多半是看别人的，呵呵

package org.mypackage;

import java.io.IOException;

import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

public class WordCount {

public static class Map extends Mapper<LongWritable, Text, Text, IntWritable> {

private final static IntWritable one = new IntWritable(1);

private Text word = new Text();

public void map(LongWritable key, Text value, Context context) 

throws IOException, InterruptedException{

String line = value.toString();

StringTokenizer tokenizer = new StringTokenizer(line);

while (tokenizer.hasMoreTokens()) {

word.set(tokenizer.nextToken());

context.write(word, one);

}

}

}

public static class Reduce extends Reducer<Text, IntWritable, Text, IntWritable> {

@Override

public void reduce(Text key, Iterable<IntWritable> values, Context context)

throws IOException, InterruptedException {

// TODO Auto-generated method stub

int sum = 0;

for (IntWritable val : values)

sum += val.get();

context.write(key, new IntWritable(sum));

}

}

public static void main(String[] args) throws Exception {

Configuration conf = new Configuration();

Job job = new Job(conf, "wordcount");

job.setJarByClass(WordCount.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(IntWritable.class);

job.setMapperClass(Map.class);

job.setReducerClass(Reduce.class);

job.setInputFormatClass(TextInputFormat.class);

job.setOutputFormatClass(TextOutputFormat.class);

FileInputFormat.addInputPath(job, new Path(args[0]));

FileOutputFormat.setOutputPath(job, new Path(args[1]));

job.waitForCompletion(true);

}

}