hadoop初学-WordCount

WordCount.java

import java.util.StringTokenizer;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
//依次对应
//LongWritable-输入数据键类型（行号）
//Text-输入数据值类型（key-value对）
//Text-输出数据键类型
//IntWritable-输出数据值类型
//举例：输入数据：[1]  [apple 1]  输出 [apple] [1]
public class WordCount extends Mapper<LongWritable, Text, Text, IntWritable>{
    protected void map(LongWritable key, Text value, org.apache.hadoop.mapreduce.Mapper<LongWritable,Text,Text,IntWritable>.Context context) 
            throws java.io.IOException ,InterruptedException {
        String line=value.toString();
        StringTokenizer st=new StringTokenizer(line);
        while(st.hasMoreTokens()){
            String word=st.nextToken();
            context.write(new Text(word), new IntWritable(1));//map的输出
        }
    };

}

WordCountReduce.java

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
//输入map以后的键-值形式
public class WordCountReduce extends Reducer<Text, IntWritable, Text, IntWritable>{
   protected void reduce(Text key, java.lang.Iterable<IntWritable> iterable, org.apache.hadoop.mapreduce.Reducer<Text,IntWritable,Text,IntWritable>.Context context) 
           throws java.io.IOException ,InterruptedException {
       int sum=0;
       //这里的iterable里是有多少个键就有多少个1，比如hello 1 //hello 1  hello 1 则iterable就是由3个1组成的迭代器
       for(IntWritable i:iterable){
           sum=sum+i.get();
       }
       context.write(key, new IntWritable(sum));
   };
}

JobRun.java

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;


public class JobRun {
    public static void main(String [] args) throws IOException, ClassNotFoundException, InterruptedException{
        Configuration conf=new Configuration();
        try {
            Job job=Job.getInstance(conf,"word count");
            job.setJarByClass(JobRun.class);
            job.setMapperClass(WordCount.class);
            job.setCombinerClass(WordCountReduce.class);
            job.setReducerClass(WordCountReduce.class);
            job.setMapOutputKeyClass(Text.class);
            job.setMapOutputValueClass(IntWritable.class);
            //设置mapreduce输入文件的目录
            FileInputFormat.addInputPath(job, new Path("/usr/input/wc"));   
            //设置mapreduce输出文件的目录
            FileOutputFormat.setOutputPath(job, new Path("/usr/output/wc"));
            //等待Job完成
            System.exit(job.waitForCompletion(true)?0:1);

        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }

}

特别注意：
如果以前运行成功过需要删除输出文件，不然重新启动会出错
执行方法：
1.把wenjian用Eclipse打包成jar包
2.命令：
/hadoop/bin/hadoop jar /home/hadoop/temp/wc.jar JobRun
/home/hadoop/temp/wc.jar为本地目录
输入文件要拷入hdfs文件系统所在目录
JobRun为主方法名

3.去输出文件夹查看效果：