Hadoop1.x 的MapReduce 简单例子WordCount

一. 前言

 之前笔记里记得案例今天看到了，所以拿出来分享。

 **首先介绍一下案例需求，统计出hadoop上的一个hello目录下的文件不同单词的个数，并输出统计结果。**

MapReduce 是一种分布式计算模型，主要分为Map和Reduce两部分，用户只需要实现map()和reduce()函数就可以，一般两个函数之间以key和value这种键值对传递参数

二.代码

import java.io.IOException;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;

public class WordCountApp {

    static final String INPUT_PATH = "hdfs://hadoop1:9000/hello";
    static final String OUT_PATH = "hdfs://hadoop1:9000/out";

    public static void main(String[] args) throws Exception {

        Configuration conf = new Configuration();

        final Job job = new Job(conf,WordCountApp.class.getSimpleName());
        //判断输出文件是否存在，存在就删除
        FileSystem fileSystem = FileSystem.get(new URI(INPUT_PATH), conf);
        Path outPath = new Path(OUT_PATH);
        if(fileSystem.exists(outPath)){
            fileSystem.delete(outPath, true);
        }


        // 指定输入目录
        FileInputFormat.setInputPaths(job, INPUT_PATH);

        //指定输入数据进行格式化处理的类
        job.setInputFormatClass(TextInputFormat.class);

        //指定自定义的Mapper类
        job.setMapperClass(MyMapper.class);
        //指定Mapper输出<k,v>类型
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(LongWritable.class);
        //分区
        job.setPartitionerClass(HashPartitioner.class);
        //设置reduce个数
        job.setNumReduceTasks(1);

        //指定自定义的Reducer类
        job.setReducerClass(MyReduce.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(LongWritable.class);

        //指定输出的路径
        FileOutputFormat.setOutputPath(job, new Path(OUT_PATH));
        //指定输出的格式化类
        job.setOutputFormatClass(TextOutputFormat.class);
        //将整个作业提交给JobTracker
        job.waitForCompletion(true);



    }
    /**
     * k1    每一行起始的位置
     * v1    每一行的文本内容
     * k2    每一行中的每个单词
     * v2    每一行中的每个单词出现的次数，固定值1
     * @author mademin
     *
     */
    static class MyMapper extends Mapper<LongWritable, Text, Text, LongWritable>{

        @Override
        protected void map(LongWritable k1, Text v1,
                Mapper<LongWritable, Text, Text, LongWritable>.Context context)
                throws IOException, InterruptedException {
            //将每行文本按照制表符来进行分割
            String[] splited = v1.toString().split("\t");
            for(String word : splited){

                context.write(new Text(word), new LongWritable(1L));
            }


        }
    }

    /**
     * k2 
     * v2
     * k3      整个文件中的不同单词
     * v3      整个文件中的不同单词出现总数
     * @author mademin
     *
     */
    static class MyReduce extends Reducer<Text, LongWritable, Text, LongWritable>{

        @Override
        protected void reduce(Text k2, Iterable<LongWritable> v2s,
                Reducer<Text, LongWritable, Text, LongWritable>.Context context)
                throws IOException, InterruptedException {
            long sum = 0L;

            for(LongWritable v2 : v2s){

                sum += v2.get();
            }

            context.write(k2, new LongWritable(sum));
        }
    }

}