MR之wordcount

2016-12-06 00:54:49 INFO LocalJobRunner Map Task Executor #0 info.zznet.baiduquestion.MR_baiduquestion - +++++zhidao.baidu.com/question/456266897398790725.html?fr=ala&word=%E6%94%AF%E4%BB%98%E5%AE%9D%E7%BA%A2%E5%8C%85%E5%A6%82%E4%BD%95%E9%A2%86%E5%8F%96&service=bdbox&uid=0avQa_P62al9uvinlPvpaga8Su_8aSaY0u218j8xSfKoL7U4B&from=1001187r&ua=_a-qi4aqBig4NE65I5me6NIy2I_UCvC5SdNqA&ut=5kSYMltqVR57ksaYravjh_h0vhgXuDPWpi3pur9PC&osname=baiduboxapp&osbranch=a0&pkgname=com.baidu.searchbox_AndroidM&network=1_0&cfrom=1001187r&ctv=2&cen=uid_ua_ut&typeid=1
package com.joywise;

import info.zznet.udf.keywordreport.AnalysisKeyword;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;

import java.io.IOException;
public class AnalyseFile {
    public static class Map extends
            Mapper {
        private final static Text one = new Text();//输出value
        private Text word = new Text();//输出key

        public void map(LongWritable key, Text value, Context context)//输入的key和value
                throws IOException, InterruptedException {
            String line = value.toString();
            String[] lineArr = line.split("\t");
            if (lineArr != null && lineArr.length == 10) {
                word.set(lineArr[8]);
                one.set(lineArr[8]);
                context.write(word, one);
            }
        }
    }

    public static class Reduce extends
            Reducer {
        AnalysisKeyword analysisKeyword = new AnalysisKeyword();
        public void reduce(Text key, Iterable values,//输入的key和value
                           Context context) throws IOException, InterruptedException {
            for (Text val : values) {
                try {
                    String result = analysisKeyword.exec(val.toString());//输出的value
                    if (result == null || result.length() == 0)
                        return;
                    char[] ch = result.toCharArray();
                    for (int i = 0; i < ch.length; i++) {
                        char c = ch[i];
                        if (CharUtil.isChinese(c)){
                            context.write(key, new Text(result));
                            break;
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
                break;//Remove the duplicate
            }
        }
    }

    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
        Job job = Job.getInstance(conf);
        job.setJobName("AnalyseFile");
        job.setJarByClass(AnalyseFile.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);

        job.setMapperClass(Map.class);
        job.setReducerClass(Reduce.class);

        job.setInputFormatClass(TextInputFormat.class);
        job.setOutputFormatClass(TextOutputFormat.class);
        
        FileInputFormat.setInputDirRecursive(job, true);
        FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
        
        Path outPath = new Path(otherArgs[1]);
        FileSystem fileSystem = outPath.getFileSystem(conf);
        if (fileSystem.exists(outPath)){
            fileSystem.delete(outPath,true);
        }
        FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));


        job.waitForCompletion(true);
    }

}

Map阶段：自动循环执行每一行

Reduce阶段：去重 因为key只能是唯一的

Mapper<LongWritable, Text, Text, IntWritable>

     行数     文本内容 文本内容  迭代值eg、词频统计中的1（map阶段输入的key/value 和输出阶段的key/value类型及所代表的意义）

Reducer<Text, IntWritable, Text, NullWritable>

  文本内容 经过shuffer阶段的迭代值eg、<1,1,1,1...> 文本内容  空值（reduce阶段输入的key/value 和输出阶段的key/value类型及所代表的意义）