【Hadoop】MP程序之统计单词数量

 

通过MP程序统计单词数量

1. WordCount.java

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.log4j.BasicConfigurator;

import java.io.IOException;

/**
 * @author 易上清净
 */
public class WordCount {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        // 自动快速地使用缺省Log4j环境
        BasicConfigurator.configure();
        Configuration configuration = new Configuration();
        String[] otherArgs = new GenericOptionsParser(configuration, args).getRemainingArgs();
        if (otherArgs.length < 2) {
            System.err.println("必须输入读取文件路径和输出路径");
            System.exit(2);
        }
        Job job = Job.getInstance();
        job.setJarByClass(WordCount.class);
        job.setJobName("Word Count");
        JobConf jobConfiguration = (JobConf) job.getConfiguration();
        // 设置读取文件的路径，都是从HDFS中读取。读取文件路径从脚本文件中传进来
        FileInputFormat.addInputPath(jobConfiguration, new Path(args[0]));
        // 设置mapreduce程序的输出路径，MapReduce的结果都是输入到文件中
        FileOutputFormat.setOutputPath(jobConfiguration, new Path(args[1]));
        // 设置实现了map函数的类
        job.setMapperClass(WordCountMap.class);
        // 设置实现了reduce函数的类
        job.setReducerClass(WordCountReduce.class);
        // 设置reduce函数的key值
        job.setOutputKeyClass(Text.class);
        // 设置reduce函数的value值
        job.setOutputValueClass(IntWritable.class);
        System.exit(job.waitForCompletion(true) ? 0 :1);
    }
}

 2.WordCount_old.java

import java.io.*;
import java.util.HashMap;
public class WordCount_Old  {
    public static void main(String[]args) throws IOException {
        File file = new File("input\\test.txt");
        BufferedReader br = new BufferedReader(new FileReader(file));

        String st;
        HashMap<String, Integer> wordAndCount = new HashMap<>();
        while ((st = br.readLine()) != null) {
            String[] words = st.split(" ");
            for(String word : words) {
                if(wordAndCount.get(word) != null) {
                    int cnt = wordAndCount.get(word) + 1;
                    wordAndCount.put(word, cnt);
                } else {  //该单词第一次出现
                    wordAndCount.put(word, 1);
                }
            }
        }
        System.out.println(wordAndCount.toString());
    }
}

 3.WordCountMap.java

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

/**
 * @author 易上清净
 */
public class WordCountMap extends Mapper<LongWritable, Text, Text, IntWritable> {

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String line = value.toString();
        String[] split = line.split(" ");
        for (String s: split) {
            context.write(new Text(s), new IntWritable(1));
        }
    }
}

 4.WordCountReduce.java

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

/**
 * @author 易上清净
 */
public class WordCountReduce extends Reducer<Text, IntWritable, Text, IntWritable> {

    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
        int count = 0;
        for (IntWritable val: values) {
            count++;
        }
        context.write(key, new IntWritable(count));
    }
}

5.最后再运行一遍WordCount_old.java就是我们所要解答的结果了-统计单词数量-MP程序