TFIDF算法Hadoop实现

程序说明：利用MapReduce计算框架，计算一组英文文档中各个单词的TFIDF。某单词在某文档的TFIDF=该单词中该文档的TF×该单词IDF。其中，
  - TF(i,j)：单词i在文档j中出现的频率（Term Frequency）。TF(i,j)=N(i,j)/N(j)，N(i,j)是单词i中文档j中出现的次数，N(j)是文档j的单词总数。
  - IDF(i)：单词i的逆文件频率（Inverse Document Frequency）。IDF(i)=LOG((M+1)/M(i))，M是文件总数，M(i)是包含单词i的文件数。M+1，为了避免M与M(i)相等，导致对数为零的情况。

 程序结构：程序由3个Job，按照chain mapreduce进行计算。其中，
  - Job1：包括Tfmapper、Tfreducer、Tfgroup，计算单词在文档的TF。
  - Job2：包括Idfmapper、Idfreducer、Idfsort，计算单词的IDF。
  - Job3：包括Tfidf_tfmapper、Tfidf_idfmapper、Tfidfreducer、Tfidfsort、Tfidfgroup，计算单词的TFIDF。
 
 代码参考：
   - http://blog.youkuaiyun.com/jackydai987/article/details/6303459

   - blog.youkuaiyun.com/ididcan/article/details/6657977

 TFIDF源代码内容：

1）编写Docword类，把相关单词、对应文件名、相关指标（如IDF、TFIDF等等）作为属性。

<span style="font-size:18px;">package tfidf;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.io.WritableComparable;

public class Docword implements WritableComparable<Docword> {
    String docname;
    String word;
    double index;
    static final String DELIMITER=",";
    public Docword() {
        super();
        // TODO Auto-generated constructor stub
    }

    public String getDocname() {
        return docname;
    }

    public void setDocname(String docname) {
        this.docname = docname;
    }

    public String getWord() {
        return word;
    }

    public void setWord(String word) {
        this.word = word;
    }

    public double getIndex() {
        return index;
    }

    public void setIndex(double index) {
        this.index = index;
    }

    @Override
    public void write(DataOutput out) throws IOException {
        // TODO Auto-generated method stub
        out.writeUTF(docname);
        out.writeUTF(word);
        out.writeDouble(index);
    }

    @Override
    public String toString() {
        return docname+DELIMITER+word;
    }

    @Override
    public void readFields(DataInput in) throws IOException {
        // TODO Auto-generated method stub
        docname=in.readUTF();
        word=in.readUTF();
        index=in.readDouble();
    }

    @Override
    public int compareTo(Docword arg0) {
        // TODO Auto-generated method stub
        int result=0;
        result=(result!=0)?result:docname.compareTo(arg0.getDocname());
        result=(result!=0)?result:word.compareTo(arg0.getWord());
        return result;
    }

}</span>

2）TFIDF代码。已有注释，具体如下。

<span style="font-size:18px;">public class Tfidf extends Configured implements Tool {

    static final String DELIMITER=",";

    /* =======================Job1：Tfidf-tf=======================
     * 程序说明：计算文档中各单词TF。程序主要思路借鉴了MapReduce Tutorial的WordCount2.0代码，
     * - 利用Local Resource，将待删除的字符串（如逗号、句号等）读入内存，对split中每个record删除特定字符串。
     * - 设置”全局“变量（Map类的属性），统计文档的单词总数。注意，文档可能由于超过split大小，需要被分为多个split处理。所以Map中”全局“变量仅统计该split中各文档单词数，实际数需要中reduce中按照filename进行合并。
     *
     * 程序输入：HDFS某目录下所有Text英文文档。
     * 程序输出：各文档中单词的TF，按照“filename, wordname, termfreq”格式输出。
     * （Shuffle）排序规则：1）filename。2）wordname。
     * （Shuffle）合并规则：1）filename。2）wordname。
     */

    public static class Tfmapper extends
            Mapper<LongWritable, Text, Docword, IntWritable> {
        Docword outputKey=new Docword();
        IntWritable outputValue=new IntWritable(1);//拆分每个单词，按频次为1，在reduce中分文件名进行合并
        HashSet<String> skipstr=new HashSet<String>(); //存储待删除的字符串
        int termnumindoc=0;//每个split中各文档中单词总数。

        @Override
        protected void cleanup(Context context) throws IOException,
                InterruptedException {
            // TODO Auto-generated method stub
            FileSplit filesplit=(FileSplit)context.getInputSplit();
            String filename=filesplit.getPath().getName();
            outputKey.setDocname(filename);
            outputKey.setWord("!termnum");//!termnum标识该记录是文件总单词数，由于map中已经对记录删除”！“，所以不会重复。
            context.write(outputKey, new IntWritable(termnumindoc));//按照”filename,!termnum,termnumindoc“格式输出。由于!的ASCII码中所有字母之前，按照Docword.compareTo()，经过shuffle后，!termnum记录会出现中该filename所有记录的第一位。
        }
        @Override
        protected void map(LongWritable key, Text value, Context context)
                throws IOException, InterruptedException {
            // TODO Auto-generated method stub
            String str=value.toString().toLowerCase();
            for(String skip:skipstr){
                str=str.replaceAll(skip, "");//对每个redcord，删除特定字符串
            }
            String []words=StringUtils.split(str, ' ');
            FileSplit filesplit=(FileSplit)context.getInputSplit();//获取InputSplit。每个InputSplit只属于一个InputFile，每个InputFile至少有一个InputSplit。
            String filename=filesplit.getPath().getName();//利用FileSplit，提取输入文件名。

            for(String word:words){
                if(word.trim().isEmpty())//删除空行和空字符串。也可以放在待删除字符串，用正则表达式实现。
                    continue;
                if(word.charAt(0)<'a' || word.charAt(0)>'z')//删除非字母开头的单词。也可以放在待删除字符串，用正则表达式实现。
                    continue;
                termnumindoc++;//文档的单词数加1.
                outputKey.setDocname(filename);//文件名
                outputKey.setWord(word);//单词名
                context.write(outputKey, outputValue);//按照”filename,wordname,1“格式输出到本地文件系统，进行shuffle
            }
        }
        @Override
        protected void setup(Context context) throws IOException,
                InterruptedException {
            // TODO Auto-generated method stub
            String line="";
            BufferedReader fs=new BufferedReader(new FileReader("skipstr"));//读入待剔除的字符串
            while((line=fs.readLine())!=null){
                skipstr.add(line);//字符串逐个加入内存中的HashSet
            }
        }

    }
    public static class Tfcombiner extends
            Reducer<Docword, IntWritable, Docword, IntWritable> {
        IntWritable outputValue=new IntWritable();
        @Override
        protected void reduce(Docword key, Iterable<IntWritable> values,
                Context context)
                throws IOException, InterruptedException {
            // TODO Auto-generated method stub
            int sumall=0;
            for(IntWritable value:values){
                sumall+=value.get();
            }
            outputValue.set(sumall);
            context.write(key, outputValue);
        }

    }
    public static class Tfreducer extends
            Reducer<Docword, IntWritable, Docword, DoubleWritable> {
        DoubleWritable outputValue=new DoubleWritable();
        int termnumindoc=0;//各文档中单词总数。由于文档可能超过split尺寸大小，被拆分在多个split被多个Map统计单词数。在Reduce中对各个Map统计的单词总数进行汇总。
        @Override
        protected void reduce(Docword key, Iterable<IntWritable> values,
                Context context)
                throws IOException, InterruptedException {
            // TODO Auto-generated method stub
            int sumall=0;
            for(IntWritable value:values){
                sumall+=value.get();
            }
            if(key.getWord().equals("!termnum")){//单词名是!termnum，标记该记录是文件的单词总数。
                termnumindoc=sumall;
            }
            else{//单词名不是!termnum，标记该记录是文件实际单词。
                // 由于!的ASCII码中所有字母之前，按照Docword.compareTo()，经过shuffle后，!termnum记录会出现中该filename所有记录的第一位。
                // 计算TF时，分母”文件中单词总数“termnumindoc已有数据。
                outputValue.set((double)1*sumall/termnumindoc);
                context.write(key, outputValue);//按照”filename,wordname,termfreq“输出到HDFS，作为Job2的输入
            }
        }

    }

    public static class Tfpartition extends Partitioner<Docword, IntWritable> {

        @Override
        public int getPartition(Docword key, IntWritable value,
                int numPartitions) {
            // TODO Auto-generated method stub
            return Math.abs((key.getDocname()).hashCode())%numPartitions;
        }

    }

    /* =======================Job2：Tfidf-idf=======================
     * 程序说明：计算各单词idf。程序读入Job1的输出数据，按照"wordname,filename"格式，对各个wordname进行合并，计算word的IDF。
     * - 在run()中，利用HDFS API获取文件总数，并传递给Reduce。
     * - 由于Job1的输出数据按照filename进行排序，顺序读入记录并比较前后两个记录的filename有没有改变。如果没有改变，则文件总数不变；如果有改变，则文件总数加1。
     *
     * 程序输入：Job1的输出文件，格式为“filename, wordname, termfreq”
     * 程序输出：单词的Idf，格式为“wordname, idf”。
     * （Shuffle）排序规则：wordname。
     * （Shuffle）合并规则：wordname。
     */

    public static class Idfmapper extends
            Mapper<LongWritable, Text, Docword, IntWritable> {
        Docword outputKey=new Docword();
        IntWritable outputValue=new IntWritable(1);//拆分单词，按频次为1，在reduce中分文件名进行合并
        int filenum=0;
        String filename="";//文件名。

        @Override
        protected void map(LongWritable key, Text value,
                Context context)
                throws IOException, InterruptedException {
            // TODO Auto-generated method stub
            String []words=StringUtils.split(value.toString(), ',');
            outputKey.setDocname(words[0]);
            outputKey.setWord(words[1]);
            context.write(outputKey, outputValue);
        }

    }
    public static class Idfcombiner extends
            Reducer<Docword, IntWritable, Docword, IntWritable> {
        IntWritable outputValue=new IntWritable();

        @Override
        protected void reduce(Docword key, Iterable<IntWritable> values,
                Context context)
                throws IOException, InterruptedException {
            // TODO Auto-generated method stub
            int sumall=0;
            for(IntWritable value:values){
                sumall+=value.get();
            }
            outputValue.set(sumall);
            context.write(key, outputValue);
        }

    }
    public static class Idfreducer extends
                Reducer<Docword, IntWritable, Text, DoubleWritable> {
            DoubleWritable outputValue=new DoubleWritable();
            Text outputKey=new Text();
            int alldoc=0;//文件总数。

            @Override
            protected void setup(Context context) throws IOException,
                    InterruptedException {
                // TODO Auto-generated method stub
                //将run()中传递的变量读入内存，获得文件总数。
                alldoc=Integer.parseInt(context.getConfiguration().get("filesnum"));
            }

            @Override
            protected void reduce(Docword key, Iterable<IntWritable> values,
                    Context context)
                    throws IOException, InterruptedException {
                // TODO Auto-generated method stub
                //由于Job1的输出数据按照filename进行排序，顺序读入记录并比较前后两个记录的filename有没有改变。
                // 如果没有改变，则文件总数不变；如果有改变，则文件总数加1。
                int termdocnum=0;
                for(IntWritable value:values){
                        termdocnum+=value.get();//单词对应文件数加1。
                }
                outputKey.set(key.getWord());
                outputValue.set((double)Math.log((double)(alldoc+1)/termdocnum));
                context.write(outputKey, outputValue);//输出idf计算结果，输出格式“wordname,idf”
            }
        }
    public static class Idfpartition extends Partitioner<Docword, IntWritable> {

        @Override
        public int getPartition(Docword key, IntWritable value,
                int numPartitions) {
            // TODO Auto-generated method stub
            return Math.abs((key.getWord().hashCode()))%numPartitions;
        }

    }
    public static class Idfsort extends WritableComparator {
        //在shuffle中，所有记录按照wordname进行排序，按照wordname进行合并。
        @Override
        public int compare(WritableComparable a, WritableComparable b) {
            // TODO Auto-generated method stub
            Docword lhs=(Docword)a;
            Docword rhs=(Docword)b;
            return lhs.getWord().compareTo(rhs.getWord());
        }

        public Idfsort() {
            super(Docword.class,true);
            // TODO Auto-generated constructor stub
        }
    }

    /* =======================Job3：Tfidf-tfidf=======================
     * 程序说明：计算各单词tfidf。程序利用MultipleInputs分别读入Job1、Job2的输出数据，类似ReduceSide Join在Reduce中进行汇总计算。
     * - MultipleInputs：配置Map，分别读入Job1、Job2的输出数据。其中，读入job2的输入数据后，设置filename为“!alldoc”。由于!的ASCII码小于所有字母，所以同一个word的job2记录在shuffle中排在job1记录前面。
     * - Reduce：设置sortComparator，按照wordname（高优先级）、filename（低优先级）进行排序；设置groupComparator，按照wordname进行合并记录。
     *
     * 程序输入：Job1的输出文件，格式为“filename, wordname, termfreq”；Job2的输出文件，格式为“wordname,idf”。
     * 程序输出：单词的tfIdf，格式为“filename,wordname, idf”。
     * （Shuffle）排序规则：1）wordname。2）filename。
     * （Shuffle）合并规则：wordname。
     */
    public static class Tfidf_tfmapper extends
            Mapper<LongWritable, Text, Docword, Docword> {
        Docword outputKey=new Docword();
        Docword outputValue=new Docword();
        @Override
        protected void map(LongWritable key, Text value,
                Context context)
                throws IOException, InterruptedException {
            // TODO Auto-generated method stub
            String []words=StringUtils.split(value.toString(), ',');
            outputKey.setWord(words[1]);
            outputKey.setDocname(words[0]);
            outputValue.setDocname(words[0]);
            outputValue.setWord(words[1]);
            outputValue.setIndex(Double.parseDouble(words[2]));
            context.write(outputKey, outputValue);//读入Job1的输出文件，格式为“filename, wordname, termfreq”
        }
    }

    public static class Tfidf_idfmapper extends
            Mapper<LongWritable, Text, Docword, Docword> {
        Docword outputValue=new Docword();
        Docword outputKey=new Docword();
        @Override
        protected void map(LongWritable key, Text value,
                Context context)
                throws IOException, InterruptedException {
            // TODO Auto-generated method stub
            String []words=StringUtils.split(value.toString(), ',');
            outputValue.setDocname("!alldoc");
            outputValue.setWord(words[0]);
            outputValue.setIndex(Double.parseDouble(words[1]));
            outputKey.setWord(words[0]);
            outputKey.setDocname("!alldoc");
            context.write(outputKey, outputValue);//读入Job2的输出文件，格式为“wordname,idf”。
        }
    }

    public static class Tfidfreducer extends
            Reducer<Docword, Docword, Text, DoubleWritable> {
        Text outputKey=new Text();
        DoubleWritable outputValue=new DoubleWritable();
        @Override
        protected void reduce(Docword key, Iterable<Docword> values,
                Context context)
                throws IOException, InterruptedException {
            // TODO Auto-generated method stub
            double termidf=0.0,termfq=0.0;
            for(Docword value:values){
                //读入job2的输入数据后，设置filename为“!alldoc”。由于!的ASCII码小于所有字母，所以同一个word的job2记录在shuffle中排在job1记录前面。
                if(value.getDocname().equals("!alldoc")){
                    termidf=value.getIndex();
                }else{
                    termfq=value.getIndex();
                    outputKey.set(value.getDocname()+","+value.getWord());
                    outputValue.set(termidf*termfq);
                    context.write(outputKey, outputValue);
                }
            }
        }
    }
    public static class Tfidfsort extends WritableComparator {

        @Override
        public int compare(WritableComparable a, WritableComparable b) {
            // TODO Auto-generated method stub
            Docword lhs=(Docword)a;
            Docword rhs=(Docword)b;
            int result=0;
            result=(result!=0)?result:lhs.getWord().compareTo(rhs.getWord());
            result=(result!=0)?result:lhs.getDocname().compareTo(rhs.getDocname());
            return result;//按照wordname、filename进行排序。
        }

        public Tfidfsort() {
            super(Docword.class,true);
            // TODO Auto-generated constructor stub
        }

    }
    public static class Tfidfpartition extends Partitioner<Docword, Docword> {

        @Override
        public int getPartition(Docword key, Docword value,
                int numPartitions) {
            // TODO Auto-generated method stub
            return Math.abs((key.getWord().hashCode()))%numPartitions;
        }

    }
    public static class Tfidfgroup extends WritableComparator {

        @Override
        public int compare(WritableComparable a, WritableComparable b) {
            // TODO Auto-generated method stub
            Docword lhs=(Docword)a;
            Docword rhs=(Docword)b;
            return lhs.getWord().compareTo(rhs.getWord());//按照wordname进行合并。
        }

        public Tfidfgroup() {
            super(Docword.class,true);
            // TODO Auto-generated constructor stub
        }

    }
    public int run(String []args) throws Exception{
        Path in1=new Path("data/wordcount");//输入文件路径
        Path out1=new Path("output/tfidf-tf");//Job1的tf结果输出路径
        Path out2=new Path("output/tfidf-idf");//Job2的idf结果输出路径
        Path out3=new Path("output/tfidf-tfidf");//Job3的tfidf结果输出路径
        URI skipstr=new URI("data/skipstr");//Job1的待删除字符串路径

        //============Job1配置============
        Job job1=Job.getInstance(getConf(), "tfidf-tf");
        Configuration conf1=job1.getConfiguration();
        job1.setJarByClass(getClass());

        FileInputFormat.setInputPaths(job1, in1);

        out1.getFileSystem(conf1).delete(out1, true);
        FileOutputFormat.setOutputPath(job1, out1);
        conf1.set(TextOutputFormat.SEPERATOR, DELIMITER);

        job1.setInputFormatClass(TextInputFormat.class);
        job1.setOutputFormatClass(TextOutputFormat.class);

        job1.setMapperClass(Tfmapper.class);
        job1.setMapOutputKeyClass(Docword.class);
        job1.setMapOutputValueClass(IntWritable.class);
        job1.setCombinerClass(Tfcombiner.class);
        job1.setReducerClass(Tfreducer.class);
        job1.setOutputKeyClass(Docword.class);
        job1.setOutputValueClass(DoubleWritable.class);
        job1.setPartitionerClass(Tfpartition.class);
        job1.addCacheFile(skipstr);
        job1.setNumReduceTasks(3);

        if(job1.waitForCompletion(true)==false)
            return 1;

        //============Job2配置============
        Job job2=Job.getInstance(getConf(), "tfidf-idf");
        Configuration conf2=job2.getConfiguration();
        job2.setJarByClass(getClass());

        FileInputFormat.setInputPaths(job2, out1);
        out2.getFileSystem(conf2).delete(out2, true);

        //利用HDFS API接口，获得输入文件总数，并通过变量filesum传递给Job2。
        FileSystem hdfs=FileSystem.get(conf2);
        FileStatus p[]=hdfs.listStatus(in1);
        conf2.set("filesnum", Integer.toString(p.length));

        FileOutputFormat.setOutputPath(job2, out2);
        conf2.set(TextOutputFormat.SEPERATOR, DELIMITER);
        job2.setInputFormatClass(TextInputFormat.class);
        job2.setOutputFormatClass(TextOutputFormat.class);

        job2.setSortComparatorClass(Idfsort.class);
        job2.setGroupingComparatorClass(Idfsort.class);

        job2.setMapperClass(Idfmapper.class);
        job2.setMapOutputKeyClass(Docword.class);
        job2.setMapOutputValueClass(IntWritable.class);
        job2.setCombinerClass(Idfcombiner.class);
        job2.setReducerClass(Idfreducer.class);
        job2.setOutputKeyClass(Text.class);
        job2.setOutputValueClass(DoubleWritable.class);
        job2.setNumReduceTasks(3);
        job2.setPartitionerClass(Idfpartition.class);

        if(job2.waitForCompletion(true)==false)
            return 1;

        //============Job3配置============
        Job job3=Job.getInstance(getConf(), "tfidf-tfidf");
        Configuration conf3=job3.getConfiguration();
        job3.setJarByClass(getClass());

        out3.getFileSystem(conf3).delete(out3, true);
        FileOutputFormat.setOutputPath(job3, out3);
        conf3.set(TextOutputFormat.SEPERATOR, DELIMITER);
        job3.setOutputFormatClass(TextOutputFormat.class);

        //利用MultipleInputs，配置Map分别读取Job1、Job2的输出文件
        MultipleInputs.addInputPath(job3, out1, TextInputFormat.class, Tfidf_tfmapper.class);
        MultipleInputs.addInputPath(job3, out2, TextInputFormat.class, Tfidf_idfmapper.class);

        job3.setMapOutputKeyClass(Docword.class);
        job3.setMapOutputValueClass(Docword.class);

        job3.setReducerClass(Tfidfreducer.class);
        job3.setOutputKeyClass(Text.class);
        job3.setOutputValueClass(DoubleWritable.class);
        job3.setNumReduceTasks(3);
        job3.setSortComparatorClass(Tfidfsort.class);
        job3.setGroupingComparatorClass(Tfidfgroup.class);
        job3.setPartitionerClass(Tfidfpartition.class);
        return job3.waitForCompletion(true)?0:1;

    }
    public static void main(String []args) throws Exception{
        int result=0;
        try{
            result=ToolRunner.run(new Configuration(), new Tfidf(), args);
        }catch(Exception e){
            e.printStackTrace();
        }
        System.exit(result);
    }

}</span>