PageRank Hadoop MapReduce

最新推荐文章于 2025-05-19 14:42:48 发布

wanhf11

最新推荐文章于 2025-05-19 14:42:48 发布

阅读量574

点赞数

CC 4.0 BY-SA版权

分类专栏：大数据

本文链接：https://blog.youkuaiyun.com/qq_17612199/article/details/51404656

大数据专栏收录该内容

21 篇文章

订阅专栏

本文介绍了一种使用MapReduce框架实现的PageRank算法，该算法基于Hadoop平台进行网页排名计算。通过迭代处理链接关系文件，逐步调整每个网页的重要性权重。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

这里写图片描述

links.txt

链接关系

A B C D
B A D
C C
D B C

part-r-00000 初试概率分布向量

a=0.8

A a 0.25
B a 0.25
C a 0.25
D a 0.25

PageRankMapReduce

package org.bigdata.pagerank;

import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.bigdata.util.HadoopCfg;
import org.bigdata.util.HadoopUtil;

/**
 * PageRank
 * 
 * @author wwhhf
 * 
 */
public class PageRankMapReduce {

    private final static String JOB_NAME = "PageRank";
    private static String LINKS = "links";

    private static Map<String, Double> rand = new HashMap<String, Double>();

    private static final double a = 0.8;

    public static void initRand(String pathin, String filename)
            throws IOException {
        List<String> lines = HadoopUtil.lslFile(pathin, filename);
        for (String line : lines) {
            String terms[] = line.toString().split("\t");
            rand.put(terms[0], Double.valueOf(terms[1]));
        }
    }

    private static class PageRankMapper extends
            Mapper<Text, Text, Text, DoubleWritable> {

        @Override
        protected void map(Text key, Text value, Context context)
                throws IOException, InterruptedException {
            String filename = key.toString();
            if (filename.startsWith(LINKS)) {
                String dests[] = value.toString().split(" ");
                double e = rand.get(dests[0]);
                for (int i = 0, len = dests.length; i < len; i++) {
                    String dest = dests[i];
                    if (i == 0) {
                        context.write(new Text(dest), new DoubleWritable(0.0));
                    } else {
                        context.write(new Text(dest), new DoubleWritable(e
                                / (len - 1)));
                    }
                }
            }
        }

    }

    private static class PageRankReducer extends
            Reducer<Text, DoubleWritable, Text, DoubleWritable> {

        @Override
        protected void reduce(Text key, Iterable<DoubleWritable> values,
                Context context) throws IOException, InterruptedException {
            Double sum = 0.0;
            for (DoubleWritable value : values) {
                sum += value.get();
            }
            double e = rand.get(key.toString());
            context.write(key, new DoubleWritable(a * sum + (1 - a) * e));
        }

    }

    public static void solve(String linksin, String pathin, String pathout)
            throws ClassNotFoundException, InterruptedException {
        try {
            Configuration cfg = HadoopCfg.getConfiguration();
            Job job = Job.getInstance(cfg);
            job.setJobName(JOB_NAME);
            job.setJarByClass(PageRankMapReduce.class);
            job.setInputFormatClass(FileNameInputFormat.class);

            // mapper
            job.setMapperClass(PageRankMapper.class);
            job.setMapOutputKeyClass(Text.class);
            job.setMapOutputValueClass(DoubleWritable.class);

            // reducer
            job.setReducerClass(PageRankReducer.class);
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(DoubleWritable.class);

            FileInputFormat.addInputPath(job, new Path(pathin));
            FileInputFormat.addInputPath(job, new Path(linksin));
            FileOutputFormat.setOutputPath(job, new Path(pathout));

            job.waitForCompletion(true);

        } catch (IllegalStateException | IllegalArgumentException | IOException e) {
            e.printStackTrace();
        }
    }

    public static void main(String[] args) throws ClassNotFoundException,
            InterruptedException, IOException {
        String path = "/pagerank";
        String links_pathin = "/pagerank_links";
        String filename = "part-r-00000";
        String tmp_pathin = path;
        for (int i = 1; i <= 5; i++) {
            initRand(tmp_pathin, filename);
            String tmp_pathout = path + i;
            System.out.println(links_pathin + " " + tmp_pathin + " "
                    + tmp_pathout);
            solve(links_pathin, tmp_pathin, tmp_pathout);
            tmp_pathin = tmp_pathout;
        }

    }
}