MapReduce的运行

Map Reduce操作步骤：

1.启动hadoop集群 ./startall.sh
2.进入jar包所在文件夹Demo
3.在hadoop新建 input hadoop fs -mkdir -p /user/input
4.把分析文件put在input中 hadoop fs -put demo.txt /user/input
5. Hadoop jar TranConfig.jar com.job.Tranconfig config.properties
/ user/input/demo.txt /user/output/res
6.Hadoop jar jar包名 包.类.方法 /user/input/分析文件/user/output/Pv

Hadoop实例

PvUvJob

package com.map;

import java.io.IOException;
import java.util.Map;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;


public class PvUvMap extends Mapper<LongWritable, Text, Text, IntWritable>{
    IntWritable one = new IntWritable(1);   
    protected void map(LongWritable key, Text value,Context context)
            throws IOException, InterruptedException {
        String []res=value.toString().split("\t");
        String url=res[28];
        context.write(new Text(url),one);
                super.map(key, value, context);
    }


}

PvUvmap

package com.map;

import java.io.IOException;
import java.util.Map;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;


public class PvUvMap extends Mapper<LongWritable, Text, Text, IntWritable>{
    IntWritable one = new IntWritable(1);   
    protected void map(LongWritable key, Text value,Context context)
            throws IOException, InterruptedException {
        String []res=value.toString().split("\t");
        String url=res[28];
        context.write(new Text(url),one);
                super.map(key, value, context);
    }


}

PvUvReduce

package com.reduce;

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class PvUvReduce extends Reducer<Text,IntWritable,Text,NullWritable> {
    protected void reduce(Text key,Iterable<IntWritable>values,Context context)
    throws IOException,InterruptedException{
        int sum = 0;
        for(IntWritable value:values){
            sum+=value.get();
        }
        context.write(new Text(key.toString()+":"+sum), NullWritable.get());
    }



}