Mapreduce《案例之数据去重复》

Mapreduce《案例之数据去重复》

源数据：

a.txt内容：

2012-3-1 b

2012-3-2 a

2012-3-3 b

2012-3-4 d

2012-3-5 a

2012-3-6 c

2012-3-7 d

2012-3-3 c

 

b.txt内容：

2012-3-1 a

2012-3-2 b

2012-3-3 c

2012-3-4 d

2012-3-5 a

2012-3-6 b

2012-3-7 c

2012-3-3 c

 

输出结果：

2012-3-1 a

2012-3-1 b

2012-3-2 a

2012-3-2 b

2012-3-3 b

2012-3-3 c

2012-3-4 d

2012-3-5 a

2012-3-6 b

2012-3-6 c

2012-3-7 c

2012-3-7 d

 

 

 

 

//===================================JAVA CODE=========================

package gq;

 

import java.io.IOException;

 

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

/**

 * 数据去重复

 * @author tallqi

 *

 */

public class Dereplication {

 

public static class Map extends Mapper<Object, Text, Text, Text>{

private static Text line = new Text();

public void map(Object key,Text value,Context context) throws IOException, InterruptedException{

line =value;

System.out.println("Map:"+value);

context.write(line, new Text(""));

}

 

}

public static class Reduce extends Reducer<Text, Text, Text, Text>{

public void reduce(Text key,Iterable<Text> value,Context context) throws IOException, InterruptedException{

System.out.println("Reduce:"+key);

context.write(key, new Text(""));;

}

 

}

public static void main(String[] args) throws Exception{

 

      Configuration conf = new Configuration();

 

Job job = new Job(conf,"Dereplication");

job.setJarByClass(Dereplication.class);

 

//设置Map、Combine和Reduce处理类

job.setMapperClass(Map.class);

// job.setCombinerClass(Reduce.class);

job.setReducerClass(Reduce.class);

   

 

//输出Key，value的类型

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(Text.class);

 

//数据源地址，数据输出地址

FileInputFormat.addInputPath(job, new Path("hdfs://h0:9000/user/tallqi/in/input"));

FileOutputFormat.setOutputPath(job, new Path("hdfs://h0:9000/user/tallqi/in/output"));

System.exit(job.waitForCompletion(true)?0:1);

 

}

 

}