数据去重

数据去重

public static class Map extends Mapper<Object, Text, Text, Text>{
//map将输入中的value复制到输出数据的key上,并直接输出
private static Text line = new Text();

	public void map(Object key,Text value,Context context)throws IOException, InterruptedException{
		line = value;
		context.write(line, new Text(""));
	}		
}



//reduce将输入中的key复制到输出数据的key上，并直接输出
public static class Reduce extends Reducer<Text, Text, Text, Text> {

	//实现reduce函数
	protected void reduce(Text key, Iterable<Text> values,Context context) throws IOException, InterruptedException {
		context.write(key, new Text(""));
	}
}