MapReduce其他功能

最新推荐文章于 2022-09-25 15:51:14 发布

原创

最新推荐文章于 2022-09-25 15:51:14 发布 · 417 阅读

1 ·

CC 4.0 BY-SA版权

本文深入探讨MapReduce的功能，包括：序列化接口Writable的使用，如在流量统计和成绩统计中的应用；自定义Partitioner实现特定分区策略，例如按城市和月份进行数据分发；以及Combiner的运用，通过预处理减少Reduce任务的负载。同时，还提及了Map阶段结束后基于键的默认排序机制和如何自定义Comparable接口以调整排序规则。

序列化 – Writable
序列化/反序列化机制：当自定义了一个类之后,如果想要产生的对象在hadoop中进行传输,那么需要这个类实现Hadoop提供的Writable的接口只需要将按序写出并进行序列化/反序列化
Writable案例1:统计每一个人产生的总流量
文件:flow.txt

13877779999 bj zs 2145
13766668888 sh ls 1028
13766668888 sh ls 9987
13877779999 bj zs 5678
13544445555 sz ww 10577
13877779999 sh zs 2145
13766668888 sh ls 9987

pojo

public class Flow implements Writable{

	private String phone;
	private String city;
	private String name;
	private int flow;

	public String getPhone() {
		return phone;
	}

	public void setPhone(String phone) {
		this.phone = phone;
	}

	public String getCity() {
		return city;
	}

	public void setCity(String city) {
		this.city = city;
	}

	public String getName() {
		return name;
	}

	public void setName(String name) {
		this.name = name;
	}

	public int getFlow() {
		return flow;
	}

	public void setFlow(int flow) {
		this.flow = flow;
	}

	// 反序列化
	@Override
	public void readFields(DataInput in) throws IOException {
		// 按照序列化的顺序一个一个将数据读取出来
		this.phone = in.readUTF();
		this.city = in.readUTF();
		this.name = in.readUTF();
		this.flow = in.readInt();
	}

	// 序列化
	@Override
	public void write(DataOutput out) throws IOException {
		// 按照顺序将属性一个一个的写出即可
		out.writeUTF(phone);
		out.writeUTF(city);
		out.writeUTF(name);
		out.writeInt(flow);
	}
}

Mapper

public class FlowMapper extends Mapper<LongWritable, Text, Text, Flow> {

	public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

		String line = value.toString();
		
		String[] arr = line.split(" ");
		
		Flow f = new Flow();
		f.setPhone(arr[0]);
		f.setCity(arr[1]);
		f.setName(arr[2]);
		f.setFlow(Integer.parseInt(arr[3]));
		
		context.write(new Text(f.getPhone()), f);
	}
}

Reducer

public class FlowReducer extends Reducer<Text, Flow, Text, IntWritable> {

	public void reduce(Text key, Iterable<Flow> values, Context context) throws IOException, InterruptedException {
		
		int sum = 0;
		String name = null;
		
		for (Flow val : values) {
			name = val.getName();
			sum += val.getFlow();
		}
		context.write(new Text(key.toString() + " " + name), new IntWritable(sum));
	}
}

Driver

public class FlowDriver {

	public static void main(String[] args) throws Exception {
		Configurat

最低0.47元/天解锁文章

新学期VIP享超值加赠