Hadoop代码

最新推荐文章于 2025-03-16 17:18:06 发布

Yan�.691

最新推荐文章于 2025-03-16 17:18:06 发布

阅读量862

点赞数 1

文章标签： hadoop 大数据分布式

本文链接：https://blog.youkuaiyun.com/qq_62702752/article/details/131501503

版权

本文详细介绍了Hadoop MapReduce在partflow、partprofit、seriaflow和sortpriofit四个部分的应用。包括Flow、PartProfit、SeriaFlow及SortProfit的Driver、Mapper、Partitioner和Reducer等关键组件的源代码解析，深入理解大数据处理流程。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

PartProfitDriver.java

PartProfitMapper.java

PartProfitPartitioner.java

PartProfitReduce.java

SortPriofitDriver.java

riofitMapper.java

SortPriofitReduce.java

一、partflow

Flow.java

public class Flow implements Writable{
	private String phone = "";
	private String addr = "";
	private String name = "";
	private int flow;
	public String getPhone() {
		return phone;
	}
	public void setPhone(String phone) {
		this.phone = phone;
	}
	public String getAddr() {
		return addr;
	}
	public void setAddr(String addr) {
		this.addr = addr;
	}
	public String getName() {
		return name;
	}
	public void setName(String name) {
		this.name = name;
	}
	public int getFlow() {
		return flow;
	}
	public void setFlow(int flow) {
		this.flow = flow;
	}
	@Override
	public void write(DataOutput out) throws IOException {
		out.writeUTF(phone);
		out.writeUTF(addr);
		out.writeUTF(name);
		out.writeInt(flow);
	}
	@Override
	public void readFields(DataInput in) throws IOException {
		this.phone = in.readUTF();
		this.addr = in.readUTF();
		this.name = in.readUTF();
		this.flow = in.readInt();
	}
}

FlowDrive,.java


public class FlowDriver {
	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		Configuration conf = new Configuration();
		Job job = Job.getInstance(conf);
		job.setJarByClass(FlowDriver.class);
		job.setMapperClass(FlowMapper.class);
		job.setReducerClass(FlowReducer.class);
		job.setPartitionerClass(FlowPartitioner.class);
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(Flow.class);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(IntWritable.class);
		FileInputFormat.addInputPath(job, new Path("hdfs://hadoop01:9000/txt/flow.txt"));
		FileOutputFormat.setOutputPath(job, new Path("hdfs://hadoop01:9000/result/partflow"));
		job.waitForCompletion(true);
	}
}

FlowMapper.java

public class FlowMapper extends Mapper<LongWritable, Text, Text, Flow>{
	@Override
	protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, Flow>.Context context)
			throws IOException, InterruptedException {
		String[] arr = value.toString().split(" ");
		Flow f = new Flow();
		f.setPhone(arr[0]);
		f.setAddr(arr[1]);
		f.setName(arr[2]);
		f.setFlow(Integer.parseInt(arr[3]));
		context.write(new Text(f.getName()), f);
	}
}

FlowPartitioner.java

public class FlowPartitioner extends Partitioner<Text, Flow>{
	@Override
	public int getPartition(Text text, Flow flow, int numPartitions) {
		String addr = flow.getAddr();
		if(addr.equals("bj")){
			return 0;
		} else if(addr.equals("sh")){  //判断地址是否是上海
			return 1;
		} else {
			return 2;  //判断地址是否是其它
		}
	}
}

FlowReducer.java

public class FlowReducer extends Reducer<Text, Flow, Text,

最低0.47元/天解锁文章