用hadoop统计A call B的次数

最新推荐文章于 2024-07-29 21:51:36 发布

转载最新推荐文章于 2024-07-29 21:51:36 发布 · 88 阅读

0 ·

CC 4.0 BY-SA版权

原文链接：https://my.oschina.net/u/242736/blog/83252

文章标签：

#大数据 #python #java

本文介绍了一个使用Hadoop MapReduce实现的Java程序，该程序用于处理电话计费记录，统计不同计费期的呼叫次数。通过对输入数据进行映射和归约操作，程序能够有效地汇总每个计费期的呼叫总数。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

2019独角兽企业重金招聘Python工程师标准>>>

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;



public class CallCount {
	public static class BILLING_TERM_Mapper extends 
	    Mapper<Object, Text, Text, IntWritable> {

    private final static IntWritable one = new IntWritable(1);
    private Text word = new Text();

    public void map(Object key, Text value, Context context)
		throws IOException, InterruptedException {

	String line = value.toString();
	String details[] = line.split("	");
	String BILLING = details[0];
	String TERM = details[1];
	String BILLING_TERM_String = BILLING + "-" + TERM;
	word.set(BILLING_TERM_String);

	if(!"null".equals(BILLING) && !"".equals(BILLING)){
		context.write(word, one);
	}
  }
 }

   public static class BILLING_TERM_Reducer extends
	  Reducer<Text, IntWritable, Text, IntWritable> {
   private IntWritable result = new IntWritable();

   public void reduce(Text key, Iterable<IntWritable> values,
		  Context context) throws IOException, InterruptedException {
	  
	  int sum = 0;
	  for (IntWritable val : values) {
		  sum += val.get();
	  }
	  result.set(sum);
	  context.write(key, result);
  }
 }
   public static void main(String[] args) throws Exception{
		Configuration conf = new Configuration();
		String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
		if (otherArgs.length != 2){
			System.err.println("Usage: callcount ");
			System.exit(2);
		}
		/** Create A Job, Give A name, Tracking It**/
		Job job = new Job(conf, "Call Count");
		job.setJarByClass(CallCount.class);
		job.setMapperClass(BILLING_TERM_Mapper.class);
		job.setCombinerClass(BILLING_TERM_Reducer.class);
		job.setReducerClass(BILLING_TERM_Reducer.class);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(IntWritable.class);
		
		FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
		FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
		
		System.exit(job.waitForCompletion(true) ? 0 : 1);
	}
}