DataCount Partitioner分区

最新推荐文章于 2019-11-14 17:14:18 发布

元永真

最新推荐文章于 2019-11-14 17:14:18 发布

阅读量338

点赞数

CC 4.0 BY-SA版权

本文链接：https://blog.youkuaiyun.com/weixin_37855495/article/details/62224058

hadoop学习总结专栏收录该内容

32 篇文章

订阅专栏

本文介绍了一个使用Hadoop MapReduce进行数据流量统计的应用案例。该案例通过Mapper读取每条记录，提取手机号码及上传下载流量，并在Reducer中汇总每个手机号码对应的总流量。

package cn.itcast.hadoop.mr.dc;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Partitioner;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class DataCount {

   public static void main(String[] args) throws Exception {
       Configuration conf = new Configuration();

       Job job = Job.getInstance(conf);

       job.setJarByClass(DataCount.class);

       job.setMapperClass(DCMapper.class);
       job.setMapOutputKeyClass(Text.class);
       job.setMapOutputValueClass(DataInfo.class);

       job.setReducerClass(DCReducer.class);
       job.setOutputKeyClass(Text.class);
       job.setOutputValueClass(DataInfo.class);

       FileInputFormat.setInputPaths(job, new Path(args[0]));

       FileOutputFormat.setOutputPath(job, new Path(args[1]));

       job.setPartitionerClass(DCPartitioner.class);//设置分区

       job.setNumReduceTasks(Integer.parseInt(args[2]));//设置reduce 数量


       job.waitForCompletion(true);

   }
   //Map
   public static class DCMapper extends Mapper<LongWritable, Text, Text, DataInfo>{

       private Text k = new Text();

       @Override
       protected void map(LongWritable key, Text value,
               Mapper<LongWritable, Text, Text, DataInfo>.Context context)
               throws IOException, InterruptedException {
           String line = value.toString();
           String[] fields = line.split("\t");
           String tel = fields[1];
           long up = Long.parseLong(fields[8]);
           long down = Long.parseLong(fields[9]);
           DataInfo dataInfo = new DataInfo(tel,up,down);
           k.set(tel);
           context.write(k, dataInfo);

       }

   }
   public static class DCReducer extends Reducer<Text, DataInfo, Text, DataInfo>{

       @Override
       protected void reduce(Text key, Iterable<DataInfo> values,
               Reducer<Text, DataInfo, Text, DataInfo>.Context context)
               throws IOException, InterruptedException {
           long up_sum = 0;
           long down_sum = 0;
           for(DataInfo d : values){
               up_sum += d.getUpPayLoad();
               down_sum += d.getDownPayLoad();
           }
           DataInfo dataInfo = new DataInfo("",up_sum,down_sum);

           context.write(key, dataInfo);
       }

   }
   //分区在map执行后 reducer 执行前执行
   public static class DCPartitioner extends Partitioner<Text, DataInfo>{

       private static Map<String,Integer> provider = new HashMap<String,Integer>();
       //可以加载数据库    创造假数据   1 中国移动 2 中国联通 3 中国电信
       static{
           provider.put("138", 1);
           provider.put("139", 1);
           provider.put("152", 2);
           provider.put("153", 2);
           provider.put("182", 3);
           provider.put("183", 3);
       }
       @Override
       public int getPartition(Text key, DataInfo value, int numPartitions) {
           //向数据库或配置信息读写
           String tel_sub = key.toString().substring(0,3);
           Integer count = provider.get(tel_sub);
           if(count == null){
               count = 0;
           }
           return count;
       }

   }

}

package cn.itcast.hadoop.mr.dc;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.io.Writable;

public class DataInfo implements Writable{

   private String tel;
   private long upPayLoad;
   private long downPayLoad;
   private long totalPayLoad;

   public DataInfo(){}

   public DataInfo(String tel, long upPayLoad, long downPayLoad) {
       this.tel = tel;
       this.upPayLoad = upPayLoad;
       this.downPayLoad = downPayLoad;
       this.totalPayLoad = upPayLoad + downPayLoad;
   }

   @Override
   public void write(DataOutput out) throws IOException {
       out.writeUTF(tel);
       out.writeLong(upPayLoad);
       out.writeLong(downPayLoad);
       out.writeLong(totalPayLoad);
   }

   @Override
   public void readFields(DataInput in) throws IOException {
       this.tel = in.readUTF();
       this.upPayLoad = in.readLong();
       this.downPayLoad = in.readLong();
       this.totalPayLoad = in.readLong();

   }

   @Override
   public String toString() {
       return upPayLoad + "\t" + downPayLoad + "\t" + totalPayLoad;
   }

   public String getTel() {
       return tel;
   }

   public void setTel(String tel) {
       this.tel = tel;
   }

   public long getUpPayLoad() {
       return upPayLoad;
   }

   public void setUpPayLoad(long upPayLoad) {
       this.upPayLoad = upPayLoad;
   }

   public long getDownPayLoad() {
       return downPayLoad;
   }

   public void setDownPayLoad(long downPayLoad) {
       this.downPayLoad = downPayLoad;
   }

   public long getTotalPayLoad() {
       return totalPayLoad;
   }

   public void setTotalPayLoad(long totalPayLoad) {
       this.totalPayLoad = totalPayLoad;
   }

}