关于hadoop序列化自己的类的 做统计的实现

POM.xml 文件的依赖：

继承Mapper 实处自己的业务
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.7.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.7.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.7.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
<version>2.7.0</version>
</dependency>


</dependencies>

import java.io.DataInput;

import java.io.DataOutput;
import java.io.IOException;


import org.apache.hadoop.io.Writable;

/**

*此处应该的hadoop的序列类 Writable

*/

public class TrafficStatistics implements Writable {


long upPackNum;
long downPackNum;
long upPayLoad;
long downPayLoad;


public void write(DataOutput out) throws IOException {
out.writeLong(upPackNum);
out.writeLong(downPackNum);
out.writeLong(upPayLoad);
out.writeLong(downPackNum);

}


public void readFields(DataInput in) throws IOException {
this.upPackNum=in.readLong();
this.downPackNum = in.readLong();
this.upPayLoad = in.readLong();
this.downPayLoad = in.readLong();
}


public void set(String upPackNum, String downPackNum, String upPayLoad, String downPayLoad) {
this.upPackNum = Long.parseLong(upPackNum);
this.downPackNum = Long.parseLong(downPackNum);
this.upPayLoad = Long.parseLong(upPayLoad);
this.downPayLoad = Long.parseLong(downPayLoad);
}
public void set(long upPackNum, long downPackNum, long upPayLoad, long downPayLoad) {
this.upPackNum = upPackNum;
this.downPackNum = downPackNum;
this.upPayLoad = upPayLoad;
this.downPayLoad = downPayLoad;
}
@Override
public String toString() {
return this.upPackNum+"\t"+this.downPackNum+"\t"+this.upPayLoad+"\t"+this.downPayLoad;
}

}

继承Mapper 实处自己的业务

import java.io.IOException;


import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;


public class TrafficMapper extends Mapper<LongWritable, Text, Text, TrafficStatistics> {


Text K2 = new Text();
TrafficStatistics V2 = new TrafficStatistics();


@Override
protected void map(LongWritable key, Text value,
Mapper<LongWritable, Text, Text, TrafficStatistics>.Context context)
throws IOException, InterruptedException {


String[] splits = value.toString().split("\t");
K2.set(splits[1]);
V2.set(splits[6], splits[7], splits[9], splits[9]);
context.write(K2, V2);


}


}

继承Reduce 实处自己的统计业务

继承Mapper 实处自己的业务
import java.io.IOException;


import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;


public class TrafficReducer extends Reducer<Text, TrafficStatistics, Text, TrafficStatistics> {
TrafficStatistics V3 = new TrafficStatistics();
@Override
protected void reduce(Text key, Iterable<TrafficStatistics> values,
Reducer<Text, TrafficStatistics, Text, TrafficStatistics>.Context context)
throws IOException, InterruptedException {


long upPackSum=0L;
long downPackSum=0L;
long upPayLoadSum=0L;
long downPayLoadSum=0L;

for (TrafficStatistics trafficStatistics : values) {
upPackSum+= trafficStatistics.upPackNum;
downPackSum+= trafficStatistics.downPackNum;
upPayLoadSum+= trafficStatistics.upPayLoad;
downPayLoadSum+= trafficStatistics.downPayLoad;
V3.set(upPackSum, downPackSum, upPayLoadSum, downPayLoadSum);
context.write(key, V3);
}

}


}

//主类：

继承Mapper 实处自己的业务
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


public class TrafficJob {


public static void main(String[] args) throws Exception {

Job job = Job.getInstance(new Configuration(), TrafficJob.class.getSimpleName());

job.setJarByClass(TrafficJob.class);
//此处设置自己的Mapper类
job.setMapperClass(TrafficMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(TrafficStatistics.class);

//此处设置自己的Reduce子类
job.setReducerClass(TrafficReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(TrafficStatistics.class);

FileInputFormat.setInputPaths(job, args[0]);
FileOutputFormat.setOutputPath(job, new Path(args[1]));

job.waitForCompletion(true);
}


}

继承Mapper 实处自己的业务