Hadoop MapReduce入门程序
1.数据准备
1363157985066 13726230503 00-FD-07-A4-72-B8:CMCC 120.196.100.82 i02.c.aliimg.com 24 27 2481 24681 200
1363157995052 13826544101 5C-0E-8B-C7-F1-E0:CMCC 120.197.40.4 4 0 264 0 200
1363157991076 13926435656 20-10-7A-28-CC-0A:CMCC 120.196.100.99 2 4 132 1512 200
1363154400022 13926251106 5C-0E-8B-8B-B1-50:CMCC 120.197.40.4 4 0 240 0 200
1363157993044 18211575961 94-71-AC-CD-E6-18:CMCC-EASY 120.196.100.99 iface.qiyi.com 视频网站 15 12 1527 2106 200
1363157995074 84138413 5C-0E-8B-8C-E8-20:7DaysInn 120.197.40.4 122.72.52.12 20 16 4116 1432 200
1363157993055 13560439658 C4-17-FE-BA-DE-D9:CMCC 120.196.100.99 18 15 1116 954 200
1363157995033 15920133257 5C-0E-8B-C7-BA-20:CMCC 120.197.40.4 sug.so.360.cn 信息安全 20 20 3156 2936 200
1363157983019 13719199419 68-A1-B7-03-07-B1:CMCC-EASY 120.196.100.82 4 0 240 0 200
1363157984041 13660577991 5C-0E-8B-92-5C-20:CMCC-EASY 120.197.40.4 s19.cnzz.com 站点统计 24 9 6960 690 200
1363157973098 15013685858 5C-0E-8B-C7-F7-90:CMCC 120.197.40.4 rank.ie.sogou.com 搜索引擎 28 27 3659 3538 200
1363157986029 15989002119 E8-99-C4-4E-93-E0:CMCC-EASY 120.196.100.99 www.umeng.com 站点统计 3 3 1938 180 200
1363157992093 13560439658 C4-17-FE-BA-DE-D9:CMCC 120.196.100.99 15 9 918 4938 200
1363157986041 13480253104 5C-0E-8B-C7-FC-80:CMCC-EASY 120.197.40.4 3 3 180 180 200
1363157984040 13602846565 5C-0E-8B-8B-B6-00:CMCC 120.197.40.4 2052.flash2-http.qq.com 综合门户 15 12 1938 2910 200
1363157995093 13922314466 00-FD-07-A2-EC-BA:CMCC 120.196.100.82 img.qfc.cn 12 12 3008 3720 200
1363157982040 13502468823 5C-0A-5B-6A-0B-D4:CMCC-EASY 120.196.100.99 y0.ifengimg.com 综合门户 57 102 7335 110349 200
1363157986072 18320173382 84-25-DB-4F-10-1A:CMCC-EASY 120.196.100.99 input.shouji.sogou.com 搜索引擎 21 18 9531 2412 200
1363157990043 13925057413 00-1F-64-E1-E6-9A:CMCC 120.196.100.55 t3.baidu.com 搜索引擎 69 63 11058 48243 200
1363157988072 13760778710 00-FD-07-A4-7B-08:CMCC 120.196.100.82 2 2 120 120 200
1363157985066 13726238888 00-FD-07-A4-72-B8:CMCC 120.196.100.82 i02.c.aliimg.com 24 27 2481 24681 200
1363157993055 13560436666 C4-17-FE-BA-DE-D9:CMCC 120.196.100.99 18 15 1116 954 200
1363157993055 13560436666 C4-17-FE-BA-DE-D9:CMCC 120.196.100.99 18 15 111s6 954 200
1363157993055
1363157985066 13726238888 00-FD-07-A4-72-B8:CMCC 120.196.100.82 i02.c.aliimg.com 24 27 2481 24681 200
2.环境准备
- Idea，使用eclipse也是可以的
- 创建maven工程
3. 设置maven相关配置
注意
–这里的maven home directory就是你自己的maven安装目录路径
–user settings file就是你自己针对maven的配置文件所在。这个文件中一般配置2个信息，一个是本地maven仓库路径，一个是国内maven仓库镜像地址.我自己的文件中2个修改之处如下，大家可以参考。
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<localRepository>D:\develop\maven_repository\repository</localRepository>
<mirror>
<id>alimaven</id>
<mirrorOf>central</mirrorOf>
<name>aliyun maven</name>
<url>http://maven.aliyun.com/nexus/content/repositories/central/</url>
</mirror>
–local repository 就是本地maven仓库的路径
ps：
从规范性来看，最好新建一个目录，把这个settings.xml文件和本地maven仓库都放在这个目录下，这样更方便进行配置
4. 拷贝以下文件，粘贴到新建的pom文件中
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>8</source>
<target>8</target>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>3.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>3.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-common</artifactId>
<version>3.2.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.alibaba/fastjson -->
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.68</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
<version>3.2.1</version>
</dependency>
</dependencies>
5.点击加载maven的第三方库按钮，耐心等待maven下载jar包结束
3.开始写java 代码
三个类，一个Driver，就是含有main方法的驱动类，一个Mapper类，一个Reeducer类。
很简单的代码
- map阶段，就是把一个输入的文件，逐行读取出来，然后使用空格进行分割，取电话号码为key，倒数第3和第2个字段相加的数作为value
- reduce阶段就是把map阶段输出的key和value进行聚合，就是把每个key对应所有的value进行累加，然后输出到本地文件夹中
3.1 Mapper类代码
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
/**
* @author hulc
* @slogan: just do it
* @date 2020/8/19 13:01
*/
public class PhoneMapper extends Mapper<LongWritable, Text, Text, DoubleWritable> {
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String line = value.toString();
String[] split = line.split("\\s+");
try {
// 电话号码 上下行流量
String phoneStr = split[1];
String upStr = split[split.length - 1- 2];
String downStr = split[split.length -1 -1];
double up = Double.parseDouble(upStr);
double down = Double.parseDouble(downStr);
context.write(new Text(phoneStr), new DoubleWritable(up+down));
}catch(Exception e) {
e.printStackTrace();
}
}
}
3.2 Reducer 类代码
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
/**
* @author hulc
* @slogan: just do it
* @date 2020/8/19 13:01
*/
public class PhoneReducer extends Reducer<Text, DoubleWritable, Text, DoubleWritable> {
@Override
protected void reduce(Text key, Iterable<DoubleWritable> values, Context context) throws IOException, InterruptedException {
double sum = 0;
for (DoubleWritable value : values) {
sum += value.get();
}
context.write(key, new DoubleWritable(sum));
}
}
3.3 Driver驱动类代码
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
/**
* @author hulc
* @slogan: just do it
* @date 2020/8/19 13:01
*/
public class PhoneDriver {
public static void main(String[] args) {
// 配置
Configuration entries = new Configuration();
// job
try {
Job niwa = Job.getInstance(entries, "niwa");
// 设置mapper reducer 的class
niwa.setMapperClass(PhoneMapper.class);
niwa.setReducerClass(PhoneReducer.class);
// 设置mapper和reduce输出的key value
niwa.setMapOutputKeyClass(Text.class);
niwa.setMapOutputValueClass(DoubleWritable.class);
niwa.setOutputKeyClass(Text.class);
niwa.setOutputValueClass(DoubleWritable.class);
// 设置输入数据，输出数据
FileInputFormat.setInputPaths(niwa, new Path("E:\\DOITLearning\\8.Hadoop\\mrdata\\flow\\input\\flow.log"));
FileOutputFormat.setOutputPath(niwa, new Path("E:\\DOITLearning\\8.Hadoop\\mrdata\\flow\\output_hulc01"));
// 设置reduce任务数
niwa.setNumReduceTasks(1);
// 启动任务
boolean b = niwa.waitForCompletion(true);
if(b){
System.out.println("ok le");
} else {
System.out.println(" failed");
}
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
}
}
温故知新，学而时习之。还是要时常复习，不用就容易忘记，敲代码的手感很重要。
不要懈怠，不要懈怠。
扫一扫
1272
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
