Hadoop之MapReduce入门经典案列三：ip去重

最新推荐文章于 2024-11-16 14:46:21 发布

静静七分醉

最新推荐文章于 2024-11-16 14:46:21 发布

阅读量1.2k

点赞数 1

CC 4.0 BY-SA版权

分类专栏： hadoop hadoop

本文链接：https://blog.youkuaiyun.com/m0_37657725/article/details/94403303

hadoop 同时被 2 个专栏收录

4 篇文章

订阅专栏

hadoop

4 篇文章

订阅专栏

本文介绍了一个使用Hadoop MapReduce处理大量IP地址数据的任务。通过自定义的Mapper和Reducer类，实现了对IP地址的计数和统计。代码示例展示了如何设置MapReduce作业，指定输入和输出路径，并运行作业。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

文件内容：

10.9.80.16
10.9.132.111
10.9.152.65
10.9.21.119
10.9.132.111
10.9.130.83
10.9.80.16
10.9.152.65
10.9.162.13
10.9.162.133
10.9.21.119
10.9.132.111
10.9.130.83
10.9.80.133
10.9.129.86
10.9.132.111
10.9.152.65
10.9.21.119
10.9.132.111
10.9.130.83
10.9.80.16
10.9.152.66
10.9.80.16
10.9.152.65
10.9.21.119
10.9.132.111
10.9.130.83
10.9.80.16
10.9.152.65
10.9.162.13
10.9.132.111
10.9.130.83
10.9.21.119
10.9.132.111
10.9.130.8
10.9.80.16
10.9.152.65
10.9.21.119
10.9.132.111
10.9.130.83
10.9.80.16
10.9.152.66
10.9.80.16
10.9.152.65
10.9.21.119
10.9.80.16
10.9.152.65
10.9.162.13
10.9.162.133
10.9.21.119
10.9.132.1
10.9.162.133
10.9.21.119
10.9.132.111
10.9.130.8
10.9.80.16
10.9.152.33
10.9.152.66
10.9.80.16
10.9.152.65
10.9.21.119
10.9.80.16
10.9.152.65
10.9.162.13
10.9.162.133
10.9.152.65
10.9.162.13
10.9.162.133
10.9.21.119
10.9.132.1
10.9.162.133
10.9.21.119
10.9.132.111
10.9.130.8
10.9.132.111
10.9.130.8
10.9.80.16
10.9.152.33
10.9.152.66
10.9.80.16
10.9.152.65

代码实现：

1.Mapper

package com.lj.ip3;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class ipMapper extends Mapper<LongWritable, Text, Text, NullWritable> {

   public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
       context.write(value, NullWritable.get());

   }

}

2.Reduce

package com.lj.ip3;

import java.io.IOException;

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class IpReduce extends Reducer<Text, NullWritable, Text, NullWritable> {

   public void reduce(Text key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {

       context.write(key, NullWritable.get());
   }

}

3.Driver

package com.lj.ip3;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class IpDriver {

   public static void main(String[] args) throws Exception {
       Configuration conf = new Configuration();
       Job job = Job.getInstance(conf, "JobName");
       job.setJarByClass(com.lj.ip3.IpDriver.class);
       // TODO: specify a mapper
       job.setMapperClass(ipMapper.class);
       // TODO: specify a reducer
       job.setReducerClass(IpReduce.class);

       // TODO: specify output types
       job.setOutputKeyClass(Text.class);
       job.setOutputValueClass(NullWritable.class);

       // TODO: specify input and output DIRECTORIES (not files)
       FileInputFormat.setInputPaths(job, new Path("hdfs://lj02:9000/txt/ip.txt"));
       FileOutputFormat.setOutputPath(job, new Path("hdfs://lj02:9000/3ip"));

       if (!job.waitForCompletion(true))
           return;
   }

}

结果：