开发Mapreduce程序

最新推荐文章于 2025-01-26 06:00:00 发布

H_Hao

最新推荐文章于 2025-01-26 06:00:00 发布

阅读量298

点赞数

分类专栏：大数据学习

本文链接：https://blog.youkuaiyun.com/haoyuexihuai/article/details/52303824

版权

大数据学习专栏收录该内容

45 篇文章

订阅专栏

本文介绍如何使用Hadoop MapReduce框架来计算HDFS中指定文件的总行数。通过自定义Mapper和Reducer实现对文件每行进行计数，并汇总结果。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

将数据复制到HDFS中

启动Hadoop，复制loaddata1文件到“/home/wcbdd/Desktop/loaddata1”；

使用命令hadoop fs –copyFromLocal 源文件目标目录

打包完成后导出jar执行：

hadoop jar linecount.jar com.trendwise.java.linecount/home/wcbdd/Desktop/loaddata1 /home/wcbdd/workspace/BIGmb

完整代码

//This wraps the modules into a single programpackage com.trendwise.java; //Package name


package com.trendwise.java;

import java.io.IOException;

import java.util.Iterator;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapred.FileInputFormat;

import org.apache.hadoop.mapred.FileOutputFormat;

import org.apache.hadoop.mapred.JobClient;

import org.apache.hadoop.mapred.JobConf;

import org.apache.hadoop.mapred.MapReduceBase;

import org.apache.hadoop.mapred.Mapper;

import org.apache.hadoop.mapred.OutputCollector;

import org.apache.hadoop.mapred.Reducer;

import org.apache.hadoop.mapred.Reporter;

import org.apache.hadoop.mapred.TextInputFormat;

import org.apache.hadoop.mapred.TextOutputFormat;

public class LineCount {

<span style="white-space:pre">	</span>public static class Map extends MapReduceBase implements
<span style="white-space:pre">			</span>Mapper<LongWritable, Text, Text, IntWritable> {
<span style="white-space:pre">		</span>private final static IntWritable obj = new IntWritable(1);
<span style="white-space:pre">		</span>private Text words = new Text("Total Lines");
<span style="white-space:pre">		</span>public void map(LongWritable key, Text value,
<span style="white-space:pre">				</span>OutputCollector<Text, IntWritable> output, Reporter reporter)
<span style="white-space:pre">				</span>throws IOException {
<span style="white-space:pre">			</span>output.collect(words, obj);

<span style="white-space:pre">		</span>}
<span style="white-space:pre">	</span>}


<span style="white-space:pre">	</span>public static class Reduce extends MapReduceBase implements
<span style="white-space:pre">			</span>Reducer<Text, IntWritable, Text, IntWritable> {
<span style="white-space:pre">		</span>public void reduce(Text key, Iterator<IntWritable> values,
<span style="white-space:pre">				</span>OutputCollector<Text, IntWritable> output, Reporter reporter)
<span style="white-space:pre">				</span>throws IOException {
<span style="white-space:pre">			</span>int sum1 = 0;
<span style="white-space:pre">			</span>while (values.hasNext()) {
<span style="white-space:pre">				</span>sum1 += values.next().get();
<span style="white-space:pre">			</span>}

<span style="white-space:pre">			</span>output.collect(key, new IntWritable(sum1));
<span style="white-space:pre">		</span>}
<span style="white-space:pre">	</span>}


<span style="white-space:pre">	</span>public static void main(String[] args) throws Exception {
<span style="white-space:pre">		</span>JobConf config = new JobConf(LineCount.class);
<span style="white-space:pre">		</span>config.setJobName("LineCount");

<span style="white-space:pre">		</span>config.setOutputKeyClass(Text.class);

<span style="white-space:pre">		</span>config.setOutputValueClass(IntWritable.class);

<span style="white-space:pre">		</span>config.setMapperClass(Map.class);

<span style="white-space:pre">		</span>config.setCombinerClass(Reduce.class);

<span style="white-space:pre">		</span>config.setReducerClass(Reduce.class);

<span style="white-space:pre">		</span>config.setInputFormat(TextInputFormat.class);

<span style="white-space:pre">		</span>config.setOutputFormat(TextOutputFormat.class);

<span style="white-space:pre">		</span>FileInputFormat.setInputPaths(config, new Path(args[0]));

<span style="white-space:pre">		</span>FileOutputFormat.setOutputPath(config, new Path(args[1]));

<span style="white-space:pre">		</span>JobClient.runJob(config);
<span style="white-space:pre">	</span>}
}