简单的在Hadoop2.6.0上安装eclipse运行WORDCOUNT的总结笔记

最新推荐文章于 2020-12-21 22:38:43 发布

原创最新推荐文章于 2020-12-21 22:38:43 发布 · 662 阅读

2 ·

CC 4.0 BY-SA版权

文章标签：

#eclipse #hadoop #ubuntu #mapreduce #WordCount

本文介绍如何在Ubuntu系统中安装配置Eclipse及Hadoop，并实现两者间的集成，以便进行WordCount程序的开发与运行。从下载安装Eclipse到配置Hadoop环境，再到编写并运行WordCount示例代码，提供了详细的步骤说明。

新手总结笔记

1.在ubuntu的软件商店中下载eclipse

2.在官网上下载hadoop-eclipse-plugin-2.6.2.jar , 并把它放到eclipse的安装路径上usr/lib/eclipse/plugins

3.进入eclipse 的Window ---> preference下的Hadoop Map/Reduce, 点击Browse选择出hadoop的根路径

4.设置好后进入Window-->Open Perspective -->Other , 选择Map/Reduce

5.看界面下方位置，选择小象头像，新建New Hadoop Location

6.设置如下，名字随意，开心就好

7.出现如下图及配置成功,不出现可右键，点击Refresh

8.选中input文件右键选中Upload Files To DFS ,选择一个新建的.txt文件，比如file.txt，在其中可随意写一些单词

9.基本配置完成，接下来是WordCount ，File-->New-->Project-->Map/Reduce Project

10.新建项目写入 wordcount代码

代码如下：

import org.apache.hadoop.conf.Configuration;  
import org.apache.hadoop.fs.Path;  
import org.apache.hadoop.io.IntWritable;  
import org.apache.hadoop.io.Text;  
import org.apache.hadoop.mapreduce.Job;  
import org.apache.hadoop.mapreduce.Mapper;  
import org.apache.hadoop.mapreduce.Reducer;  
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;  
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;  
import org.apache.hadoop.util.GenericOptionsParser;  
  
import java.io.IOException;  
import java.nio.file.FileSystem;
import java.util.StringTokenizer;  
  
@SuppressWarnings("deprecation")  
public class wordcount {  
  
    //在map阶段接收输入的<key, value>(key是当前输入的行号，value是对应的内容)，  
    //然后对内容进行切词，每切下一个词就将其组织成<word,1>的形式输出（输出即写到context中）  
    //设置map的输入类型为<Object, Text>  
    //输出类型为<Text, IntWritable>  
    public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> {  
        //one表示单词出现1次  
        private final static IntWritable one = new IntWritable(1);  
        //word存储切下的单词  
        private Text word = new Text();  
  
        @Override  
        public void map(Object key, Text value, Context context) throws IOException, InterruptedException {  
            //参数key表示的是行号，下面并没有用到key  
            StringTokenizer itr = new StringTokenizer(value.toString()); //对输入的行进行切词  
            while (itr.hasMoreTokens()) { //切下单词，存入word  
                word.set(itr.nextToken());  
                context.write(word, one);  
            }  
        }  
    }  
  
    //Reducer是对相同key下的所有value进行处理.  
    //在reduce阶段，TaskTracker会接收到<word,{1,1,1,1}>形式的数据，也就是特定单词出现次数的情况  
    //设置reduce的输入数据类型为<Text, IntWritable>  
    //输出数据类型为<Text, IntWritable>  
    public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> {  
        //result记录单词的个数  
        private IntWritable result = new IntWritable();  
  
        @Override  
        public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {  
            int sum = 0;  
            //对获取的<key, value-list>计算value的和  
            for (IntWritable val : values) {  
                sum += val.get();  
            }  
            //将频数设置到result中  
            result.set(sum);  
            //收集结果  
            context.write(key, result);  
        }  
    }  
  
    public static void main(String[] args) throws Exception {  
        Configuration conf = new Configuration();  
        String[] otherArgs = new GenericOptionsParser(conf, args)  
                .getRemainingArgs();  
        if (otherArgs.length != 2) {  
            System.err.println("Usage: wordcount <in> <out>");  
            System.exit(2);  
        }  
        //配置作业名  
        Job job = new Job(conf, "word count");  
        //配置作业各个类  
        job.setJarByClass(wordcount.class);  
        job.setMapperClass(TokenizerMapper.class);  
        job.setCombinerClass(IntSumReducer.class);  
        job.setReducerClass(IntSumReducer.class);  
        job.setOutputKeyClass(Text.class);  
        job.setOutputValueClass(IntWritable.class);  
        Path path = new Path(otherArgs[1]);// 取第1个表示输出目录参数（第0个参数是输入目录）  
        org.apache.hadoop.fs.FileSystem fileSystem = path.getFileSystem(conf);// 根据path找到这个文件  
        if (fileSystem.exists(path)) {  
            fileSystem.delete(path, true);// true的意思是，就算output有东西，也一带删除  
        }  
        FileInputFormat.addInputPath(job, new Path(otherArgs[0]));  
        FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));  
  
        System.exit(job.waitForCompletion(true) ? 0 : 1);  
    }  
}

**附加**

** ********
：配置日志文件

（1）选择 Other

（2）新建file文件,取名log4j.properties

（3）log4j.properties文件

（4）代码如是

log4j.rootCategory=INFO, stdout , R   
   
log4j.appender.stdout=org.apache.log4j.ConsoleAppender   
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout   
log4j.appender.stdout.layout.ConversionPattern=[QC] %p [%t] %C.%M(%L) | %m%n   
    
log4j.appender.R=org.apache.log4j.DailyRollingFileAppender   
log4j.appender.R.File=D:\\Tomcat 5.5\\logs\\qc.log   
log4j.appender.R.layout=org.apache.log4j.PatternLayout   
1log4j.appender.R.layout.ConversionPattern=%d-[TS] %p %t %c - %m%n   
   
log4j.logger.com.neusoft=DEBUG   
log4j.logger.com.opensymphony.oscache=ERROR   
log4j.logger.net.sf.navigator=ERROR   
log4j.logger.org.apache.commons=ERROR   
log4j.logger.org.apache.struts=WARN   
log4j.logger.org.displaytag=ERROR   
log4j.logger.org.springframework=DEBUG   
log4j.logger.com.ibatis.db=WARN   
log4j.logger.org.apache.velocity=FATAL   
   
log4j.logger.com.canoo.webtest=WARN   
   
log4j.logger.org.hibernate.ps.PreparedStatementCache=WARN   
log4j.logger.org.hibernate=DEBUG   
log4j.logger.org.logicalcobwebs=WARN  

log4j.rootCategory=INFO, stdout , R

log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=[QC] %p [%t] %C.%M(%L) | %m%n
 
log4j.appender.R=org.apache.log4j.DailyRollingFileAppender
log4j.appender.R.File=D:\\Tomcat 5.5\\logs\\qc.log
log4j.appender.R.layout=org.apache.log4j.PatternLayout
1log4j.appender.R.layout.ConversionPattern=%d-[TS] %p %t %c - %m%n

log4j.logger.com.neusoft=DEBUG
log4j.logger.com.opensymphony.oscache=ERROR
log4j.logger.net.sf.navigator=ERROR
log4j.logger.org.apache.commons=ERROR
log4j.logger.org.apache.struts=WARN
log4j.logger.org.displaytag=ERROR
log4j.logger.org.springframework=DEBUG
log4j.logger.com.ibatis.db=WARN
log4j.logger.org.apache.velocity=FATAL

log4j.logger.com.canoo.webtest=WARN

log4j.logger.org.hibernate.ps.PreparedStatementCache=WARN
log4j.logger.org.hibernate=DEBUG
log4j.logger.org.logicalcobwebs=WARN

******************************************************************************************************

11.完成后，进行环境设置，右键java文件Run as 选择最后一项