hadoop wordcount windows idea本地测试方法

搭建hadoop集群环境这里不做过多说明

windows下hadoop环境搭建

1. 下载hadoop对应的包，这里给出百度网盘链接：链接：https://pan.baidu.com/s/1Br9fzRBEQFtEELPWjXWqLQ
   提取码：xcmt
2. 解压
3. 下载windows辅助文件：winutils.exe,链接：链接：https://pan.baidu.com/s/1gL1xCgvGCbBrTWVKODGHfQ
   提取码：duoq
4. 将winutils.exe放入hadoop解压的bin目录下
5. 配置hadoop的环境变量：
6. 至此完成配置

编写MR代码

1. idea创建普通maven项目
2. 配置pom

   <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.7.4</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.7.4</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>2.7.4</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-core</artifactId>
            <version>1.2.1</version>
        </dependency>

    </dependencies>

3. 编写mapper

package cl.wordcount;

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.util.StringUtils;

public class WordCountMapper extends Mapper<LongWritable, Text, Text, IntWritable>{
   

	//该方法循环调用，从文件的split中读取每行调用一次，把该行所在的下标为key，该行的内容为value
	protected void map(LongWritable key, Text value,
			Context context)
			throws IOException, InterruptedException {
   
		String[] words = StringUtils.split(value.toString(), ' ');
		for(String w :words){
   
			context.write(new Text(w), new IntWritable(1));
		}
	}
}

4. 编写reducer

package cl.wordcount;

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class WordCountReducer extends Reducer<Text, IntWritable, Text, IntWritable>{
   

	//每组调用一次，这一组数据特点：key相同，value可能有多个。
	protected void reduce(Text arg0, Iterable<IntWritable> arg1,
			Context arg2)
			throws IOException, InterruptedException {
   
		int sum =0;
		for(IntWritable i: arg1){
   
			sum=sum+i.get();
		}
		arg2.write(arg0, new IntWritable(sum));
	}
}

5. 编写主类，先将word单词文件要放到hdfs里对应的input目录里

package cl.wordcount;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.util.Properties;

public class RunJob {
   

	public static void main(String[] args) {
   
	    //解决permission deny问题
		Properties properties = System.getProperties();
		properties.setProperty("HADOOP_USER_NAME", "root");
		Configuration config =new Configuration();
		config.set("fs.defaultFS", "hdfs://node1:9000");
		config.set("yarn.resourcemanager.hostname", "node1");
//		config.set("mapred.jar", "C:\\Users\\Administrator\\Desktop\\wc.jar");
		try {
   
			FileSystem fs =FileSystem.get(config);
			
			Job job =Job.getInstance(config);
			job.setJarByClass(RunJob.class);
			
			job.setJobName("wc");
			
			job.setMapperClass(WordCountMapper.class);
			job.setReducerClass(WordCountReducer.class);
			
			job.setMapOutputKeyClass(Text.class);
			job.setMapOutputValueClass(IntWritable.class);
			
			FileInputFormat.addInputPath(job, new Path("hdfs://node1:9000/hadoop/input/wc"));
			
			Path outpath =new Path("hdfs://node1:9000/hadoop/output/wc");
			if(fs.exists(outpath)){
   
				fs.delete(outpath, true);
			}
			FileOutputFormat.setOutputPath(job, outpath);
			
			boolean f= job.waitForCompletion(true);
			if(f){
   
				System.out.println("finished");
			}
		} catch (Exception e) {
   
			e.printStackTrace();
		}
	}
}

8. 解决Exception in thread “main” java.lang.UnsatisfiedLinkError: org.apache.hadoop.io错误:重写hadoop源文件，更改返回值为true

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.io.nativeio;

import java.io.File;
import java.io.FileDescriptor;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.lang.reflect.Field;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.io.SecureIOUtils.AlreadyExistsException;
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.hadoop.util.Shell;
import org.apache.commons.logging.Log;
import org.apache.</