Oozie MapReduce Action配置的要点

oozie调度wordcount mapreduce

介绍

创建一个wordcount程序

package com.hadoop.wordcount.mr;

import java.io.IOException;

import javax.naming.spi.DirStateFactory.Result;

import org.apache.commons.collections.map.StaticBucketMap;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Mapper.Context;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.P;

public class WordCount {

    public static class WordCountMapper extends Mapper<Object, Text, Text, IntWritable>{
        private final static IntWritable one = new IntWritable(1);
        private Text word = new Text();

        @Override
        protected void map(Object key, Text value, Context context)
                throws IOException, InterruptedException {
            StringBuilder sBuilder = new StringBuilder(value.toString());
            String[] strings = sBuilder.toString().split("\t");
            for (String tempString: strings){
                word.set(tempString);
                context.write(word, one);
            }

        }

    }


    public static class WordCountReducer extends Reducer<Text, IntWritable, Text, IntWritable>{

        private IntWritable result = new IntWritable();
    @Override
    protected void reduce(Text key, Iterable<IntWritable> values,Context context)
            throws IOException, InterruptedException {


        // TODO Auto-generated method stub
        int sum =0;
        for (IntWritable value : values){
            sum+=value.get();
        }
        result.set(sum);

        context.write(key, result);
    }   

    }


    public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException{
        Configuration configuration = new Configuration();
        Job job = Job.getInstance(configuration, "word count");
        job.setJarByClass(WordCount.class);
        job.setMapperClass(WordCountMapper.class);
        job.setCombinerClass(WordCountReducer.class);
        job.setReducerClass(WordCountReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        System.exit(job.waitForCompletion(true)?0:1);
    }

}

bin/hdfs dfs -put /opt/jars/file1.txt wordcount/input    

 bin/hadoop jar /opt/jars/mr-wc.jar /user/beifeng/wordcount/input /user/beifeng/wordcount/output1

配置job.properties

nameNode=hdfs://hadoop.beifeng.com:8020
jobTracker=hadoop.beifeng.com:8032
queueName=default
examplesRoot=wordcount

oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/oozieapps/map-reduce/workflow.xml
outputDir=map-reduce

配置workflow.xml

<workflow-app xmlns="uri:oozie:workflow:0.5" name="map-reduce-wordcount">
    <start to="mr-node"/>
    <action name="mr-node">
        <map-reduce>
            <job-tracker>${jobTracker}</job-tracker>
            <name-node>${nameNode}</name-node>
            <prepare>
                <delete path="${nameNode}/user/${wf:user()}/${examplesRoot}/output/${outputDir}"/>
            </prepare>
            <configuration>
                <!--                        mapper properties                         -->
                <property>
                    <name>mapred.mapper.new-api</name>
                    <value>true</value>
                </property>
                <property>
                    <name>mapred.job.queue.name</name>
                    <value>${queueName}</value>
                </property>
                <property>
                    <name>mapreduce.job.map.class</name>
                    <value>com.hadoop.wordcount.mr.WordCount$WordCountMapper</value>
                </property>
                <property>
                    <name>mapreduce.job.output.key.class</name>
                    <value>org.apache.hadoop.io.Text</value>
                </property>
                <property>
                    <name>mapreduce.job.output.value.class</name>
                    <value>org.apache.hadoop.io.IntWritable</value>
                </property>
                <property>
                    <name>mapreduce.input.fileinputformat.inputdir</name>
                    <value>/user/${wf:user()}/${examplesRoot}/input</value>
                </property>
                <!--                         reducer properties                         -->
                <property>
                    <name>mapred.reducer.new-api</name>
                    <value>true</value>
                </property>
                <property>
                    <name>mapreduce.job.reduce.class</name>
                    <value>com.hadoop.wordcount.mr.WordCount$WordCountReducer</value>
                </property>
                <property>
                    <name>mapred.map.tasks</name>
                    <value>1</value>
                </property>
                <property>
                    <name>mapreduce.output.fileoutputformat.outputdir</name>
                    <value>/user/${wf:user()}/${examplesRoot}/output/${outputDir}</value>
                </property>
            </configuration>
        </map-reduce>
        <ok to="end"/>
        <error to="fail"/>
    </action>
    <kill name="fail">
        <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
    </kill>
    <end name="end"/>
</workflow-app>

测试

bin/oozie job -oozie http://hadoop.beifeng.com:11000/oozie -config oozieapps/map-reduce/job.properties -run

参考资料：

http://blog.youkuaiyun.com/nsrainbow/article/details/43746111
http://blog.youkuaiyun.com/xiao_jun_0820?viewmode=contents