hadoop中job.setOutputFormatClass(PartitionByCoun...-优快云博客

本文详细介绍了使用Hadoop MapReduce进行数据处理时遇到的问题，特别是关于如何自定义输出格式并解决输出格式类不适用的错误。通过提供具体的代码示例和解释，读者可以了解到如何正确配置和解决问题。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

2019独角兽企业重金招聘Python工程师标准>>>

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.lib.MultipleTextOutputFormat;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;



public class MultiFile extends Configured  {
    public static class MapClass extends Mapper<LongWritable, Text, NullWritable, Text> {
        public void map(LongWritable key, Text value,
                        OutputCollector<NullWritable, Text> output,
                        Reporter reporter) throws IOException {
            output.collect(NullWritable.get(), value);
        }
    }
    public static class PartitionByCountryMTOF
        extends MultipleTextOutputFormat<NullWritable,Text>
    {
        protected String generateFileNameForKeyValue(NullWritable key,
                                                     Text value,
                                                     String filename)
        {
            String[] arr = value.toString().split(",", -1);
            String country = arr[4].substring(1,3);
            return country + "/" + filename;
        }
    }

    public static void main(String[] args) throws Exception { 
    	Configuration conf = new Configuration();
        Job job = new Job(conf, "MultiFile");
        job.setJarByClass(MultiFile.class);
        Path in = new Path(args[0]);
        Path out = new Path(args[1]);
        FileInputFormat.setInputPaths(job, in);//输入路径
        FileOutputFormat.setOutputPath(job, out);//输出路径
        job.setJobName("MultiFile");//job名称
        job.setMapperClass(MapClass.class);
        job.setInputFormatClass(TextInputFormat.class);//输入格式
        job.setOutputFormatClass(PartitionByCountryMTOF.class);//输出格式设置
        job.setMapOutputValueClass(LongWritable.class);//
        job.setOutputKeyClass(NullWritable.class);输出key的类型
        job.setOutputValueClass(Text.class);//输出value的类型
        job.setNumReduceTasks(0);//设置reduce个数
        System.exit(job.waitForCompletion(true)?0:1);
    }
}

job.setOutputFormatClass(PartitionByCountryMTOF.class);

显示The method setOutputFormatClass(Class<? extends OutputFormat>) in the type Job is

not applicable for the arguments (Class<MultiFile.PartitionByCountryMTOF>)

请问怎么解决？？

转载于:https://my.oschina.net/u/242736/blog/83042