hadoop分区

最新推荐文章于 2024-04-17 15:52:39 发布

RZH_long

最新推荐文章于 2024-04-17 15:52:39 发布

阅读量621

点赞数

分类专栏： hadoop 文章标签： hadoop

本文链接：https://blog.youkuaiyun.com/RZH_long/article/details/121452075

版权

hadoop 专栏收录该内容

26 篇文章

订阅专栏

本文详细探讨了Hadoop的分区策略，包括默认分区器的工作原理、自定义分区的实现方式以及分区在大数据处理中的重要性。通过实例分析，阐述如何根据业务需求调整分区策略，以优化MapReduce作业的性能和数据分布。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

package com.a.b.partion_demo;

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Partitioner;

public class MyPartioner extends Partitioner<Text,NullWritable> {
    @Override
    public int getPartition(Text text, NullWritable nullWritable, int i) {
        String[] strings = text.toString().split("\t");
        System.out.println(strings[5]);

        if(Integer.parseInt(strings[5])>15){
            return  1;
        }else{
            return 0;
        }

    }

}

package com.a.b.partion_demo;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class PartionMain extends Configured implements Tool {

    @Override
    public int run(String[] strings) throws Exception {
        Job job = Job.getInstance(new Configuration(),"partion_demo");
        job.setJarByClass(PartionMain.class);

        job.setInputFormatClass(TextInputFormat.class);
        job.setOutputFormatClass(TextOutputFormat.class);

        TextInputFormat.addInputPath(job,new Path("hdfs://node01:8020/partition"));
        TextOutputFormat.setOutputPath(job,new Path("hdfs://node01:8020/partition_out"));

        job.setMapperClass(PartionMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(NullWritable.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);
        job.setReducerClass(PartionReduce.class);

        //分区,分区个数
        job.setPartitionerClass(MyPartioner.class);
        job.setNumReduceTasks(2);

        boolean b = job.waitForCompletion(true);
        return b?0:1;

    }

    public static void main(String[] args) throws Exception {
        int i = ToolRunner.run(new Configuration(),new PartionMain(),args);
        System.exit(i);
    }

}