案例: Reduce 端实现 JOIN 和 Map端 实现JOIN

测试数据连接：
链接：https://pan.baidu.com/s/1TBHvrfO3dKBO8xOaeFXS3Q
提取码：4zug

1. 需求 Reduce 端实现 JOIN

假如数据量巨大，两表的数据是以文件的形式存储在 HDFS 中, 需要用 MapReduce 程序来实现以下 SQL 查询运算

select  a.id,a.date,b.name,b.category_id,b.price from t_order a left join t_product b on a.pid = b.id

商品表

id	pname	category_id	price
P0001	小米5	1000	2000
P0002	锤子T1	1000	3000

订单数据表

id	date	pid	amount
1001	20150710	P0001	2
1002	20150710	P0002	3

###2.2 实现步骤

通过将关联的条件作为map输出的key，将两表满足join条件的数据并携带数据所来源的文件信息，发往同一个reduce task，在reduce中进行数据的串联

Step 1: 定义 Mapper

package com.mjoin;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;

import java.io.IOException;

public class ReduceJoinMapper extends Mapper<LongWritable, Text, Text, Text> {

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        //1. 判断文件来自哪里
        FileSplit fileSplit = (FileSplit) context.getInputSplit();
        String fileName = fileSplit.getPath().getName();
        System.out.println("fileName为；" + fileName);

        if(fileName.equals("product.txt")) {
            //数据来自商品表
            //2. 将k1 和 v2转为 k2 v2，写入上下文
            String[] strs = value.toString().split(",");
            String productId = strs[0];

            context.write(new Text(productId), value);
        } else {
            //数据来自订单表
            //2.将 k1 和 v1  转为 k2 v2，写入上下文
            String[] strs = value.toString().split(",");
            String productId = strs[2];

            context.write(new Text(productId), value);
        }
    }
}

Step 2: 定义 Reducer

package com.mjoin;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class ReduceJoinReducer extends Reducer<Text, Text, Text, Text> {

    @Override
    protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
        //1.遍历集合，获取v3 （first + second）
        String first = "";
        String second = "";
        for(Text text: values) {
            if(text.toString().startsWith("p")) {
                first = text.toString();
            } else {
                second += text.toString();
            }
        }

        //2: 将k3 和 v3写入上下文
        context.write(key, new Text(first + "\t" + second));
    }
}

Step 3: 定义主类

package com.mjoin;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class ReduceJoinMain extends Configured implements Tool {

    @Override
    public int run(String[] strings) throws Exception {
        //获取job
        Job job = Job.getInstance(super.getConf(), "rejoin");
        TextInputFormat.addInputPath(job, new Path("file:///C:\\xu\\xuexi\\hadoop\\test\\join\\input"));

        //2.设置map类型及类型数据
        job.setMapperClass(ReduceJoinMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);

        //3,4,5,6

        //7 设置reducer类型和数据
        job.setReducerClass(ReduceJoinReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);

        //8 设置输出类型和输出路径
        job.setOutputFormatClass(TextOutputFormat.class);
        TextOutputFormat.setOutputPath(job, new Path("file:///C:\\xu\\xuexi\\hadoop\\test\\join\\output"));

        //3:等待job任务结束
        boolean bl = job.waitForCompletion(true);

        return bl ? 0: 1;
    }

    public static void main(String[] args) throws Exception {
        Configuration configuration = new Configuration();
        //启动job任务
        int run = ToolRunner.run(configuration, new ReduceJoinMain(), args);

        System.exit(run);
    }
}

测试结果：

2. 案例: Map端实现 JOIN

2.1 概述

​ 适用于关联表中有小表的情形.

​ 使用分布式缓存,可以将小表分发到所有的map节点，这样，map节点就可以在本地对自己所读到的大表数据进行join并输出最终结果，可以大大提高join操作的并发度，加快处理速度

2.2 实现步骤

先在mapper类中预先定义好小表，进行join

引入实际场景中的解决方案：一次加载数据库或者用

Step 1：定义Mapper

package com.mpjoin;

import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.FileSystemCounter;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import java.util.HashMap;

public class MapJoinMapper extends Mapper<LongWritable, Text, Text, Text> {

    private HashMap<String, String> map = new HashMap<>();
    /**
     * 1. 将分布式缓存的小表数据读取到本地map集合（只需要做一次)
     */
    @Override
    protected void setup(Context context) throws IOException, InterruptedException {
        //1.获取分布式缓存文件列表
        URI[] cacheFiles = context.getCacheFiles();

        //2.获取指定的分石化缓存文件系统
        FileSystem fileSystem = FileSystem.get(cacheFiles[0], context.getConfiguration());

        //3.获取文件输入流
        FSDataInputStream  inputStream = fileSystem.open(new Path(cacheFiles[0]));

        //4.读取文件内容并将数据存入map集合
        //4.1 将字节流转为字符缓冲流 FSDataInputStream  > BufferedReader
        BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream));
        //4.2 将读取小标文件中内容，以行为单位，并将读取数据存入map中
        String line = null;
        while((line = bufferedReader.readLine()) != null) {
            String[] strs = line.split(",");

            map.put(strs[0], line);
        }

        bufferedReader.close();
        fileSystem.close();
    }

    //2. 对大表进行业务处理
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        //1.从行文本中获取商品id， p0001, p0002 得到 k2
        String[] strs = value.toString().split(",");
        String productId = strs[2];

        //2. 在Map集合中，将商品的id作为键，获取值（商品的行文本数据），将value和值拼接得到 v2
        String proudctLine = map.get(productId);
        String valueLine = proudctLine + "\t" + value.toString();

        //3.将 k2 v2写入上下文中
        context.write(new Text(productId), new Text((valueLine)));
    }
}

Step 2：定义主类

package com.mpjoin;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

import java.net.URI;

public class JobMain extends Configured implements Tool {

    @Override
    public int run(String[] strings) throws Exception {

        //1.获取job对象
        Job job = Job.getInstance(super.getConf(), "mapJoin");
        job.addCacheFile(new URI("hdfs://node01:8020/cache_file/product.txt"));

        //1.设置输入类和输入类型
        job.setInputFormatClass(TextInputFormat.class);
        TextInputFormat.addInputPath(job, new Path("file:///C:\\xu\\xuexi\\hadoop\\test\\join\\input2"));

        //2.设置mapper类和数据类型
        job.setMapperClass(MapJoinMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);

        //8设置输出类型和输出路径
        job.setOutputFormatClass(TextOutputFormat.class);
        TextOutputFormat.setOutputPath(job, new Path("file:///C:\\xu\\xuexi\\hadoop\\test\\join\\output2"));

        //等待吗任务结束
        boolean bl = job.waitForCompletion(true);


        return bl ? 0:1;
    }

    public static void main(String[] args) throws Exception {
        Configuration configuration = new Configuration();
        int run = ToolRunner.run(configuration, new JobMain(), args);
        System.exit(run);
    }
}