Hbase之缓存扫描加快读取速度

最新推荐文章于 2025-06-20 09:47:41 发布

_miccretti

最新推荐文章于 2025-06-20 09:47:41 发布

阅读量313

点赞数

CC 4.0 BY-SA版权

文章标签：大数据 java

原文链接：http://www.cnblogs.com/similarface/p/5800509.html

本文通过具体示例展示了如何在HBase中通过调整扫描参数来优化数据读取效率，包括缓存大小、批量读取数量及扫描范围等关键配置。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.client.metrics.ScanMetrics;

import java.io.IOException;

/**
 * Created by similarface on 16/8/23.
 */
public class ScanDataUseCache {
    private static Table table=null;
    public static Table getTable() {
        if(table==null){
            try {
                Configuration configuration = HBaseConfiguration.create();
                Connection connection = ConnectionFactory.createConnection(configuration);
                //建立表的连接
                return connection.getTable(TableName.valueOf("testtable"));
            }catch (IOException e){
                return table;
            }
        }
        return table;
    }
    private static void scan(int caching,int batch,boolean small) {
        int count=0;
        //setCaching 设置的值为每次rpc的请求记录数，默认是1；cache大可以优化性能，但是太大了会花费很长的时间进行一次传输。
        //setBatch 设置每次取的column size；有些row特别大，所以需要分开传给client，就是一次传一个row的几个column。
        //setSmall 是否为小扫描
        //setScanMetricsEnabled 使用了集合
        Scan scan = new Scan().setCaching(caching).setBatch(batch).setSmall(small).setScanMetricsEnabled(true);
        ResultScanner scanner=null;
        try {
            scanner = getTable().getScanner(scan);
        }catch (IOException e){
            System.out.println(e);
        }
        if (scanner!=null){
            for (Result result:scanner){
                count++;
            }
        scanner.close();
        ScanMetrics metrics = scan.getScanMetrics();
        System.out.println("Caching: " + caching + ", Batch: " + batch + ", Small: " + small + ", Results: " + count + ", RPCs: " + metrics.countOfRPCcalls);
        }
        else {
            System.out.println("Error");
        }
    }

    public static void main(String[] args) throws IOException {
        // Caching: 1, Batch: 1, Small: false, Results: 9, RPCs: 12
        scan(1, 1, false);

        //Caching: 1, Batch: 0, Small: false, Results: 4, RPCs: 7
        scan(1, 0, false);

        // Caching: 1, Batch: 0, Small: true, Results: 4, RPCs: 0
        scan(1, 0, true);

        //Caching: 200, Batch: 1, Small: false, Results: 9, RPCs: 3
        scan(200, 1, false);

        //Caching: 200, Batch: 0, Small: false, Results: 4, RPCs: 3
        scan(200, 0, false);

        //Caching: 200, Batch: 0, Small: true, Results: 4, RPCs: 0
        scan(200, 0, true);

        // Caching: 2000, Batch: 100, Small: false, Results: 4, RPCs: 3
        scan(2000, 100, false);

        // Caching: 2, Batch: 100, Small: false, Results: 4, RPCs: 5
        scan(2, 100, false);

        // Caching: 2, Batch: 10, Small: false, Results: 4, RPCs: 5
        scan(2, 10, false);

        // Caching: 2, Batch: 10, Small: false, Results: 4, RPCs: 5
        scan(5, 100, false);

        // Caching: 5, Batch: 100, Small: false, Results: 4, RPCs: 3
        scan(5, 20, false);

        // Caching: 10, Batch: 10, Small: false, Results: 4, RPCs: 3
        scan(10, 10, false);
    }
}

/**
 Caching: 1, Batch: 0, Small: false, Results: 5, RPCs: 8
 Caching: 1, Batch: 0, Small: true, Results: 5, RPCs: 0
 Caching: 200, Batch: 1, Small: false, Results: 1009, RPCs: 8
 Caching: 200, Batch: 0, Small: false, Results: 5, RPCs: 3
 Caching: 200, Batch: 0, Small: true, Results: 5, RPCs: 0
 Caching: 2000, Batch: 100, Small: false, Results: 14, RPCs: 3
 Caching: 2, Batch: 100, Small: false, Results: 14, RPCs: 10
 Caching: 2, Batch: 10, Small: false, Results: 104, RPCs: 55
 Caching: 5, Batch: 100, Small: false, Results: 14, RPCs: 5
 Caching: 5, Batch: 20, Small: false, Results: 54, RPCs: 13
 Caching: 10, Batch: 10, Small: false, Results: 104, RPCs: 13
 **/

这是一个9行数据的表

每行包含一些列

使用缓存为6 批量为3的扫描器

需要3个RPC

3个列装入一个Result实例

6个result到缓存中组成一个RPC

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.client.metrics.ScanMetrics;

import java.io.IOException;

/**
 * Created by similarface on 16/8/24.
 */
public class ScanWithOffsetAndLimit {
    private static Table table = null;

    public static Table getTable() {
        if (table == null) {
            try {
                Configuration configuration = HBaseConfiguration.create();
                Connection connection = ConnectionFactory.createConnection(configuration);
                //建立表的连接
                return connection.getTable(TableName.valueOf("testtable"));
            } catch (IOException e) {
                return table;
            }
        }
        return table;
    }

    /**
     * 遍历访问数据
     * @param num 运行次序
     * @param caching
     * @param batch
     * @param offset
     * @param maxResults
     * @param maxResultSize
     * @param dump
     * @throws IOException
     */
    private static void scan(int num, int caching, int batch, int offset, int maxResults, int maxResultSize, boolean dump
    ) throws IOException {
        int count = 0;
        Scan scan = new Scan().setCaching(caching).setBatch(batch)
                .setRowOffsetPerColumnFamily(offset)
                .setMaxResultsPerColumnFamily(maxResults)
                .setMaxResultSize(maxResultSize)
                .setScanMetricsEnabled(true);
        ResultScanner scanner = getTable().getScanner(scan);
        System.out.println("Scan #" + num + " running...");
        for (Result result : scanner) {
            count++;
            if (dump)
                System.out.println("Result [" + count + "]:" + result);
        }
        scanner.close();
        ScanMetrics metrics = scan.getScanMetrics();
        System.out.println("Caching: " + caching + ", Batch: " + batch +
                ", Offset: " + offset + ", maxResults: " + maxResults +
                ", maxSize: " + maxResultSize + ", Results: " + count +
                ", RPCs: " + metrics.countOfRPCcalls);
    }

    public static void main(String[] args) throws IOException {
        //偏移为0 最大2个cell 所以会扫描到列1 和列2
        scan(1, 11, 0, 0, 2, -1, true);
        //偏移为4 最大2个cell 所以会扫描到列5 和列6
        scan(2, 11, 0, 4, 2, -1, true);
        //
        scan(3, 5, 0, 0, 2, -1, false);
        scan(4, 11, 2, 0, 5, -1, true);
        scan(5, 11, -1, -1, -1, 1, false);
        scan(6, 11, -1, -1, -1, 10000, false);
    }
}

/**
 Caching: 11, Batch: 0, Offset: 0, maxResults: 2, maxSize: -1, Results: 5005, RPCs: 458
 Caching: 11, Batch: 0, Offset: 4, maxResults: 2, maxSize: -1, Results: 1, RPCs: 3
 Caching: 5, Batch: 0, Offset: 0, maxResults: 2, maxSize: -1, Results: 5005, RPCs: 1004
 Caching: 11, Batch: 2, Offset: 0, maxResults: 5, maxSize: -1, Results: 5009, RPCs: 458
 Caching: 11, Batch: -1, Offset: -1, maxResults: -1, maxSize: 1, Results: 5005, RPCs: 11012
 Caching: 11, Batch: -1, Offset: -1, maxResults: -1, maxSize: 10000, Results: 5005, RPCs: 469
**/

转载于:https://www.cnblogs.com/similarface/p/5800509.html