Hbase-Filter算子大全

最新推荐文章于 2024-04-26 23:33:31 发布
原创最新推荐文章于 2024-04-26 23:33:31 发布 · 6.4k 阅读
1 ·
CC 4.0 BY-SA版权
文章标签：
#Hbase #Filter #过滤器
Hbase 专栏收录该内容
20 篇文章
订阅专栏
本文详细介绍了HBase中各种过滤器的应用场景及实现方式，包括单值过滤器、过滤器列表、分页过滤器、行过滤器、列过滤器、单元格过滤器等，并提供了具体的Java实现代码。
摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >
public static void ValueFilterExam() throws Exception{
        Configuration config = HBaseConfiguration.create();
        config.set("hbase.zookeeper.quorum","node2");
        config.set("hbase.zookeeper.property.clientPort","2181");
        try(Connection connection = ConnectionFactory.createConnection(config)) {
            Table table = connection.getTable(TableName.valueOf("test"));
            //类似于sql的like"%wang%"  SubstringComparator是是否包含这个str
            Scan scan =new Scan(Bytes.toBytes("rk-001"));
            Filter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, new SubstringComparator("wanghua"));
            //单值过滤器
            Filter single = new SingleColumnValueFilter(Bytes.toBytes("cf1"),Bytes.toBytes("001"),CompareFilter.CompareOp.EQUAL, new SubstringComparator("wanghua"));

            scan.setFilter(valueFilter);

            ResultScanner scannerResult = table.getScanner(scan);
            for (Result rs:scannerResult) {
                System.out.println(Bytes.toString(rs.getValue(Bytes.toBytes("cf1"),Bytes.toBytes("001"))));
            }
        }
    }

    //执行了三个过滤器，速度比较慢
    public static void FilterListExam() throws Exception{
        Configuration config = HBaseConfiguration.create();
        config.set("hbase.zookeeper.quorum","node2");
        config.set("hbase.zookeeper.property.clientPort","2181");
        try(Connection connection = ConnectionFactory.createConnection(config)) {
            Table table = connection.getTable(TableName.valueOf("test"));

            Scan scan =new Scan(Bytes.toBytes("rk-001"));
            FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);

            //只有列簇为cf1的才会放入结果集
            Filter familyFilter = new FamilyFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("wanghua")));
            filterList.addFilter(familyFilter);

            //只有列为00的被放入
            Filter qualiFilter = new QualifierFilter(CompareFilter.CompareOp.EQUAL,new BinaryComparator(Bytes.toBytes("00")));
            filterList.addFilter(qualiFilter);

            //只有值包含‘wang’的被放入
            Filter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, new SubstringComparator("wanghua"));
            filterList.addFilter(valueFilter);

            scan.setFilter(filterList);

            ResultScanner scannerResult = table.getScanner(scan);
            for (Result rs:scannerResult) {
                System.out.println(Bytes.toString(rs.getValue(Bytes.toBytes("cf1"),Bytes.toBytes("001"))));
            }
        }
    }

    //分页过滤器
    public static void PageFilterExam() throws Exception{
        Configuration config = HBaseConfiguration.create();
        config.set("hbase.zookeeper.quorum","node2");
        config.set("hbase.zookeeper.property.clientPort","2181");
        try(Connection connection = ConnectionFactory.createConnection(config)) {
            Table table = connection.getTable(TableName.valueOf("test"));

            Scan scan =new Scan(Bytes.toBytes("rk-001"));

            Filter pageFilter = new PageFilter(2L);

            //行过滤器  过滤出rowkey > rk-001的
            Filter rowFilter = new RowFilter(CompareFilter.CompareOp.GREATER,new BinaryComparator(Bytes.toBytes("rk-001")));

            scan.setFilter(pageFilter);

            ResultScanner scannerResult = table.getScanner(scan);
            for (Result rs:scannerResult) {
                System.out.println(Bytes.toString(rs.getValue(Bytes.toBytes("cf1"),Bytes.toBytes("001"))));
            }
        }
    }

    //行过滤器
    public static void RowFilterExam() throws Exception{
        Configuration config = HBaseConfiguration.create();
        config.set("hbase.zookeeper.quorum","node2");
        config.set("hbase.zookeeper.property.clientPort","2181");
        try(Connection connection = ConnectionFactory.createConnection(config)) {
            List<MultiRowRangeFilter.RowRange> rowlist =new ArrayList<>();

            //前缀过滤器
            PrefixFilter prefixFilter = new PrefixFilter(Bytes.toBytes("rk-0"));

            //模糊过滤器  查找出  尾号4567 年份2016的     0,1对应位置是否是？
            FuzzyRowFilter fuzzyRowFilter = new FuzzyRowFilter(Arrays.asList(new Pair<byte[], byte[]>(
                    Bytes.toBytesBinary("2016_??_??_4567"),
                    new byte[]{0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0}
            )));


            Table table = connection.getTable(TableName.valueOf("test"));
            //构建从rok-001 到rk-003的range
            MultiRowRangeFilter.RowRange rowRange1 = new MultiRowRangeFilter.RowRange("rk-001", true, "rk-003", true);

            //构建从rok-005 到rk-007的range
            MultiRowRangeFilter.RowRange rowRange2 = new MultiRowRangeFilter.RowRange("rk-005", true, "rk-007", true);

            rowlist.add(rowRange1);

            MultiRowRangeFilter multiRowRangeFilter = new MultiRowRangeFilter(rowlist);

            Scan scan =new Scan();
            //行过滤器  过滤出rowkey > rk-001的
            //Filter rowFilter = new RowFilter(CompareFilter.CompareOp.GREATER,new BinaryComparator(Bytes.toBytes("rk-001")));

            scan.setFilter(multiRowRangeFilter);

            ResultScanner scannerResult = table.getScanner(scan);
            for (Result rs:scannerResult) {
                System.out.println(Bytes.toString(rs.getValue(Bytes.toBytes("cf1"),Bytes.toBytes("001"))));
            }
        }
    }

    //列过滤器
    public static void QualiFilterExam() throws Exception{
        Configuration config = HBaseConfiguration.create();
        config.set("hbase.zookeeper.quorum","node2");
        config.set("hbase.zookeeper.property.clientPort","2181");
        try(Connection connection = ConnectionFactory.createConnection(config)) {
            Table table = connection.getTable(TableName.valueOf("test"));

            Scan scan =new Scan(Bytes.toBytes("rk-001"));

            //前缀过滤器
            Filter columnPrefixFilter = new ColumnPrefixFilter(Bytes.toBytes("cf"));

            //列等于cf1的
            QualifierFilter cf1 = new QualifierFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("cf1")));

            //依赖过滤器
            //先指定一个列为依赖列，然后以该依赖列的时间戳去过滤其他的列，凡是比依赖列时间戳大的都过滤
            //目的：防止有些没写完成的数据被读取到导致信息不全
            Filter depent = new DependentColumnFilter(Bytes.toBytes("cf1"),Bytes.toBytes("updatetime"));

            //首次列键过滤，用来统计数据多少行
            Filter firstkey =new FirstKeyOnlyFilter();

            //列名返回过滤器   按照字典顺序  如  ab  -  ad   只会过滤出 ab ac  ad的
            Filter rangeFilter = new ColumnRangeFilter(Bytes.toBytes("active"),true,Bytes.toBytes("city"),true);

            scan.setFilter(firstkey);

            ResultScanner scannerResult = table.getScanner(scan);
            int count = 0;
            for (Result rs:scannerResult) {
               count++;
            }
        }
    }

    //单元格和装饰过滤器
    public static void CellFilterExam() throws Exception{
        Configuration config = HBaseConfiguration.create();
        config.set("hbase.zookeeper.quorum","node2");
        config.set("hbase.zookeeper.property.clientPort","2181");
        try(Connection connection = ConnectionFactory.createConnection(config)) {
            Table table = connection.getTable(TableName.valueOf("test"));

            Scan scan =new Scan(Bytes.toBytes("rk-001"));

            List<Long> time= new ArrayList<>();
            time.add(5L);
            time.add(7L);

            //精确到毫秒，适用于自定义时间的场合
            Filter timrFilter = new TimestampsFilter(time);

            //装饰过滤器，一般和其他过滤器结合使用
            //跳过value中不含有wang的
            Filter valueFilter = new ValueFilter(CompareFilter.CompareOp.NOT_EQUAL,new BinaryComparator(Bytes.toBytes("wang")));
            Filter skipFilter = new SkipFilter(valueFilter);

            //全匹配过滤器，有一个false，全部终止
            WhileMatchFilter whileMatchFilter = new WhileMatchFilter(valueFilter);

            scan.setFilter(skipFilter);

            ResultScanner scannerResult = table.getScanner(scan);
            for (Result rs:scannerResult) {
                System.out.println(Bytes.toString(rs.getValue(Bytes.toBytes("cf1"),Bytes.toBytes("001"))));
            }
        }
    }