Hbase操作

于 2019-09-18 11:27:45 发布
阅读量273 收藏
点赞数
CC 4.0 BY-SA版权
分类专栏： 大数据
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.youkuaiyun.com/m_target/article/details/100977078

1. Hbase的Shell操作

  • 列出所有的命名空间(相当于mysql中的show databases)

    • list_namespace

  • 列出指定命名空间下的所有表

    • list_namespace_tables ‘ns_name’

  • 创建命名空间

    • create_namespace ‘ns1’

  • 创建表

    • create ‘ns1:t1’,‘f1’

  • 禁用表,因为删除表之前首先需要禁用了

    • disable ‘ns1:t1’

  • 启用表

    • enable ‘ns1:t1’

  • 删除表

    • drop ‘ns1:t1’

  • 添加数据

    • put ‘ns1:t1’,‘row001’,‘f1:name’,‘xiaohua’

  • 查询数据

    • get ‘ns1:t1’,‘row001’,{COLUMN=>‘f1:name’}

  • 删除数据

    • delete ‘ns1:t1’,‘row001’,‘f1:name’

  • 删除一行数据

    • deleteall ‘ns1:t1’,‘row001’

  • 统计表的行数

    • count ‘ns1:t1’

  • 创建Hbase表时指定列族的显示版本数

    • create ‘ns1:t1’,{NAME=>‘f1’,VERSIONS=>3}

  • 修改Hbase表中的列族的显示版本数

    • alter ‘ns1:t1’,{NAME=>‘f1’,VERSIONS=>5}

  • 查询指定版本数的数据

    • get ‘ns1:t1’,{COLUMN=>‘f1:name’,VERSIONS=>3}

  • 版本号的作用

    根据显示的版本数,查询出来想要版本的时间戳,根据时间戳找出具体值

2. Hbase中API的基本操作

<properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <hbase-version>1.1.5</hbase-version>
</properties>
<dependencies>
    <dependency>
        <groupId>org.apache.hbase</groupId>
        <artifactId>hbase-client</artifactId>
        <version>${hbase-version}</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hbase</groupId>
        <artifactId>hbase-server</artifactId>
        <version>${hbase-version}</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hive</groupId>
        <artifactId>hive-hbase-handler</artifactId>
        <version>2.1.0</version>
    </dependency>

    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>4.12</version>
    </dependency>
</dependencies>

public class HbaseTest {
    //添加数据
    @Test
    public void testPut() throws IOException {
        Configuration conf = HBaseConfiguration.create();
        //指定zk的地址
        conf.set("hbase.zookeeper.quorum", "mini03:2181,mini04:2181,mini05:2181");
        Connection conn = ConnectionFactory.createConnection(conf);
        Table table = conn.getTable(TableName.valueOf("ns1:t1"));
        Put put = new Put(Bytes.toBytes("row001"));
        put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("name"), Bytes.toBytes("admin02"));
        table.put(put);
    }


    //删除数据
    @Test
    public void testDelete() throws IOException {
        Configuration conf = HBaseConfiguration.create();
        //指定zk的地址
        conf.set("hbase.zookeeper.quorum", "mini03:2181,mini04:2181,mini05:2181");
        Connection conn = ConnectionFactory.createConnection(conf);
        Table table = conn.getTable(TableName.valueOf("ns1:t1"));
        Delete delete = new Delete(Bytes.toBytes("row001"));
        table.delete(delete);
    }

    //查询数据
    @Test
    public void testGet() throws IOException {
        Configuration conf = HBaseConfiguration.create();
        //指定zk的地址
        conf.set("hbase.zookeeper.quorum", "mini03:2181,mini04:2181,mini05:2181");
        Connection conn = ConnectionFactory.createConnection(conf);
        Table table = conn.getTable(TableName.valueOf("ns1:t1"));
        Get get = new Get(Bytes.toBytes("row001"));
        Result result = table.get(get);
        String s = Bytes.toString(result.getValue(Bytes.toBytes("f1"),Bytes.toBytes("name")));
        System.out.println(s);
    }
}

3. Hbase中的API的管理操作

public class HbaseAdminTest {

    private Connection connection;

    @Before
    public void init() throws Exception {
        Configuration conf = new Configuration();
        conf.set("hbase.zookeeper.quorum", "mini03:2181,mini04:2181,mini05:2181");
        connection = ConnectionFactory.createConnection(conf);
    }

    /**
     * 创建表
     *
     * @throws Exception
     */
    @Test
    public void testCreateTable() throws Exception {
        //获取管理对象
        Admin admin = connection.getAdmin();
        HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("t2"));
        HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("f1"));
        htd.addFamily(hcd);
        admin.createTable(htd);
    }


    /**
     * 列出所有的表
     * @throws Exception
     */
    @Test
    public void testListTableNames() throws Exception {
        //获取管理对象
        Admin admin = connection.getAdmin();
        TableName[] tableNames = admin.listTableNames("ns1:.*");
        for (TableName tableName : tableNames) {
            System.out.println(tableName);
        }
    }

}

4. Hbase高级查询


    //查询数据
    @Test
    public void testRowFilter() throws IOException {
        Configuration conf = HBaseConfiguration.create();
        //指定zk的地址
        conf.set("hbase.zookeeper.quorum", "mini03:2181,mini04:2181,mini05:2181");
        Connection conn = ConnectionFactory.createConnection(conf);
        Table table = conn.getTable(TableName.valueOf("ns1:t1"));
        Scan scan = new Scan();
        byte[] cf = Bytes.toBytes("f1");
        byte[] column = Bytes.toBytes("name");

        Filter filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("row001")));
        scan.setFilter(filter);
        //获取包含多行数据的对象
        ResultScanner resultScanner = table.getScanner(scan);
        for (Result result : resultScanner) {
            System.out.println(Bytes.toString(result.getValue(Bytes.toBytes("f1"), Bytes.toBytes("name"))));
        }
    }


    //使用FamilyFilter过滤
    @Test
    public void testFamilyFilter() throws IOException {
        Configuration conf = HBaseConfiguration.create();
        //指定zk的地址
        conf.set("hbase.zookeeper.quorum", "mini03:2181,mini04:2181,mini05:2181");
        Connection conn = ConnectionFactory.createConnection(conf);
        Table table = conn.getTable(TableName.valueOf("ns1:t1"));
        Scan scan = new Scan();
        byte[] cf = Bytes.toBytes("f1");
        byte[] column = Bytes.toBytes("name");

        Filter filter = new FamilyFilter(CompareFilter.CompareOp.LESS, new BinaryComparator(Bytes.toBytes("f2")));
        scan.setFilter(filter);
        //获取包含多行数据的对象
        ResultScanner resultScanner = table.getScanner(scan);
        for (Result result : resultScanner) {
            System.out.println(Bytes.toString(result.getValue(Bytes.toBytes("f1"), Bytes.toBytes("name"))));
        }
    }


    //使用QualifierFilter过滤
    @Test
    public void testQualifierFilter() throws IOException {
        Configuration conf = HBaseConfiguration.create();
        //指定zk的地址
        conf.set("hbase.zookeeper.quorum", "mini03:2181,mini04:2181,mini05:2181");
        Connection conn = ConnectionFactory.createConnection(conf);
        Table table = conn.getTable(TableName.valueOf("ns1:t1"));
        Scan scan = new Scan();
        byte[] cf = Bytes.toBytes("f1");
        byte[] column = Bytes.toBytes("name");

        Filter filter = new QualifierFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("name")));
        scan.setFilter(filter);
        //获取包含多行数据的对象
        ResultScanner resultScanner = table.getScanner(scan);
        for (Result result : resultScanner) {
            System.out.println(Bytes.toString(result.getValue(Bytes.toBytes("f1"), Bytes.toBytes("name"))));
        }
    }

    //使用ValueFilter过滤
    @Test
    public void testValueFilter() throws IOException {
        Configuration conf = HBaseConfiguration.create();
        //指定zk的地址
        conf.set("hbase.zookeeper.quorum", "mini03:2181,mini04:2181,mini05:2181");
        Connection conn = ConnectionFactory.createConnection(conf);
        Table table = conn.getTable(TableName.valueOf("ns1:t1"));
        Scan scan = new Scan();
        byte[] cf = Bytes.toBytes("f1");
        byte[] column = Bytes.toBytes("name");

        Filter filter = new ValueFilter(CompareFilter.CompareOp.GREATER, new BinaryComparator(Bytes.toBytes("admin80")));
        scan.setFilter(filter);
        //获取包含多行数据的对象
        ResultScanner resultScanner = table.getScanner(scan);
        for (Result result : resultScanner) {
            System.out.println(Bytes.toString(result.getValue(Bytes.toBytes("f1"), Bytes.toBytes("name"))));
        }
    }

//查询数据
@Test
public void testScan() throws IOException {
    Configuration conf = HBaseConfiguration.create();
    //指定zk的地址
    conf.set("hbase.zookeeper.quorum", "mini03:2181,mini04:2181,mini05:2181");
    Connection conn = ConnectionFactory.createConnection(conf);
    Table table = conn.getTable(TableName.valueOf("ns1:t1"));
    Scan scan = new Scan();
    byte[] cf = Bytes.toBytes("f1");
    byte[] column = Bytes.toBytes("name");
    Filter filter = new SingleColumnValueFilter(cf, column, CompareFilter.CompareOp.EQUAL, Bytes.toBytes("admin123"));
    scan.setFilter(filter);
    //获取包含多行数据的对象
    ResultScanner resultScanner = table.getScanner(scan);
    for (Result result : resultScanner) {
        System.out.println(Bytes.toString(result.getValue(Bytes.toBytes("f1"), Bytes.toBytes("age"))));
    }
}

@Test
public void filterTable()throws Exception{
    Table table = connection.getTable(TableName.valueOf("ns1:t1"));
    Scan scan = new Scan();

    byte[] f1 = Bytes.toBytes("f1");//指定过滤列族
    byte[] clo = Bytes.toBytes("name");//指定过滤列名
    byte[] value = Bytes.toBytes("admin");//指定过滤列值
    Filter filter = new SingleColumnValueFilter(f1,clo, CompareFilter.CompareOp.EQUAL,value);
    scan.setFilter(filter);

    ResultScanner results = table.getScanner(scan);
    for (Result result : results) {//获取过滤后的的性别
        byte[] value1 = result.getValue(Bytes.toBytes("f1"), Bytes.toBytes("sex"));
        System.out.println(Bytes.toString(value1));
    }
}

5. 百万数据的插入

5.1 mysql百万数据写入

​ 耗时约20分钟

5.2 hbase百万数据的写入

/**
 * 百万数据的插入
 */
public class HbaseMiTest {

    private Connection connection;

    @Before
    public void init() throws Exception {
        Configuration conf = new Configuration();
        conf.set("hbase.zookeeper.quorum", "mini03:2181,mini04:2181,mini05:2181");
        connection = ConnectionFactory.createConnection(conf);
    }

    @Test
    public void test01() throws IOException {
        HTable table = (HTable) connection.getTable(TableName.valueOf("ns1:t1"));
        //不使用每个put操作都刷出一次
        table.setAutoFlush(false);
        long startTime = System.currentTimeMillis();
        for (int i = 0; i < 1000000; i++) {
            Put put = new Put(Bytes.toBytes("row" + i));
            //关闭预写日志,但是不建议使用,因为这样做不安全
            put.setWriteToWAL(false);
            put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("name"), Bytes.toBytes("admin" + i));
            table.put(put);
            if (i % 100000 == 0) {
                table.flushCommits();
            }
        }
        table.flushCommits();
        long endTime = System.currentTimeMillis();
        System.out.println("总耗时:" + (endTime - startTime) + "ms");
    }
}

大约耗时27s

评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值