javaAPI调用链接Hbase
javaAPI链接Hbase的基础demo,后面的过滤部分也以此table为主
pom
<!--HBASE START-->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>1.4.9</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>1.4.9</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-common</artifactId>
<version>1.4.9</version>
</dependency>
<!--HBASE END-->
java
public class HbaseDemo {
private static Admin admin;
public static void main(String[] args){
System.setProperty("hadoop.home.dir", "d:/");
String tableName ="user_table";
try {
createTable(tableName, new String[] { "information", "contact" });
showTable(tableName);
User user = new User("001", "xiaoming", "123456", "man", "20", "13355550021", "1232821@bmaoo.com");
insertData("user_table", user);
User user2 = new User("002", "xiaohong", "654321", "female", "18", "18757912212", "214214@bmaoo.com");
insertData(tableName, user2);
System.out.println("--------------------获取原始数据-----------------------");
getNoDealData(tableName);
System.out.println("--------------------行列簇值对应关系数据-----------------------");
getAllRows(tableName);
List<User> list = getAllData(tableName);
System.out.println("--------------------原始数据解析处理后--------------------");
for (User user3 : list){
System.out.println(user3.toString());
}
System.out.println("--------------------根据rowKey查询--------------------");
User user4 = getDataByRowKey("user_table", "user-001");
System.out.println(user4.toString());
System.out.println("--------------------获取指定列族:列的属性值数据-------------------");
String user_phone = getCellData("user_table", "user-001", "contact", "phone");
System.out.println(user_phone);
System.out.println("--------------------插入测试数据--------------------");
// User user5 = new User("test-003", "xiaoguang", "789012", "man", "22", "12312132214", "856832@bmaoo.com");
// insertData("user_table", user5);
// List<User> list2 = getAllData("user_table");
//
// for (User user6 : list2){
// System.out.println(user6.toString());
// }
// System.out.println("--------------------删除测试数据--------------------");
// deleteByRowKey("user_table", "user-test-003");
// List<User> list3 = getAllData("user_table");
//
// for (User user7 : list3){
// System.out.println(user7.toString());
// }
} catch (Exception e) {
e.printStackTrace();
}
}
//连接集群
public static Connection initHbase() throws IOException {
Configuration configuration = HBaseConfiguration.create();
configuration.set("hbase.zookeeper.quorum", "192.168.1.101");
configuration.set("hbase.zookeeper.property.clientPort", "2181");
//集群配置↓
//configuration.set("hbase.zookeeper.quorum", "101.236.39.141,101.236.46.114,101.236.46.113");
// configuration.set("hbase.master", "192.168.1.101:60000");
Connection connection = ConnectionFactory.createConnection(configuration);
return connection;
}
public static void scanDataStep1(String tableName) throws Exception {
Table table= initHbase().getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
ResultScanner scanner = table.getScanner(scan);
for (Result result : scanner) {
if (result.getValue(Bytes.toBytes("info"), Bytes.toBytes("f_pid")) == null) {
for (KeyValue kv : result.raw()) {
System.out.print(new String(kv.getRow()) + " ");
System.out.print(new String(kv.getFamily()) + ":");
System.out.print(new String(kv.getQualifier()) + " = ");
System.out.print(new String(kv.getValue()));
System.out.print(" timestamp = " + kv.getTimestamp() + "\n");
}
}
}
}
//创建表
public static void createTable(String tableNmae, String[] cols) throws IOException {
TableName tableName = TableName.valueOf(tableNmae);
admin = initHbase().getAdmin();
if (admin.tableExists(tableName)) {
System.out.println("表已存在！");
} else {
HTableDescriptor hTableDescriptor = new HTableDescriptor(tableName);
for (String col : cols) {
HColumnDescriptor hColumnDescriptor = new HColumnDescriptor(col);
hTableDescriptor.addFamily(hColumnDescriptor);
}
admin.createTable(hTableDescriptor);
}
}
// 显示表结构
public static void showTable(String tableNmae) throws Exception {
admin = initHbase().getAdmin();
HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf(tableNmae));
System.out.println("表结构--------------------------");
byte[] name = tableDescriptor.getName();
System.out.println("下面开始输出结果：");
System.out.println("表名：" + new String(name));
HColumnDescriptor[] columnFamilies = tableDescriptor.getColumnFamilies();
for (HColumnDescriptor d : columnFamilies)
{
System.out.println("列族名：" + d.getNameAsString());
}
}
//插入数据
public static void insertData(String tableName, User user) throws IOException {
TableName tablename = TableName.valueOf(tableName);
Put put = new Put(("user-" + user.getId()).getBytes());
//参数：1.列族名 2.列名 3.值
put.addColumn("information".getBytes(), "username".getBytes(), user.getUsername().getBytes()) ;
put.addColumn("information".getBytes(), "age".getBytes(), user.getAge().getBytes()) ;
put.addColumn("information".getBytes(), "gender".getBytes(), user.getGender().getBytes()) ;
put.addColumn("contact".getBytes(), "phone".getBytes(), user.getPhone().getBytes());
put.addColumn("contact".getBytes(), "email".getBytes(), user.getEmail().getBytes());
//HTable table = new HTable(initHbase().getConfiguration(),tablename);已弃用
Table table = initHbase().getTable(tablename);
table.put(put);
}
//获取原始数据
public static void getNoDealData(String tableName){
try {
Table table= initHbase().getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
ResultScanner resutScanner = table.getScanner(scan);
for(Result result: resutScanner){
System.out.println("scan: " + result);
}
} catch (IOException e) {
e.printStackTrace();
}
}
// 得到所有数据,行,列簇,列名,值
public static void getAllRows(String tableName) throws IOException{
Table table = initHbase().getTable(TableName.valueOf(tableName));
//得到用于扫描 region 的对象
Scan scan = new Scan();
//使用 HTable 得到 resultcanner 实现类的对象
ResultScanner resultScanner = table.getScanner(scan);
System.out.print("行键\t\t列族\t列\t值\n");
for(Result result : resultScanner){
Cell[] cells = result.rawCells();
for(Cell cell : cells){
//得到 rowkey
System.out.print(Bytes.toString(CellUtil.cloneRow(cell)) +"\t");
//得到列族
System.out.print( Bytes.toString(CellUtil.cloneFamily(cell)) +"\t");
System.out.print( Bytes.toString(CellUtil.cloneQualifier(cell))+"\t");
System.out.println( Bytes.toString(CellUtil.cloneValue(cell)));
}
System.out.println("--------------------------------");
}
}
//根据rowKey进行查询
public static User getDataByRowKey(String tableName, String rowKey) throws IOException {
Table table = initHbase().getTable(TableName.valueOf(tableName));
Get get = new Get(rowKey.getBytes());
User user = new User();
user.setId(rowKey);
//先判断是否有此条数据
if(!get.isCheckExistenceOnly()){
Result result = table.get(get);
for (Cell cell : result.rawCells()){
String colName = Bytes.toString(cell.getQualifierArray(),cell.getQualifierOffset(),cell.getQualifierLength());
String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
if(colName.equals("username")){
user.setUsername(value);
}
if(colName.equals("age")){
user.setAge(value);
}
if (colName.equals("gender")){
user.setGender(value);
}
if (colName.equals("phone")){
user.setPhone(value);
}
if (colName.equals("email")){
user.setEmail(value);
}
}
}
return user;
}
//查询指定单cell内容
public static String getCellData(String tableName, String rowKey, String family, String col){
try {
Table table = initHbase().getTable(TableName.valueOf(tableName));
String result = null;
Get get = new Get(rowKey.getBytes());
if(!get.isCheckExistenceOnly()){
get.addColumn(Bytes.toBytes(family),Bytes.toBytes(col));
Result res = table.get(get);
byte[] resByte = res.getValue(Bytes.toBytes(family), Bytes.toBytes(col));
return result = Bytes.toString(resByte);
}else{
return result = "查询结果不存在";
}
} catch (IOException e) {
e.printStackTrace();
}
return "出现异常";
}
//查询指定表名中所有的数据
public static List<User> getAllData(String tableName){
Table table = null;
List<User> list = new ArrayList<User>();
try {
table = initHbase().getTable(TableName.valueOf(tableName));
ResultScanner results = table.getScanner(new Scan());
User user = null;
for (Result result : results){
String id = new String(result.getRow());
System.out.println("用户名:" + new String(result.getRow()));
user = new User();
for(Cell cell : result.rawCells()){
String row = Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
//String family = Bytes.toString(cell.getFamilyArray(),cell.getFamilyOffset(),cell.getFamilyLength());
String colName = Bytes.toString(cell.getQualifierArray(),cell.getQualifierOffset(),cell.getQualifierLength());
String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
user.setId(row);
if(colName.equals("username")){
user.setUsername(value);
}
if(colName.equals("age")){
user.setAge(value);
}
if (colName.equals("gender")){
user.setGender(value);
}
if (colName.equals("phone")){
user.setPhone(value);
}
if (colName.equals("email")){
user.setEmail(value);
}
}
list.add(user);
}
} catch (IOException e) {
e.printStackTrace();
}
return list;
}
//删除指定cell数据
public static void deleteByRowKey(String tableName, String rowKey) throws IOException {
Table table = initHbase().getTable(TableName.valueOf(tableName));
Delete delete = new Delete(Bytes.toBytes(rowKey));
//删除指定列
//delete.addColumns(Bytes.toBytes("contact"), Bytes.toBytes("email"));
table.delete(delete);
}
//删除表
public static void deleteTable(String tableName){
try {
TableName tablename = TableName.valueOf(tableName);
admin = initHbase().getAdmin();
admin.disableTable(tablename);
admin.deleteTable(tablename);
} catch (IOException e) {
e.printStackTrace();
}
}
}
HBase 过滤器
基础API中的查询操作在面对大量数据的时候是非常苍白的，这里Hbase提供了高级的查询方法：Filter。Filter可以根据簇、列、版本等更多的条件来对数据进行过滤，基于Hbase本身提供的三维有序（主键有序、列有序、版本有序），这些Filter可以高效的完成查询过滤的任务。带有Filter条件的RPC查询请求会把Filter分发到各个RegionServer，是一个服务器端（Server-side）的过滤器，这样也可以降低网络传输的压力。
要完成一个过滤的操作，至少需要两个参数。一个是抽象的操作符，Hbase提供了枚举类型的变量来表示这些抽象的操作符：LESS/LESS_OR_EQUAL/EQUAL/NOT_EUQAL等；另外一个就是具体的比较器（Comparator），代表具体的比较逻辑，如果可以提高字节级的比较、字符串级的比较等。有了这两个参数，我们就可以清晰的定义筛选的条件，过滤数据。
抽象操作符（比较运算符）
LESS <
LESS_OR_EQUAL <=
EQUAL =
NOT_EQUAL <>
GREATER_OR_EQUAL >=
GREATER >
NO_OP 排除所有
比较器（指定比较机制）
BinaryComparator 按字节索引顺序比较指定字节数组，采用 Bytes.compareTo(byte[])
BinaryPrefixComparator 跟前面相同，只是比较左端的数据是否相同
NullComparator 判断给定的是否为空
BitComparator 按位比较
RegexStringComparator 提供一个正则的比较器，仅支持 EQUAL 和非 EQUAL
SubstringComparator 判断提供的子串是否出现在 value 中
java编码
输出结果
user-006/contact:email/1546667518090/Put/vlen=16/seqid=0
user-006/contact:phone/1546667518090/Put/vlen=11/seqid=0
user-006/information:age/1546667518090/Put/vlen=2/seqid=0
user-006/information:gender/1546667518090/Put/vlen=3/seqid=0
user-006/information:password/1546667518090/Put/vlen=6/seqid=0
user-006/information:username/1546667518090/Put/vlen=7/seqid=0
user-007/contact:email/1546667518117/Put/vlen=16/seqid=0
user-007/contact:phone/1546667518117/Put/vlen=11/seqid=0
user-007/information:age/1546667518117/Put/vlen=2/seqid=0
user-007/information:gender/1546667518117/Put/vlen=6/seqid=0
user-007/information:password/1546667518117/Put/vlen=6/seqid=0
user-007/information:username/1546667518117/Put/vlen=7/seqid=0
user-008/contact:email/1546667518144/Put/vlen=16/seqid=0
user-008/contact:phone/1546667518144/Put/vlen=11/seqid=0
user-008/information:age/1546667518144/Put/vlen=2/seqid=0
user-008/information:gender/1546667518144/Put/vlen=3/seqid=0
user-008/information:password/1546667518144/Put/vlen=6/seqid=0
user-008/information:username/1546667518144/Put/vlen=7/seqid=0
user-009/contact:email/1546667518173/Put/vlen=16/seqid=0
user-009/contact:phone/1546667518173/Put/vlen=11/seqid=0
user-009/information:age/1546667518173/Put/vlen=2/seqid=0
user-009/information:gender/1546667518173/Put/vlen=6/seqid=0
user-009/information:password/1546667518173/Put/vlen=6/seqid=0
user-009/information:username/1546667518173/Put/vlen=7/seqid=0
列簇过滤器 FamilyFilter :过滤info信息----------------------------------------------------------------------
user-000/contact:email/1546667517904/Put/vlen=16/seqid=0
user-000/contact:phone/1546667517904/Put/vlen=11/seqid=0
user-001/contact:email/1546667517960/Put/vlen=16/seqid=0
user-001/contact:phone/1546667517960/Put/vlen=11/seqid=0
user-002/contact:email/1546667517996/Put/vlen=16/seqid=0
user-002/contact:phone/1546667517996/Put/vlen=11/seqid=0
user-003/contact:email/1546667518018/Put/vlen=16/seqid=0
user-003/contact:phone/1546667518018/Put/vlen=11/seqid=0
user-004/contact:email/1546667518041/Put/vlen=16/seqid=0
user-004/contact:phone/1546667518041/Put/vlen=11/seqid=0
user-005/contact:email/1546667518067/Put/vlen=16/seqid=0
user-005/contact:phone/1546667518067/Put/vlen=11/seqid=0
user-006/contact:email/1546667518090/Put/vlen=16/seqid=0
user-006/contact:phone/1546667518090/Put/vlen=11/seqid=0
user-007/contact:email/1546667518117/Put/vlen=16/seqid=0
user-007/contact:phone/1546667518117/Put/vlen=11/seqid=0
user-008/contact:email/1546667518144/Put/vlen=16/seqid=0
user-008/contact:phone/1546667518144/Put/vlen=11/seqid=0
user-009/contact:email/1546667518173/Put/vlen=16/seqid=0
user-009/contact:phone/1546667518173/Put/vlen=11/seqid=0
值过滤器 ValueFilter :过滤性别为女的数据----------------------------------------------------------------------
user-001/information:gender/1546667517960/Put/vlen=6/seqid=0
user-003/information:gender/1546667518018/Put/vlen=6/seqid=0
user-005/information:gender/1546667518067/Put/vlen=6/seqid=0
user-007/information:gender/1546667518117/Put/vlen=6/seqid=0
user-009/information:gender/1546667518173/Put/vlen=6/seqid=0
时间戳过滤器 TimestampsFilter ValueFilter :过滤时间戳为1546665288963的数据----------------------------------------------------------------------
单列值过滤器 SingleColumnValueFilter :过滤名字为hbase_4的数据----------------------------------------------------------------------
user-004 contact email base_4@hbase.com 1546667518041
user-004 contact phone 13355550024 1546667518041
user-004 information age 20 1546667518041
user-004 information gender man 1546667518041
user-004 information password 123456 1546667518041
user-004 information username hbase_4 1546667518041
前缀过滤器 PrefixFilter----只对行键有用----------------------------------------------------------------------
user-000 contact email base_0@hbase.com 1546667517904
user-000 contact phone 13355550020 1546667517904
user-000 information age 20 1546667517904
user-000 information gender man 1546667517904
user-000 information password 123456 1546667517904
user-000 information username hbase_0 1546667517904
user-001 contact email base_1@bmaoo.com 1546667517960
user-001 contact phone 13355550021 1546667517960
user-001 information age 21 1546667517960
user-001 information gender female 1546667517960
user-001 information password 123456 1546667517960
user-001 information username hbase_1 1546667517960
user-002 contact email base_2@hbase.com 1546667517996
user-002 contact phone 13355550022 1546667517996
user-002 information age 20 1546667517996
user-002 information gender man 1546667517996
user-002 information password 123456 1546667517996
user-002 information username hbase_2 1546667517996
user-003 contact email base_3@bmaoo.com 1546667518018
user-003 contact phone 13355550023 1546667518018
user-003 information age 21 1546667518018
user-003 information gender female 1546667518018
user-003 information password 123456 1546667518018
user-003 information username hbase_3 1546667518018
user-004 contact email base_4@hbase.com 1546667518041
user-004 contact phone 13355550024 1546667518041
user-004 information age 20 1546667518041
user-004 information gender man 1546667518041
user-004 information password 123456 1546667518041
user-004 information username hbase_4 1546667518041
user-005 contact email base_5@bmaoo.com 1546667518067
user-005 contact phone 13355550025 1546667518067
user-005 information age 21 1546667518067
user-005 information gender female 1546667518067
user-005 information password 123456 1546667518067
user-005 information username hbase_5 1546667518067
user-006 contact email base_6@hbase.com 1546667518090
user-006 contact phone 13355550026 1546667518090
user-006 information age 20 1546667518090
user-006 information gender man 1546667518090
user-006 information password 123456 1546667518090
user-006 information username hbase_6 1546667518090
user-007 contact email base_7@bmaoo.com 1546667518117
user-007 contact phone 13355550027 1546667518117
user-007 information age 21 1546667518117
user-007 information gender female 1546667518117
user-007 information password 123456 1546667518117
user-007 information username hbase_7 1546667518117
user-008 contact email base_8@hbase.com 1546667518144
user-008 contact phone 13355550028 1546667518144
user-008 information age 20 1546667518144
user-008 information gender man 1546667518144
user-008 information password 123456 1546667518144
user-008 information username hbase_8 1546667518144
user-009 contact email base_9@bmaoo.com 1546667518173
user-009 contact phone 13355550029 1546667518173
user-009 information age 21 1546667518173
user-009 information gender female 1546667518173
user-009 information password 123456 1546667518173
user-009 information username hbase_9 1546667518173
列前缀过滤器 ColumnPrefixFilter匹配password,phone两个----------------------------------------------------------------------
user-000 contact phone 13355550020 1546667517904
user-000 information password 123456 1546667517904
user-001 contact phone 13355550021 1546667517960
user-001 information password 123456 1546667517960
user-002 contact phone 13355550022 1546667517996
user-002 information password 123456 1546667517996
user-003 contact phone 13355550023 1546667518018
user-003 information password 123456 1546667518018
user-004 contact phone 13355550024 1546667518041
user-004 information password 123456 1546667518041
user-005 contact phone 13355550025 1546667518067
user-005 information password 123456 1546667518067
user-006 contact phone 13355550026 1546667518090
user-006 information password 123456 1546667518090
user-007 contact phone 13355550027 1546667518117
user-007 information password 123456 1546667518117
user-008 contact phone 13355550028 1546667518144
user-008 information password 123456 1546667518144
user-009 contact phone 13355550029 1546667518173
user-009 information password 123456 1546667518173
更多hadoop进阶源码请参看我的github地址
springboot-habse操作整合
https://gitee.com/susonglin/springboot_hbase_kafka
1727
扫一扫
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
