使用spring项目的HbaseTemplate对hbase进行复杂查询（or 和and组合查询）

1.hbase连接管理类:

package com.xcsqjr.acs.web.config;

import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.hadoop.hbase.HbaseTemplate;

import java.io.File;
import java.io.IOException;

/**
 * Hbase连接管理类
 * @author ycg
 *
 */
@Configuration
public class HbaseConfiguration {
    @Value("${hbase.zookeeper.property.clientPort}")
    private String clientPort;

    @Value("${hbase.zookeeper.quorum}")
    private String quorum;

    @Value("${hbase.master}")
    private String master;

    @Bean
    public HbaseTemplate getHbaseTemplate() {
        org.apache.hadoop.conf.Configuration configuration = new org.apache.hadoop.conf.Configuration();
        configuration = org.apache.hadoop.hbase.HBaseConfiguration.create();
        configuration.set("hbase.zookeeper.property.clientPort", clientPort);//Hbase中zookeeper的端口号;默认是2181
        configuration.set("hbase.zookeeper.quorum", quorum);//hadoop的地址,这里需要在系统的host文件配置如:hadoop1,hadoop2,host文件中未:192.168.0.1 hadoop1  192.168.0.2 hadoop2
        configuration.set("hbase.master", master);
//        configuration.set("hbase.client.scanner.timeout.period", "10000");


        File workaround = new File(".");
        System.getProperties().put("hadoop.home.dir",
                workaround.getAbsolutePath());
//        new File("./bin").mkdirs();
        new File(".".concat(File.separator).concat("bin")).mkdirs();
        try {
            new File(".".concat(File.separator).concat("bin").concat(File.separator).concat("winutils.exe")).createNewFile();
        } catch (IOException e) {
            e.printStackTrace();
        }

        HbaseTemplate ht = new HbaseTemplate(configuration);
        return ht;
    }

}

 

2.HbaseTemplateAPI

package com.xcsqjr.acs.web.api;

import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;

import java.util.List;
import java.util.Map;

/**
 * hbaseTemplate服务接口
 * @author ycg
 *
 */
public interface  IHbaseTemplateService {

    /**
     * 在HBase上面创建表
     * @param tableName  表名
     * @param family 列族名(可以同时传入多个列族名)
     * @return
     */
    public boolean createTable(String tableName,String ...family);

    /**
     * Scan 查询所有的hbase数据
     * @param tableName 表名
     * @param <T> 返回数据类型
     * @return
     */
    public <T> List<T> searchAll(String tableName, Class<T> c);

    /**
     * 向表中插入数据
     * @param object
     * @param tableName
     * @param family
     * @param rowkey
     * @return
     */
    public Object createPro(Object object,String tableName,String family,String rowkey);

    /**
     * 通过表名和rowkey获取一行数据转object
     * @param <T> 数据类型
     * @param tableName 表名
     * @param rowkey
     * @return
     */
    public <T> T getOneToClass(Class<T> c,String tableName,String rowkey);

    /**
     * 根据表名组合查询
     * @param c
     * @param tableName
     * @param filterList 查询条件过滤器列表
     * @param <T>
     * @return
     */
    public <T>List<T> getListByCondition(Class<T> c, String tableName, FilterList filterList);

    /**
     * 通过表名和rowkey获取一行map数据
     * @param tableName
     * @param rowName
     * @return
     */
    public Map<String, Object> getOneToMap(String tableName, String rowName);

    /**
     * 查询一条记录一个column的值
     * @param tableName 表名
     * @param rowkey
     * @param family 列族
     * @param column 列
     * @return
     */
    public String getColumn(String tableName,String rowkey,String family,String column);

    /**
     * 查询开始row和结束row之间的数据
     * @param <T> 数据类型
     * @param tableName 表名
     * @param startRow 开始row
     * @param endRow 结束row
     * @return
     */
    public <T> List<T> findByRowRange(Class<T> c,String tableName,String startRow,String endRow);


    /**
     * *SingleColumnValueFilter scvf = new SingleColumnValueFilter(
     * Bytes.toBytes(family),  //搜索哪个列族
     * Bytes.toBytes(column),   //搜素哪一列
     * CompareFilter.CompareOp.EQUAL, //对比关系
     * Bytes.toBytes(Keywords)); //这里传入 SubstringComparator 比较器,搜索的结果为列值(value)包含关键字,传入bytes数组,则进行完全匹配
     * scvf.setLatestVersionOnly(true); //属性设置为true时,如果查询的列族下,没有colume这个列,则不返回这行数据,反之就返回这行数据
     * @param tableName
     * @param scvf
     * @param clazz
     * @param <T>
     * @return
     */
    public <T> List<T> searchAllByFilter(Class<T> clazz,String tableName,SingleColumnValueFilter scvf);

3.HbaseTemplate实现类

package com.xcsqjr.acs.web.service;

import com.xcsqjr.acs.web.api.IHbaseTemplateService;

import java.beans.PropertyDescriptor;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.util.Bytes;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.BeanWrapper;
import org.springframework.beans.PropertyAccessorFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.hadoop.hbase.HbaseTemplate;
import org.springframework.data.hadoop.hbase.RowMapper;
import org.springframework.data.hadoop.hbase.TableCallback;
import org.springframework.stereotype.Service;



import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;

/**
 * @author ycg
 *
 */
@Service
public class HbaseTemplateService implements IHbaseTemplateService {
    @Autowired
    private HbaseTemplate hbaseTemplate;

    @SuppressWarnings({"resource", "deprecation"})
    @Override
    public boolean createTable(String tableName, String... column) {
        HBaseAdmin admin;
        try {
            // 从hbaseTemplate 获取configuration对象,用来初始化admin
            admin = new HBaseAdmin(hbaseTemplate.getConfiguration());
            HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);
            for (int i = 0; i < column.length; i++) {
                tableDescriptor.addFamily(new HColumnDescriptor(column[i]));
            }
            admin.createTable(tableDescriptor);
            return admin.tableExists(tableName);
        } catch (MasterNotRunningException e) {
            e.printStackTrace();
        } catch (ZooKeeperConnectionException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return false;
    }

    @Override
    public <T> List<T> searchAll(String tableName, Class<T> c) {
        return hbaseTemplate.find(tableName, new Scan(), new RowMapper<T>() {
            @Override
            public T mapRow(Result result, int rowNum) throws Exception {
                T pojo = c.newInstance();
                BeanWrapper beanWrapper = PropertyAccessorFactory.forBeanPropertyAccess(pojo);
                List<Cell> ceList = result.listCells();
                for (Cell cellItem : ceList) {
                    String cellName = new String(CellUtil.cloneQualifier(cellItem));
                    if (!"class".equals(cellName)) {
                        beanWrapper.setPropertyValue(cellName, new String(CellUtil.cloneValue(cellItem)));
                    }
                }
                return pojo;
            }
        });
    }

    @Override
    public Object createPro(Object pojo, String tableName, String column, String rowkey) {
        if (pojo == null || StringUtils.isBlank(tableName) || StringUtils.isBlank(column)) {
            return null;
        }
        return hbaseTemplate.execute(tableName, new TableCallback<Object>() {
            @Override
            public Object doInTable(HTableInterface table) throws Throwable {
                PropertyDescriptor[] pds = BeanUtils.getPropertyDescriptors(pojo.getClass());
                BeanWrapper beanWrapper = PropertyAccessorFactory.forBeanPropertyAccess(pojo);
                Put put = new Put(Bytes.toBytes(rowkey));
                for (PropertyDescriptor propertyDescriptor : pds) {
                    String properName = propertyDescriptor.getName();
                    String value = beanWrapper.getPropertyValue(properName).toString();
                    if (!StringUtils.isBlank(value)) {
                        put.add(Bytes.toBytes(column), Bytes.toBytes(properName), Bytes.toBytes(value));
                    }
                }
                table.put(put);
                return null;
            }
        });
    }

    @Override
    public <T> T getOneToClass(Class<T> c, String tableName, String rowkey) {
        if (c == null || StringUtils.isBlank(tableName) || StringUtils.isBlank(rowkey)) {
            return null;
        }
        return hbaseTemplate.get(tableName, rowkey, new RowMapper<T>() {
            public T mapRow(Result result, int rowNum) throws Exception {
                List<Cell> ceList = result.listCells();
                JSONObject obj = new JSONObject();
                T item = c.newInstance();
                if (ceList != null && ceList.size() > 0) {
                    for (Cell cell : ceList) {
//                        cell.getTimestamp();
//                        //rowKey
//                        CellUtil.cloneRow(cell);
                        obj.put(
                                Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(),
                                        cell.getQualifierLength()),
                                Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
                    }
                } else {
                    return null;
                }
                item = JSON.parseObject(obj.toJSONString(), c);
                return item;
            }
        });
    }

    @Override
    public <T>List<T> getListByCondition(Class<T> c, String tableName, FilterList filterList){
        if (c == null || StringUtils.isBlank(tableName)) {
            return null;
        }
//        List<Filter>  list=new ArrayList<>();
//        String targetSet=jsonObject.getString("targetSet");
//        String targetSonSet=jsonObject.getString("targetSonSet");
//        String target=jsonObject.getString("target");
//        if(StringUtils.isNotBlank(targetSet)){
//            list.add(new SingleColumnValueFilter(Bytes.toBytes("targetSet"),null,
//                    CompareFilter.CompareOp.EQUAL,Bytes.toBytes(targetSet)));
//        }
//        if(StringUtils.isNotBlank(targetSonSet)){
//            list.add(new SingleColumnValueFilter(Bytes.toBytes("targetSonSet"),null,
//                    CompareFilter.CompareOp.EQUAL,Bytes.toBytes(targetSonSet)));
//        }
//        if(StringUtils.isNotBlank(target)){
//            list.add(new SingleColumnValueFilter(Bytes.toBytes("target"),null,
//                    CompareFilter.CompareOp.EQUAL,Bytes.toBytes(target)));
//        }
//        FilterList filterList=new FilterList(list);
        Scan scan=new Scan();
        scan.setFilter(filterList);
        return hbaseTemplate.find(tableName, scan, new RowMapper<T>() {
            @Override
            public T mapRow(Result result, int rowNum) throws Exception {
                T pojo = c.newInstance();
                BeanWrapper beanWrapper = PropertyAccessorFactory.forBeanPropertyAccess(pojo);
                List<Cell> ceList = result.listCells();
                for (Cell cellItem : ceList) {
                    String cellName = new String(CellUtil.cloneQualifier(cellItem));
                    if (!"class".equals(cellName)) {
                        beanWrapper.setPropertyValue(cellName, new String(CellUtil.cloneValue(cellItem)));
                    }
                }
                return pojo;
            }
        });
    }


    @Override
    public Map<String, Object> getOneToMap(String tableName, String rowName) {
        return hbaseTemplate.get(tableName, rowName,new RowMapper<Map<String,Object>>(){
            @Override
            public Map<String, Object> mapRow(Result result, int i) throws Exception {
                List<Cell> ceList =   result.listCells();
                Map<String,Object> map = new HashMap<String, Object>();
                if(ceList!=null&&ceList.size()>0){
                    for(Cell cell:ceList){
                        map.put(Bytes.toString(cell.getFamilyArray(),cell.getFamilyOffset(),cell.getFamilyLength())+
                                        "_"+Bytes.toString( cell.getQualifierArray(),cell.getQualifierOffset(),cell.getQualifierLength()),
                                Bytes.toString( cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
                    }
                }
                return map;
            }
        });
    }

    @Override
    public String getColumn(String tableName, String rowkey, String family, String column) {
        if (StringUtils.isBlank(tableName) || StringUtils.isBlank(family)
                || StringUtils.isBlank(rowkey) || StringUtils.isBlank(column)) {
            return null;
        }
        return hbaseTemplate.get(tableName, rowkey, family, column, new RowMapper<String>() {
            public String mapRow(Result result, int rowNum) throws Exception {
                List<Cell> ceList = result.listCells();
                String res = "";
                if (ceList != null && ceList.size() > 0) {
                    for (Cell cell : ceList) {
                        res =
                                Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
                    }
                }
                return res;
            }
        });
    }

    @Override
    public <T> List<T> findByRowRange(Class<T> c, String tableName, String startRow, String endRow) {
        if (c == null || StringUtils.isBlank(tableName) || StringUtils.isBlank(startRow)
                || StringUtils.isBlank(endRow)) {
            return null;
        }
        Scan scan = new Scan();
        scan.setStartRow(Bytes.toBytes(startRow));
        scan.setStopRow(Bytes.toBytes(endRow));
        scan.setCacheBlocks(false);
        scan.setCaching(2000);
        return hbaseTemplate.find(tableName, scan, new RowMapper<T>() {
            @Override
            public T mapRow(Result result, int rowNum) throws Exception {
                T pojo = c.newInstance();
                BeanWrapper beanWrapper = PropertyAccessorFactory.forBeanPropertyAccess(pojo);
                List<Cell> ceList = result.listCells();
                for (Cell cellItem : ceList) {
                    String cellName = new String(CellUtil.cloneQualifier(cellItem));
                    if (!"class".equals(cellName)) {
                        beanWrapper.setPropertyValue(cellName, new String(CellUtil.cloneValue(cellItem)));
                    }
                }
                return pojo;
            }
        });
    }

    @Override
    public <T> List<T> searchAllByFilter(Class<T> clazz,String tableName,SingleColumnValueFilter scvf) {
        Scan scan = new Scan();
//        scan.addFamily(Bytes.toBytes(family));
        scan.setFilter(scvf);
        return hbaseTemplate.find(tableName, scan, new RowMapper<T>() {
            @Override
            public T mapRow(Result result, int rowNum) throws Exception {
//        T pojo = c.newInstance();
//        BeanWrapper beanWrapper = PropertyAccessorFactory.forBeanPropertyAccess(pojo);
//        List<Cell> ceList = result.listCells();
//        for (Cell cellItem : ceList) {
//          String cellName = new String(CellUtil.cloneQualifier(cellItem));
//          if (!"class".equals(cellName)) {
//            beanWrapper.setPropertyValue(cellName, new String(CellUtil.cloneValue(cellItem)));
//          }
//        }
//        return pojo;
                List<Cell> ceList = result.listCells();
                JSONObject obj = new JSONObject();
                T item = clazz.newInstance();
                if (ceList != null && ceList.size() > 0) {
                    for (Cell cell : ceList) {
                        obj.put(
                                Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(),
                                        cell.getQualifierLength()),
                                Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
                    }
                } else {
                    return null;
                }
                item = JSON.parseObject(obj.toJSONString(), clazz);
                return item;
            }
        });
    }
}

 

4.业务API：

package com.xcsqjr.acs.web.api;

import java.io.Serializable;

/**
 * Hbase提供给omc的 API
 * @author ycg
 *
 */
public interface IHbaseToOmcService extends Serializable {

    /**
     * 获取指标表相关信息
     * @param jsonString
     * @return
     * @throws Exception
     */
    Serializable getTargetInfos       (String jsonString)        throws Exception;


}

5. 业务实现类：or 和and组合查询

此处查询相当于where (a=? and b=? c=? ) or (a=? and b=? c=? )

package com.xcsqjr.acs.web.service;

import com.alibaba.fastjson.JSONObject;
import com.xcsqjr.acs.domain.entity.TargetParamsVo;
import com.xcsqjr.acs.domain.entity.TargetVO;
import com.xcsqjr.acs.web.api.IHbaseTemplateService;
import com.xcsqjr.acs.web.api.IHbaseToOmcService;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.SubstringComparator;
import org.apache.hadoop.hbase.util.Bytes;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.io.Serializable;
import java.util.List;

/**
 * @author ycg
 *
 */
@Service
public class HbaseToOmcService implements  IHbaseToOmcService{


    private static final long serialVersionUID = 1042946646492744467L;

    @Autowired
    private IHbaseTemplateService hbaseTemplateService;

    @Override
    public Serializable getTargetInfos(String jsonString) throws Exception {
        JSONObject jsonObject=JSONObject.parseObject(jsonString);
         List<TargetParamsVo> paramsVoList=jsonObject.getJSONArray("targetParams").toJavaList(TargetParamsVo.class);


//        List<Filter> list=new ArrayList<>();
//        List<Filter> cellIdList=new ArrayList<>();
//        List<Filter> targetSetList=new ArrayList<>();
//        List<Filter> targetSonSetList=new ArrayList<>();
//        List<Filter> targetList=new ArrayList<>();

        String targetSet=null;
        String targetSonSet=null;
        String target=null;
        String cellId=null;
        FilterList filterList=new FilterList(FilterList.Operator.MUST_PASS_ONE);
        FilterList queryFilters=null;
        for(TargetParamsVo tpv:paramsVoList){
            targetSet=tpv.getTargetSet();
            targetSonSet=tpv.getTargetSonSet();
            target=tpv.getTarget();
            cellId=tpv.getCellId();

            queryFilters=new FilterList(FilterList.Operator.MUST_PASS_ALL);
            if(StringUtils.isNotBlank(cellId)){
                queryFilters.addFilter(new SingleColumnValueFilter(Bytes.toBytes("info"),Bytes.toBytes("cellId"),
                        CompareFilter.CompareOp.EQUAL,new SubstringComparator(cellId)));

            }
            if(StringUtils.isNotBlank(targetSet)){
                queryFilters.addFilter(new SingleColumnValueFilter(Bytes.toBytes("info"),Bytes.toBytes("targetSet"),
                        CompareFilter.CompareOp.EQUAL,new SubstringComparator(targetSet)));
            }
            if(StringUtils.isNotBlank(targetSonSet)){
                queryFilters.addFilter(new SingleColumnValueFilter(Bytes.toBytes("info"),Bytes.toBytes("targetSonSet"),
                        CompareFilter.CompareOp.EQUAL,new SubstringComparator(targetSonSet)));

            }
            if(StringUtils.isNotBlank(target)){
                queryFilters.addFilter(new SingleColumnValueFilter(Bytes.toBytes("info"),Bytes.toBytes("target"),
                        CompareFilter.CompareOp.EQUAL,new SubstringComparator(target)));

            }
//        list.add( new RowFilter(CompareFilter.CompareOp.EQUAL, new SubstringComparator("cell")));

//        https://blog.youkuaiyun.com/lr131425/article/details/72676254
            //     new BinaryPrefixComparator(value) //匹配字节数组前缀
////       new RegexStringComparator(expr) // 正则表达式匹配
//     new SubstringComparator(substr)// 子字符串匹配
//            new FamilyFilter(CompareFilter.CompareOp.EQUAL,new BinaryComparator(Bytes.toBytes("info")));

            filterList.addFilter(queryFilters);

        }

        //配置查询过滤器
//        FilterList cellIdFilters=new FilterList(FilterList.Operator.MUST_PASS_ONE,cellIdList);
//        FilterList targetSetFilters=new FilterList(FilterList.Operator.MUST_PASS_ONE,targetSetList);
//        FilterList targetSonSetFilters=new FilterList(FilterList.Operator.MUST_PASS_ONE,targetSonSetList);
//        FilterList targetFilters=new FilterList(FilterList.Operator.MUST_PASS_ONE,targetList);

        List<TargetVO> rsList=hbaseTemplateService.getListByCondition(TargetVO.class,"target",filterList);



        return (Serializable)rsList;
    }
}

参考博客：

https://blog.youkuaiyun.com/lr131425/article/details/72676254 等等