HBase常用服务调用封装

一、前言

      生产刚刚接入HBase，应用对其数据的获取的几种方式如get，scan，scan range进行了相关服务封装

二、服务封装

   

package com.hbase.sources;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.RegexStringComparator;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.util.Bytes;
import java.util.ConcurrentHashMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;

import com.hbase.utils.StringUtilsx;

/**
 * 
 * 类名: HbaseService.java 
 * 描述:Hbase数据获取操作类
 * @author tanjie 
 * 创建时间: 2017年4月12日 上午10:44:37
 * @version V1.0.0
 */
@Service("hbaseSource")
public class HBaseSource {
	
	private final Logger LOGGER = LoggerFactory.getLogger(HBaseSource.class);

	/**
	 * HbaseSource连接池
	 */
	private static Map<String, HBaseSource> hbasepool = new ConcurrentHashMap<String, HBaseSource>();

	/**
	 * HbaseSource实例创建时间池
	 */
	private static Map<String,Date> createTimes = new ConcurrentHashMap<String, Date>();
	
	/**
	 * HbaseSource实例访问时间池
	 */
	private static Map<String,Date> lastTimes = new ConcurrentHashMap<String,Date>();
	
	/**
	 *  hbase连接的配置属性
	 */
	private Configuration conf;

	/** 
	 * hbase的连接
	 */
	private Connection connection;
	
	
	/**
	 * 创建时间多长就清除连接
	 */
	private long intervalsTime = 4 * 1000 * 60 * 60;
	
	/**
	 * 多长时间没调用就清除连接
	 */
//	private long instanceTime = 2*60*1000;
	
	
	/**
	 * 
	 * @author tanjie
	 * 创建时间:  2017年4月15日 上午10:56:28
	 * 描述: 定时销毁创建时间过久或空闲时间过长的连接
	 */
	@Scheduled(cron ="0 0/5 * * * ?")
	public void clearHbaseConnection(){
		LOGGER.info("定时任务开始工作!");
		LOGGER.info("clear hbase connection hbasepool :" + hbasepool.size());
		LOGGER.info("clear hbase connection createTimes :" + createTimes.size());
		LOGGER.info("clear hbase connection lastTimes :" + lastTimes.size());
	    Date currentDate = new Date();
	    if(null != createTimes && createTimes.size() !=0){
	    	for(final Map.Entry<String, Date> ctime : createTimes.entrySet()){
	    		if(null != ctime.getValue() && currentDate.getTime()-ctime.getValue().getTime()>=intervalsTime){
	    			LOGGER.info("clear hbase connection 创建时间太长被清理 :" + ctime.getKey());
	    		    hbasepool.get(ctime).closeConnection();
	    		    hbasepool.remove(ctime);
	    		    createTimes.remove(ctime.getKey());
	    		    lastTimes.remove(ctime.getKey());
	    		}
	    	}
	    }
	    if(null != lastTimes && lastTimes.size()!=0){
			for(final Map.Entry<String,Date> m:lastTimes.entrySet()){
				if(m.getValue()!=null&& currentDate.getTime()-m.getValue().getTime()>=1000*60*20){
					LOGGER.info("clear hbase connection 调用时间间隔太长被清理 :" + m.getKey());
					hbasepool.get(m.getKey()).closeConnection();
					hbasepool.remove(m.getKey());
					createTimes.remove(m.getKey());
					lastTimes.remove(m.getKey());
				}
			}
		}
	}

	/**
	 * 
	 * @author tanjie
	 * 创建时间:  2017年4月15日 上午9:11:54
	 * 描述: 创建表
	 * @param hbaseTableName 表名
	 * @param columnFamily 列族民
	 * @throws IOException
	 */
	public void createHbaseTable(String hbaseTableName, String columnFamily)
			throws IOException {
		if(null == connection){
			return;
		}
		Admin admin = connection.getAdmin();
		TableName tableName = TableName.valueOf(hbaseTableName);
		if (admin.tableExists(tableName)) {
			return;
		}
		HTableDescriptor hTableDescriptor = new HTableDescriptor(tableName);
		HColumnDescriptor hColumnDescriptor = new HColumnDescriptor(
				columnFamily);
		hTableDescriptor.addFamily(hColumnDescriptor);
		admin.createTable(hTableDescriptor);
		boolean avail = admin.isTableAvailable(tableName);
		LOGGER.info("创建的表是否可用：" + avail);
		admin.close();
	}
	
	
	/**
	 * 通过表名和rowkey查询所需数据
	 * @param tableName hbase的表名
	 * @param keyData hbase rowkey的List集合
	 * @return Map<String, Map<String, byte[]>>  
	 * 外层的Map key值是Rowkey的值   内层map key值是列族:列   value是该列对应的值
	 * @throws HasNotHbasePremission 
	 * @throws IOException 
	 * @exception  java.io.Exception
	 */
	public  Map<String, Map<String, byte[]>> searchData(String tableName,
			                                            List<String> keyData) throws IOException{
		Map<String, Map<String,byte[]>> re = new HashMap<String, Map<String,byte[]>>();
		Table table=connection.getTable(TableName.valueOf(tableName));
		List<Get> list = new ArrayList<Get>();
		for(String strKey : keyData){
			Get get = new Get(strKey.getBytes());
			list.add(get);
		}
		Result[] rs=table.get(list);
		if(rs==null){
			return re;
		}
		for(Result r:rs){
			Map<String,byte[]> tmpmap = new HashMap<String,byte[]>();
			if(r!=null&&r.getRow()!=null){
				String rowkey = new String(r.getRow());
				Cell[] cells=r.rawCells();
				for(Cell cell: cells){
					tmpmap.put(new String(CellUtil.cloneFamily(cell)) +":" + 
							   new String(CellUtil.cloneQualifier(cell)),
							              CellUtil.cloneValue(cell));
				}
				re.put(rowkey, tmpmap);
			}
		}
		table.close();
		return re;
	}
	

	/**
	 * 
	 * @author tanjie
	 * 创建时间:  2017年4月15日 上午9:56:35
	 * 描述: 根据表名，rowkey模拟查询数据 
	 * 根据rowkey模糊匹配查询数据 Map<String,Map<String,byte[]>>
	 * @param tableName 表名
	 * @param rowkey 表对应的rowkey
	 * @return  Map<String,Map<String,byte[]>> 外层Map key是rowkey,里层Map key是列，value是具体数据
	 */
	public Map<String, Map<String, byte[]>> searchData(String tableName,
			String rowkey) {
		if(null == connection){
			return null;
		}
		HashMap<String, Map<String, byte[]>> result = new HashMap<String, Map<String, byte[]>>();
		Table table = null;
		ResultScanner rs = null;
		try {
			table = connection.getTable(TableName.valueOf(tableName));
			Scan scan = new Scan();
			if (StringUtilsx.isNotEmpty(rowkey)) {
				Filter filter = new RowFilter(CompareOp.EQUAL,
						new RegexStringComparator(rowkey));
				scan.setFilter(filter);
			}
			rs = table.getScanner(scan);
			for (final Result r : rs) {
				String keyData = Bytes.toString(r.getRow());
				Cell[] cells = r.rawCells();
				Map<String, byte[]> tmpMap = new HashMap<String, byte[]>();
				for (final Cell cell : cells) {
					tmpMap.put(new String(CellUtil.cloneFamily(cell)) + ":" +
				               new String(CellUtil.cloneQualifier(cell)),
				                CellUtil.cloneValue(cell));
				}
				result.put(keyData, tmpMap);
			}
		} catch (IOException e) {
			LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";rowkey:" + rowkey
					+ ";异常:"+ e.getMessage(),e);
		} finally{
			if(null != table){
				try {
					table.close();
				} catch (IOException e) {
					LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";rowkey:" + rowkey
							+ ";异常:"+ e.getMessage(),e);
				}
			}
			if(null != rs){
				rs.close();
			}
		}
		return result;
	}
	
	/**
	 * 
	 * @author tanjie
	 * 创建时间:  2017年6月1日 下午4:20:16
	 * 描述: 根据表名，rowkey查询单个具体的信息
	 * @param tableName 表名
	 * @param rowkey rowkey
	 * @return Result Result对象
	 */
	public Result searchDataByGet(String tableName,
			String rowkey) {
		if(null == connection){
			return null;
		}
		Table table = null;
		try {
			table = connection.getTable(TableName.valueOf(tableName));
			Get get = new Get(rowkey.getBytes());
			return table.get(get);
		} catch (IOException e) {
			LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";rowkey:" + rowkey
					+ ";异常:"+ e.getMessage(),e);
		} finally{
			if(null != table){
				try {
					table.close();
				} catch (IOException e) {
					LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";rowkey:" + rowkey
							+ ";异常:"+ e.getMessage(),e);
				}
			}
		}
		return null;
	}

	/**
	 * 
	 * @author tanjie
	 * 创建时间:  2017年4月15日 上午10:09:33
	 * 描述: 查询某张表从startRowKey到entRowkey区间的数据
	 * @param tableName 表名
	 * @param startRowKey 开始rowkey
	 * @param endRowKey 结束rowkey
	 * @return Map<String,Map<String,byte[]>> 外层Map key是rowkey,里层Map key是列，value是具体数据
	 */
	public Map<String, Map<String, byte[]>> searchData(String tableName,
			String startRowKey, String stopRowKey) {
		if(null == connection){
			return null;
		}
		HashMap<String, Map<String, byte[]>> result = new HashMap<String, Map<String, byte[]>>();
		Table table = null;
		ResultScanner rs = null;
		try {
			table = connection.getTable(TableName.valueOf(tableName));
			Scan scan = new Scan();
			if (StringUtilsx.isNotEmpty(startRowKey)) {
				scan.setStartRow(Bytes.toBytes(startRowKey));
			}
			if (StringUtilsx.isNotEmpty(stopRowKey)) {
				scan.setStopRow(Bytes.toBytes(stopRowKey));
			}
			rs = table.getScanner(scan);
			for (final Result r : rs) {
				String keyData = new String(r.getRow());
				Cell[] cells = r.rawCells();
				Map<String, byte[]> tmpMap = new HashMap<String, byte[]>();
				for (final Cell cell : cells) {
					tmpMap.put(new String(CellUtil.cloneFamily(cell)) + ":" +
				               new String(CellUtil.cloneQualifier(cell)),
				                CellUtil.cloneValue(cell));
				}
				result.put(keyData, tmpMap);
			}
		} catch (IOException e) {
			LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";startRowkey:" + startRowKey
					+";stopRowKey:" + stopRowKey + ";异常:"+ e.getMessage(),e);
		} finally{
			if(null != table){
				try {
					table.close();
				} catch (IOException e) {
					LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";startRowkey:" + startRowKey
							+";stopRowKey:" + stopRowKey + ";异常:"+ e.getMessage(),e);
				}
			}
			if(null != rs){
				rs.close();
			}
		}
		return result;
	}

	/**
	 * 
	 * @author tanjie 
	 * 创建时间: 2017年4月13日 下午8:18:35 
	 * 描述: 查询打标数据表
	 * @param tableName 表名
	 * @param startRowKey 开始rowkey
	 * @param stopRowKey 结束rowkey
	 * @param keyData1 rowkey包含的数据1
	 * @param keyData2  rowkey包含的数据2
	 * @return Map<String,Map<String,byte[]>> 外层Map key是rowkey,里层Map key是列，value是具体数据
	 */
	public Map<String, Map<String, byte[]>> searchData(String tableName,
			String startRowKey, String stopRowKey, String keyData1,
			String keyData2) {
		if(null == connection){
			return null;
		}
		HashMap<String, Map<String, byte[]>> result = new HashMap<String, Map<String, byte[]>>();
		Table table = null;
		ResultScanner rs = null;
		try {
			table = connection.getTable(TableName.valueOf(tableName));
			Scan scan = new Scan();
			if (StringUtilsx.isNotEmpty(startRowKey)) {
				scan.setStartRow(Bytes.toBytes(startRowKey));
			}
			if (StringUtilsx.isNotEmpty(stopRowKey)) {
				scan.setStopRow(Bytes.toBytes(stopRowKey));
			}
			rs = table.getScanner(scan);
			for (final Result r : rs) {
				String rowKeyData = new String(r.getRow());
				if (StringUtilsx.isNotEmpty(rowKeyData)) {
					if ((StringUtilsx.isNotEmpty(keyData1) && rowKeyData
							.contains(keyData1))
							|| (StringUtilsx.isNotEmpty(keyData2) && rowKeyData
									.contains(keyData2))) {
						Cell[] cells = r.rawCells();
						Map<String, byte[]> tmpMap = new HashMap<String, byte[]>();
						for (final Cell cell : cells) {
							tmpMap.put(new String(CellUtil.cloneFamily(cell)) + ":" +
						               new String(CellUtil.cloneQualifier(cell)),
						                CellUtil.cloneValue(cell));
						}
						result.put(rowKeyData, tmpMap);
					}
				}
			}
		} catch (IOException e) {
			LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";startRowkey:" + startRowKey
					+";stopRowKey:" + stopRowKey + ";keyData1:" + keyData1 + ";keyData2:" + keyData2+  ";异常:"+ e.getMessage(),e);
		} finally{
			if(null != table){
				try {
					table.close();
				} catch (IOException e) {
					LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";startRowkey:" + startRowKey
							+";stopRowKey:" + stopRowKey + ";keyData1:" + keyData1 + ";keyData2:" + keyData2+  ";异常:"+ e.getMessage(),e);
				}
			}
			if(null != rs){
				rs.close();
			}
		}
		return result;
	}

	/**
	 * 
	 * @author tanjie
	 * 创建时间:  2017年4月15日 上午10:34:37
	 * 描述: 通过rowkey前缀搜索数据
	 * @param tableName 表名
	 * @param prefixRowKey 前缀
	 * @return Map<String,Map<String,byte[]>> 外层Map key是rowkey,里层Map key是列，value是具体数据
	 */
	public Map<String, Map<String, byte[]>> searchDataByPrefix(
			String tableName, String prefixRowKey) {
		if(null == connection){
			return null;
		}
		HashMap<String, Map<String, byte[]>> result = new HashMap<String, Map<String, byte[]>>();
		Table table = null;
		ResultScanner rs = null;
		try {
			table = connection.getTable(TableName.valueOf(tableName));
			Scan scan = new Scan();
			if (StringUtilsx.isNotEmpty(prefixRowKey)) {
				scan.setRowPrefixFilter(prefixRowKey.getBytes());
			}
			rs = table.getScanner(scan);
			for (final Result r : rs) {
				String keyData = new String(r.getRow());
				Cell[] cells = r.rawCells();
				Map<String, byte[]> tmpMap = new HashMap<String, byte[]>();
				for (final Cell cell : cells) {
					tmpMap.put(new String(CellUtil.cloneFamily(cell)) + ":" +
				               new String(CellUtil.cloneQualifier(cell)),
				                CellUtil.cloneValue(cell));
				}
				result.put(keyData, tmpMap);
			}
		}catch (IOException e) {
			LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";prefixRowKey:" + prefixRowKey + ";异常:"+ e.getMessage(),e);
		} finally{
			if(null != table){
				try {
					table.close();
				} catch (IOException e) {
					LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";prefixRowKey:" + prefixRowKey + ";异常:"+ e.getMessage(),e);
				}
			}
			if(null != rs){
				rs.close();
			}
		}
		return result;
	}
	
	
	/**
	 * 
	 * @author tanjie
	 * 创建时间:  2017年4月15日 上午10:34:37
	 * 描述: 通过rowkey前缀搜索数据
	 * @param tableName 表名
	 * @param prefixRowKey 前缀
	 * @param keyData1 包含数据
	 * @return Map<String,Map<String,byte[]>> 外层Map key是rowkey,里层Map key是列，value是具体数据
	 */
	public Map<String, Map<String, byte[]>> searchDataByPrefix(
			String tableName, String prefixRowKey,String keyData1) {
		if(null == connection){
			return null;
		}
		HashMap<String, Map<String, byte[]>> result = new HashMap<String, Map<String, byte[]>>();
		Table table = null;
		ResultScanner rs = null;
		try {
			table = connection.getTable(TableName.valueOf(tableName));
			Scan scan = new Scan();
			if (StringUtilsx.isNotEmpty(prefixRowKey)) {
				scan.setRowPrefixFilter(prefixRowKey.getBytes());
			}
			rs = table.getScanner(scan);
			for (final Result r : rs) {
				String rowKeyData = new String(r.getRow());
				if (StringUtilsx.isNotEmpty(rowKeyData)) {
					if ((StringUtilsx.isNotEmpty(keyData1) && rowKeyData
							.contains(keyData1))) {
						Cell[] cells = r.rawCells();
						Map<String, byte[]> tmpMap = new HashMap<String, byte[]>();
						for (final Cell cell : cells) {
							tmpMap.put(new String(CellUtil.cloneFamily(cell)) + ":" +
						               new String(CellUtil.cloneQualifier(cell)),
						                CellUtil.cloneValue(cell));
						}
						result.put(rowKeyData, tmpMap);
					}
				}
			}
		}catch (IOException e) {
			LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";prefixRowKey:" + prefixRowKey + ";异常:"+ e.getMessage(),e);
		} finally{
			if(null != table){
				try {
					table.close();
				} catch (IOException e) {
					LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";prefixRowKey:" + prefixRowKey + ";异常:"+ e.getMessage(),e);
				}
			}
			if(null != rs){
				rs.close();
			}
		}
		return result;
	}

	/**
	 * 
	 * @author tanjie
	 * 创建时间:  2017年6月1日 下午4:19:27
	 * 描述: 失效一张表
	 * @param disableTableName
	 */
	public void disableTable(String disableTableName){
		TableName tableName = TableName.valueOf(disableTableName);
		Admin admin = null;
		try {
			admin = connection.getAdmin();
			admin.disableTable(tableName);
		} catch (IOException e) {
			e.printStackTrace();
		} finally{
			if(null !=  admin){
				try {
					admin.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
	}
	
	/**
	 * 
	 * @author tanjie
	 * 创建时间:  2017年6月1日 下午4:19:40
	 * 描述: 删除一张表
	 * @param deleteTableName
	 */
	public void deleteTable(String deleteTableName){
		TableName tableName = TableName.valueOf(deleteTableName);
		Admin admin = null;
		try {
			admin = connection.getAdmin();
			admin.deleteTable(tableName);
		} catch (IOException e) {
			e.printStackTrace();
		} finally{
			if(null !=  admin){
				try {
					admin.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
	}

	/**
	 * 
	 * @author tanjie 创建时间: 2017年4月13日 上午10:56:30 描述: 批量插入
	 * @param tableName
	 * @param list
	 * @throws IOException
	 * @throws InterruptedException
	 */
	public void batchPut(String tableName, List<Put> list){
		Table table = null;
		try {
			if(null == connection){
				return;
			}
			table = connection.getTable(TableName.valueOf(tableName));
			if (null == table) {
				return;
			}
			int size = list.size();
			table.batch(list, new Object[size]);
		} catch (IOException e) {
			e.printStackTrace();
		} catch (InterruptedException e) {
			e.printStackTrace();
		} finally{
			if(null != table){
				try {
					table.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
	}

	
	/**
	 * 
	 * @author tanjie 创建时间: 2017年4月13日 上午10:56:30 描述: 单个put操作
	 * @param tableName 表名
	 * @param put  Put
	 * @throws IOException
	 */
	public void singlePut(String tableName, Put put){
		Table table = null;
		try {
			if(null == connection){
				return;
			}
			table = connection.getTable(TableName.valueOf(tableName));
			if (null == table) {
				return;
			}
			table.put(put);
		} catch (IOException e) {
			e.printStackTrace();
		} finally{
			if(null != table){
				try {
					table.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
	}

	/**
	 * 
	 * @author tanjie 
	 * 创建时间: 2017年4月13日 上午10:56:39 
	 * 描述: 添加每一行数据
	 * @param put Put对象
	 * @param family 列族
	 * @param column 列
	 * @param value 值
	 */
	public void addColumn(Put put, String family, String column, String value) {
		put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column),
				System.currentTimeMillis(), Bytes.toBytes(value));
	}

	/**
	 * 
	 * @author tanjie 
	 * 创建时间: 2017年4月10日 下午6:52:13
	 * 描述: 获取某个应用对应的hbaseSource
	 * @throws IOException
	 */
	public static HBaseSource getHbaseSource(String appName) throws IOException {
		HBaseSource hbaseSource = hbasepool.get(appName);
		Date date = new Date();
		if (null == hbaseSource) {
			hbaseSource = new HBaseSource();
			hbasepool.put(appName, hbaseSource);
			createTimes.put(appName,date);
			lastTimes.put(appName, date);
			return hbaseSource;
		}
		lastTimes.put(appName, date);
		return hbaseSource;
	}

	/**
	 * 
	 * @author tanjie
	 * 创建时间:  2017年4月15日 上午10:32:13
	 * 描述: 打开连接
	 * @throws IOException
	 */
	public void openConnection(){
		try {
			if (null != connection) {
				return;
			}
			conf = HBaseConfiguration.create();
			connection = ConnectionFactory.createConnection(conf);
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	/**
	 * 
	 * @author tanjie
	 * 创建时间:  2017年4月15日 上午11:51:05
	 * 描述: 判断数据源连接是否打开
	 * @return  boolean
	 */
	public boolean isOpen(){
		if(null == connection || connection.isClosed() ){
			return false;
		}
		return true;
	}
	/**
	 * 关闭与数据源的连接
	 */
	public void closeConnection() {
		if (null != connection) {
			try {
				connection.close();
				connection = null;
			} catch (IOException e) {
				e.printStackTrace();
			} finally {
				connection = null;
			}
		}
	}

	/**
	 * 获取hbase的连接属性
	 * 
	 * @return Configuration this.conf 连接属性
	 */
	public Configuration getConfiguration() {
		return this.conf;
	}

	/**
	 * 构造hbase的连接配置属性
	 * 
	 * @param config
	 *            连接属性
	 */
	public void setConfiguration(Configuration config) {
		this.conf = config;
	}

}