Hbase Endpoint Coprocessor

最新推荐文章于 2022-11-21 12:37:57 发布

tom_fans

最新推荐文章于 2022-11-21 12:37:57 发布

阅读量502

点赞数

CC 4.0 BY-SA版权

分类专栏： Hbase 文章标签： hbase

本文链接：https://blog.youkuaiyun.com/tom_fans/article/details/78104889

Hbase 专栏收录该内容

15 篇文章

订阅专栏

本文介绍如何使用 HBase 的 Endpoint Coprocessor 实现远程过程调用 (RPC) 来获取表中特定列的求和结果。涵盖 proto 文件定义、Java 类生成、Endpoint 类编写、客户端代码实现及协处理器加载等步骤。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

Endpoint Coprocessor比observer Coprocessor要复杂很多，这里不做具体解释了，大家慢慢看代码仔细琢磨。另外此篇文章代码是从HBASE官方文档拷贝过来的。但是官方文档实际也没有给出很详细的解释。

1) 定义proto文件

// 定义常用选项
option java_package = "com.isesol.mapreduce";   //指定生成Java代码的包名
option java_outer_classname = "Sum";    //指定生成Java代码的外部类名称
option java_generic_services = true;    //基于服务定义产生抽象服务代码
option optimize_for = SPEED;    //指定优化级别
// 定义请求包
message SumRequest {
        required string family = 1;     //列族
        required string column = 2;     //列名
}
// 定义回复包
message SumResponse {
        required int64 sum = 1 [default = 0];   //求和结果
}
// 定义RPC服务
service SumService {
        //获取求和结果
        rpc getSum(SumRequest)
                returns (SumResponse);
}

2) 通过proto命令生成Sum.java类, 并拷贝到项目中

protoc endpoint.proto --java_out=./

3)编写endpoint 类

package com.isesol.mapreduce;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
import org.apache.hadoop.hbase.coprocessor.CoprocessorException;
import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.protobuf.ResponseConverter;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.Cell;  
import org.apache.hadoop.hbase.CellUtil;  
import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
import com.google.protobuf.Service;
import com.isesol.mapreduce.Sum.SumRequest;
import com.isesol.mapreduce.Sum.SumResponse;
import com.isesol.mapreduce.Sum.SumService;

public class endpointTrigger extends SumService implements Coprocessor, CoprocessorService {

	private RegionCoprocessorEnvironment env;

	@Override
	public void getSum(RpcController controller, SumRequest request, RpcCallback<SumResponse> done) {
		// TODO Auto-generated method stub

		SumResponse response = null;
		InternalScanner scanner = null;
		Scan scan = new Scan();
		scan.addFamily(Bytes.toBytes(request.getFamily()));
		scan.addColumn(Bytes.toBytes(request.getFamily()), Bytes.toBytes(request.getColumn()));
		// 扫描每个region，取值后求和
		try {
			scanner = env.getRegion().getScanner(scan);
			List<Cell> results = new ArrayList<Cell>();
			boolean hasMore = false;
			Long sum = 0L;
			do {
				hasMore = scanner.next(results);
				for (Cell cell : results) {
					sum += Long.parseLong(new String(CellUtil.cloneValue(cell)));
				}
				results.clear();
			} while (hasMore);
			// 设置返回结果
			response = SumResponse.newBuilder().setSum(sum).build();
		} catch (IOException e) {
			ResponseConverter.setControllerException(controller, e);
		} finally {
			if (scanner != null) {
				try {
					scanner.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}

		done.run(response);
	}

	public Service getService() {
		// TODO Auto-generated method stub
		return this;
	}

	public void start(CoprocessorEnvironment env) throws IOException {
		// TODO Auto-generated method stub

		if (env instanceof RegionCoprocessorEnvironment) {
			this.env = (RegionCoprocessorEnvironment) env;
		} else {
			throw new CoprocessorException("can not start endpoint");
		}

	}

	public void stop(CoprocessorEnvironment env) throws IOException {
		// TODO Auto-generated method stub

	}

}

4)编写客户端

package com.isesol.mapreduce;

import java.io.IOException;
import java.util.Map;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;

import com.google.protobuf.ServiceException;
import com.isesol.mapreduce.Sum.SumRequest;
import com.isesol.mapreduce.Sum.SumResponse;
import com.isesol.mapreduce.Sum.SumService;;


/**
 * @author developer
 * 说明：hbase协处理器endpooint的客户端代码
 * 功能：从服务端获取对hbase表指定列的数据的求和结果
 */
public class Sumclient {

    public static void main(String[] args) throws ServiceException, Throwable {
        
        long sum = 0L;
        
        int count = 0;
        
        // 配置HBse
        Configuration conf = HBaseConfiguration.create();
        conf.set("hbase.zookeeper.quorum",
				"datanode01.isesol.com,datanode02.isesol.com,datanode03.isesol.com,datanode04.isesol.com,cmserver.isesol.com");
        conf.set("hbase.zookeeper.property.clientPort", "2181");
        // 建立一个数据库的连接
        Connection conn = ConnectionFactory.createConnection(conf);
        // 获取表
        HTable table = (HTable) conn.getTable(TableName.valueOf("t_axes_feeding_status"));
        table.setOperationTimeout(10);
        // 设置请求对象
        final SumRequest request = SumRequest.newBuilder().setFamily("cf").setColumn("AXABSFBPOS A").build();
        
        
        System.out.println("start to invoke result");
        
        // 获得返回值
        Map<byte[], Long> result = table.coprocessorService(Sum.SumService.class, null, null, 
                new Batch.Call<Sum.SumService, Long>() {

                    public Long call(SumService service) throws IOException {
                        BlockingRpcCallback<SumResponse> rpcCallback = new BlockingRpcCallback<SumResponse>();
                        service.getSum(null, request, rpcCallback);
                        SumResponse response = (SumResponse) rpcCallback.get();
                        return response.hasSum() ? response.getSum() : 0L;
                    }
        });
        
        System.out.println("satrt to count the value");
        // 将返回值进行迭代相加
        for (Long v : result.values()) {
        	
            count++;
        }
        // 结果输出
        System.out.println("count: " + count);
        // 关闭资源
        table.close();
        conn.close();
    }

}

5)加载协处理器

  alter 't_axes_feeding_status', METHOD => 'table_att', 'COPROCESSOR'=> 'hdfs://nameservice1/tmp/endpointTrigger.jar|com.isesol.mapreduce.endpointTrigger||'
  
  alter  't_axes_feeding_status',   METHOD => 'table_att_unset', NAME => 'coprocessor$1'

6) 执行client代码，看看返回的数据是否正确。