copy a voldemort tools

最新推荐文章于 2019-05-22 09:34:00 发布

转载最新推荐文章于 2019-05-22 09:34:00 发布 · 155 阅读

本文介绍了一个用于Voldemort集群管理的工具类实现，该工具能够执行诸如获取节点上的所有键、按前缀过滤键以及删除键等操作。通过该工具可以更高效地管理和维护Voldemort集群。

a tools class to find objects in voldemort and massive
operations.

Bye

package com.components.repository;

import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.net.Socket;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import voldemort.VoldemortException;
import voldemort.client.protocol.RequestFormatType;
import voldemort.client.protocol.admin.AdminClient;
import voldemort.client.protocol.admin.AdminClientConfig;
import voldemort.client.protocol.pb.ProtoUtils;
import voldemort.client.protocol.pb.VAdminProto;
import voldemort.cluster.Node;
import voldemort.store.socket.SocketAndStreams;
import voldemort.store.socket.SocketDestination;
import voldemort.store.socket.SocketPool;
import voldemort.utils.ByteUtils;

import com.components.util.Config;

public class VoldemortAdmin {
public static final int DEFAULT_MAX_CONNECTIONS_PER_NODE = 10;
public static final int DEFAULT_MAX_CONNECTIONS = 50;
public static final int DEFAULT_SOCKET_BUFFER_SIZE = 32 * 1024;
public static final int DEFAULT_CONNECTION_TIMEOUT_MS = 1000000;
public static final int DEFAULT_SO_TIMEOUT_MS = 1000000;
private SocketPool pool;
private AdminClient adminClient;
private List storeNames;
private VoldemortClient voldemortClient;
private List partitionIdList;

public VoldemortAdmin() {
super();
init();
}

private void init() {
setVoldemortClient(new VoldemortClient());
adminClient = new
AdminClient(Config.getInstance().getDataRepositoryUrl(), new
AdminClientConfig());
storeNames = new ArrayList();
storeNames.add(Config.getInstance().getDataRepositoryStoreName());
partitionIdList = new ArrayList();
partitionIdList.add(0);
partitionIdList.add(1);
setPool(new SocketPool(DEFAULT_MAX_CONNECTIONS_PER_NODE,
DEFAULT_CONNECTION_TIMEOUT_MS, DEFAULT_SO_TIMEOUT_MS,
DEFAULT_SOCKET_BUFFER_SIZE));
}

public VoldemortAdmin(SocketPool poolExtern, List
storeNamesExtern) {
super();
setVoldemortClient(new VoldemortClient());
adminClient = new
AdminClient(Config.getInstance().getDataRepositoryUrl(), new
AdminClientConfig());
partitionIdList = new ArrayList();
partitionIdList.add(0);
partitionIdList.add(1);
setPool(poolExtern);
setStoreNames(storeNamesExtern);
}
public VoldemortAdmin(List storeNamesExtern, AdminClient
adminClientExtern) {
super();
setVoldemortClient(new VoldemortClient());
adminClient = adminClientExtern;
partitionIdList = new ArrayList();
partitionIdList.add(0);
partitionIdList.add(1);
setPool(new SocketPool(DEFAULT_MAX_CONNECTIONS_PER_NODE,
DEFAULT_CONNECTION_TIMEOUT_MS, DEFAULT_SO_TIMEOUT_MS,
DEFAULT_SOCKET_BUFFER_SIZE));
setStoreNames(storeNamesExtern);
}
public Set getAllKeysByNode(int nodeId) {
List stores = storeNames;
Set keys = new HashSet();
for (String storeName : stores) {
Node node =
adminClient.getAdminClientCluster().getNodeById(nodeId);
final SocketDestination destination = new
SocketDestination(node.getHost(), node.getAdminPort(),
RequestFormatType.ADMIN_PROTOCOL_BUFFERS);
final SocketAndStreams sands = pool.checkout(destination);
DataOutputStream outputStream = sands.getOutputStream();
final DataInputStream inputStream = sands.getInputStream();
try {
initiateFetchRequest(outputStream, storeName, partitionIdList);
} catch (IOException e) {
close(sands.getSocket());
pool.checkin(destination, sands);
throw new VoldemortException(e);
}
try {
while (true) {
int size = inputStream.readInt();
if (size == -1) {
pool.checkin(destination, sands);
break;
}
VAdminProto.FetchPartitionEntriesResponse response =
responseFromStream(inputStream, size);
if (response.hasError()) {
pool.checkin(destination, sands);
System.out.println(response.getError().getErrorMessage());
} else {
keys.add(response.getKey().toStringUtf8());
}
}
} catch (IOException e) {
close(sands.getSocket());
pool.checkin(destination, sands);
throw new VoldemortException(e);
}
}
return keys;
}

public void deleteAllKeysByNode(int nodeId) {
List stores = storeNames;
for (String storeName : stores) {
Node node =
adminClient.getAdminClientCluster().getNodeById(nodeId);
final SocketDestination destination = new
SocketDestination(node.getHost(), node.getAdminPort(),
RequestFormatType.ADMIN_PROTOCOL_BUFFERS);
final SocketAndStreams sands = pool.checkout(destination);
DataOutputStream outputStream = sands.getOutputStream();
final DataInputStream inputStream = sands.getInputStream();
try {
initiateFetchRequest(outputStream, storeName, partitionIdList);
} catch (IOException e) {
close(sands.getSocket());
pool.checkin(destination, sands);
throw new VoldemortException(e);
}
try {
while (true) {
int size = inputStream.readInt();
if (size == -1) {
pool.checkin(destination, sands);
break;
}
VAdminProto.FetchPartitionEntriesResponse response =
responseFromStream(inputStream, size);
if (response.hasError()) {
pool.checkin(destination, sands);
System.out.println(response.getError().getErrorMessage());
} else {
voldemortClient.delete(response.getKey().toStringUtf8());
}
}
} catch (IOException e) {
close(sands.getSocket());
pool.checkin(destination, sands);
throw new VoldemortException(e);
}
}
}

public Set getAllKeysByPrefixAndByNode(int nodeId, String
prefix) {
List stores = storeNames;
Set keys = new HashSet();
for (String storeName : stores) {
Node node =
adminClient.getAdminClientCluster().getNodeById(nodeId);
final SocketDestination destination = new
SocketDestination(node.getHost(), node.getAdminPort(),
RequestFormatType.ADMIN_PROTOCOL_BUFFERS);
final SocketAndStreams sands = pool.checkout(destination);
DataOutputStream outputStream = sands.getOutputStream();
final DataInputStream inputStream = sands.getInputStream();
try {
initiateFetchRequest(outputStream, storeName, partitionIdList);
} catch (IOException e) {
close(sands.getSocket());
pool.checkin(destination, sands);
throw new VoldemortException(e);
}
try {
while (true) {
int size = inputStream.readInt();
if (size == -1) {
pool.checkin(destination, sands);
break;
}
VAdminProto.FetchPartitionEntriesResponse response =
responseFromStream(inputStream, size);
if (response.hasError()) {
pool.checkin(destination, sands);
System.out.println(response.getError().getErrorMessage());
} else {
if (response.getKey().toStringUtf8().startsWith(prefix)) {
keys.add(response.getKey().toStringUtf8());
}
}
}
} catch (IOException e) {
close(sands.getSocket());
pool.checkin(destination, sands);
throw new VoldemortException(e);
}
}
return keys;
}

public void deleteAllKeysByPrefixAndByNode(int nodeId, String prefix)
{
List stores = storeNames;
for (String storeName : stores) {
Node node =
adminClient.getAdminClientCluster().getNodeById(nodeId);
final SocketDestination destination = new
SocketDestination(node.getHost(), node.getAdminPort(),
RequestFormatType.ADMIN_PROTOCOL_BUFFERS);
final SocketAndStreams sands = pool.checkout(destination);
DataOutputStream outputStream = sands.getOutputStream();
final DataInputStream inputStream = sands.getInputStream();
try {
initiateFetchRequest(outputStream, storeName, partitionIdList);
} catch (IOException e) {
close(sands.getSocket());
pool.checkin(destination, sands);
throw new VoldemortException(e);
}
try {
while (true) {
int size = inputStream.readInt();
if (size == -1) {
pool.checkin(destination, sands);
break;
}
VAdminProto.FetchPartitionEntriesResponse response =
responseFromStream(inputStream, size);
if (response.hasError()) {
pool.checkin(destination, sands);
System.out.println(response.getError().getErrorMessage());
} else {
if (response.getKey().toStringUtf8().startsWith(prefix)) {
voldemortClient.delete(response.getKey().toStringUtf8());
}
}
}
} catch (IOException e) {
close(sands.getSocket());
pool.checkin(destination, sands);
throw new VoldemortException(e);
}
}
}

public Set getAllKeysByNodes(List nodesId) {
Set listAll = new HashSet();
for (Integer nodeId : nodesId) {
listAll.addAll(getAllKeysByNode(nodeId));
}
return listAll;
}

private void initiateFetchRequest(DataOutputStream outputStream,
String storeName, List partitionList) throws IOException {
VAdminProto.FetchPartitionEntriesRequest.Builder fetchRequest =
VAdminProto.FetchPartitionEntriesRequest.newBuilder().addAllPartitions(partitionList).setStore(storeName);
VAdminProto.VoldemortAdminRequest request =
VAdminProto.VoldemortAdminRequest.newBuilder().setType(VAdminProto.AdminRequestType.FETCH_PARTITION_ENTRIES).setFetchPartitionEntries(fetchRequest).build();
ProtoUtils.writeMessage(outputStream, request);
outputStream.flush();
}

private void close(Socket socket) {
try {
socket.close();
} catch (IOException e) {
e.printStackTrace();
}
}

private VAdminProto.FetchPartitionEntriesResponse
responseFromStream(DataInputStream inputStream, int size) throws
IOException {
byte[] input = new byte[size];
ByteUtils.read(inputStream, input);
VAdminProto.FetchPartitionEntriesResponse.Builder response =
VAdminProto.FetchPartitionEntriesResponse.newBuilder();
response.mergeFrom(input);
return response.build();
}

public void setPool(SocketPool pool) {
this.pool = pool;
}

public SocketPool getPool() {
return pool;
}

public void setAdminClient(AdminClient adminClient) {
this.adminClient = adminClient;
}

public AdminClient getAdminClient() {
return adminClient;
}

public void setPartitionIdList(List partitionIdList) {
this.partitionIdList = partitionIdList;
}

public List getPartitionIdList() {
return partitionIdList;
}

public List getStoreNames() {
return storeNames;
}

public void setStoreNames(List storeNames) {
this.storeNames = storeNames;
}

public void setVoldemortClient(VoldemortClient voldemortClient) {
this.voldemortClient = voldemortClient;
}

public VoldemortClient getVoldemortClient() {
return voldemortClient;
}

}

[@more@]voldemort