HBase 1.1.2 优化插入 Region预分配-优快云博客

本文介绍了一种使用Java API向HBase批量插入大量记录的方法，并展示了如何预先创建分区以优化数据分布。通过实例代码详细解释了从创建表、设置预分区到执行批量插入的具体步骤。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

  秒插入   并且蛋疼的是相当于人为制造的网络风暴。

package com.rocky.util;
import com.rocky.dao.HBaseFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.UUID;
/**
* Created by rocky_24 on 2016/2/25.
*/
public class putData {
public static void main(String[] args) throws IOException {
// /**
// * 分16
// */
// byte[][] preforkRegions = new byte[10+6][];
// int k=0;
// System.out.println(preforkRegions.length);
// for(char i='0';i<='9';i++){
// preforkRegions[k++]=(""+i).getBytes();
// }
// for(char i='a';i<='f';i++){
// preforkRegions[k++]=(""+i).getBytes();
// }
byte[][] preforkRegions = new byte[3][];
preforkRegions [0] = ("5").getBytes();
preforkRegions [1] = ("10").getBytes();
preforkRegions [2] = ("z").getBytes();
createTable("access_logs","f",preforkRegions);
System.out.println(TimeUtils.getIntradayDateAndTime());
String [] pages = {"/","/a.html","/b.html","/c.html"};
Configuration con = HBaseFactory.getConf();
HTable table = new HTable(con,"access_logs");
// Table table = HBaseFactory.getHBaseConnection().getTable(TableName.valueOf("access_logs"));
table.setWriteBufferSize(1024 * 1024 * 6);
table.setAutoFlushTo(false);
int totalRecords = 10000;
Random rand = new Random();
System.out.println("importing " + totalRecords + " records ....");
List<Put> list = new ArrayList<Put>();
for (int i=0; i < totalRecords * 100; i++) {
String rowkey = UUID.randomUUID().toString();
String randomPage = pages[rand.nextInt(pages.length)];
Put put = new Put(rowkey.getBytes());
put.addColumn(Bytes.toBytes("f"), null, Bytes.toBytes(randomPage));
list.add(put);
}
System.out.println("put数据装载完毕："+list.size());
System.out.println(TimeUtils.getIntradayDateAndTime());
table.put(list);
table.flushCommits();
table.close();
System.out.println("done");
System.out.println(TimeUtils.getIntradayDateAndTime());
}
/**
* 创建表
* @param tableName
* @param columnFamily
* @param spilts
* @throws IOException
*/
public static void createTable(String tableName, String columnFamily, byte[][] spilts) throws IOException {
Connection connection = HBaseFactory.getHBaseConnection();
Admin admin = connection.getAdmin();
if (admin.tableExists(TableName.valueOf(tableName))) {
admin.disableTable(TableName.valueOf(tableName));
admin.deleteTable(TableName.valueOf(tableName));
}
HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(tableName));
tableDesc.addFamily(new HColumnDescriptor(columnFamily));
if (spilts == null) {
admin.createTable(tableDesc);
} else {
admin.createTable(tableDesc, spilts);
}
admin.close();
}
}