发布一个k8s部署视频：https://edu.youkuaiyun.com/course/detail/26967
课程内容：各种k8s部署方式。包括minikube部署，kubeadm部署，kubeasz部署，rancher部署，k3s部署。包括开发测试环境部署k8s，和生产环境部署k8s。
腾讯课堂连接地址https://ke.qq.com/course/478827?taid=4373109931462251&tuin=ba64518
第二个视频发布 https://edu.youkuaiyun.com/course/detail/27109
腾讯课堂连接地址https://ke.qq.com/course/484107?tuin=ba64518
介绍主要的k8s资源的使用配置和命令。包括configmap，pod，service，replicaset，namespace，deployment，daemonset，ingress，pv，pvc，sc，role，rolebinding，clusterrole，clusterrolebinding，secret，serviceaccount，statefulset，job，cronjob，podDisruptionbudget，podSecurityPolicy,networkPolicy,resourceQuota，limitrange，endpoint，event，conponentstatus，node，apiservice，controllerRevision等。
第三个视频发布：https://edu.youkuaiyun.com/course/detail/27574
详细介绍helm命令，学习helm chart语法，编写helm chart。深入分析各项目源码，学习编写helm插件
————————————————------------------------------------------------------------------------------------------------------------------
public class HBaseConnectionPool {
protected static ConcurrentHashMap<String,HBaseConnectionEntity> idelConnections=null;
protected static ConcurrentHashMap<String,HBaseConnectionEntity> activeConnections=null;
protected static int initSize;
protected static int maxSize;
protected static AtomicInteger idelSize=new AtomicInteger(0);
protected static AtomicInteger activeSize=new AtomicInteger(0);
protected static HBaseConnectionPool instance=null;
protected static Lock lock= new ReentrantLock();
protected Object object=new Object();
protected static volatile boolean isShutdown=false;
private HBaseConnectionPool(int initSize,int maxSize){
this.initSize=initSize;
this.maxSize=maxSize;
idelConnections=new ConcurrentHashMap<String,HBaseConnectionEntity>();
activeConnections=new ConcurrentHashMap<String,HBaseConnectionEntity>();
initConnections();
new HBaseDetectFailConnection().start();
}
public HBaseConnectionEntity getConnection(){
if(isShutdown){
throw new RuntimeException("pool is shutdown.");
}
lock.lock();
try{
if(idelSize.get()>0){
if(idelConnections.size()<=0){
throw new RuntimeException("");
}
Entry<String,HBaseConnectionEntity> entry=idelConnections.entrySet().iterator().next();
String key=entry.getKey();
HBaseConnectionEntity entity=entry.getValue();
entity.setStatus(HBase_Connection_Status.active);
idelConnections.remove(key);
idelSize.decrementAndGet();
if(entity.getConnection().isClosed()){
return getConnection();
}
activeConnections.put(key, entity);
activeSize.incrementAndGet();
return entity;
}
}finally{
lock.unlock();
}
if(activeSize.get()>maxSize){
throw new RuntimeException("活跃数量大于最大值");
}
if(activeSize.get()>=maxSize){
synchronized (object) {
try {
object.wait();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
return getConnection();
}
if(isShutdown){
throw new RuntimeException("pool is shutdown.");
}
Connection conn=HBaseUtils.getConnection();
String id=UUID.randomUUID().toString();
HBaseConnectionEntity entity=new HBaseConnectionEntity();
entity.setId(id);
entity.setConnection(conn);
entity.setStatus(HBase_Connection_Status.active);
activeConnections.put(id, entity);
activeSize.incrementAndGet();
return entity;
}
private void initConnections(){
for(int i=0;i<this.initSize;i++){
HBaseConnectionEntity entity=new HBaseConnectionEntity();
String id=UUID.randomUUID().toString();
entity.setId(id);
Connection conn=HBaseUtils.getConnection();
if(conn==null){
continue;
}
entity.setConnection(conn);
entity.setStatus(HBase_Connection_Status.idel);
idelConnections.put(id, entity);
idelSize.getAndAdd(1);
}
}
public static HBaseConnectionPool getInstance(){
if(isShutdown){
throw new RuntimeException("pool is already shutdown.");
}
if(instance!=null){
return instance;
}
return getInstance(40,40);
}
public static HBaseConnectionPool getInstance(int initSize,int maxSize){
if(isShutdown){
throw new RuntimeException("pool is already shutdown.");
}
if(initSize<0||maxSize<1){
throw new RuntimeException("initSize必须不小于0，maxsize必须大于等于1");
}
if(initSize>maxSize){
initSize=maxSize;
}
synchronized (ConnectionPool.class) {
if(instance==null){
instance=new HBaseConnectionPool(initSize,maxSize);
}
}
return instance;
}
public void releaseConnection(String id){
if(isShutdown){
throw new RuntimeException("pool is shutdown.");
}
if(idelSize.get()==maxSize){
HBaseUtils.closeConnection(activeConnections.remove(id).getConnection());
}else{
HBaseConnectionEntity entity=activeConnections.remove(id);
entity.setStatus(HBase_Connection_Status.idel);
idelConnections.put(id, entity);
idelSize.incrementAndGet();
activeSize.decrementAndGet();
synchronized (object) {
object.notify();
}
}
}
public void shutdown(){
isShutdown=true;
synchronized (object) {
object.notifyAll();
}
Iterator<String> idelIt=idelConnections.keySet().iterator();
while(idelIt.hasNext()){
String key=idelIt.next();
HBaseConnectionEntity entity=idelConnections.get(key);
HBaseUtils.closeConnection(entity.getConnection());
}
Iterator<String>activeIt=activeConnections.keySet().iterator();
while(activeIt.hasNext()){
String key=activeIt.next();
HBaseConnectionEntity entity=activeConnections.get(key);
HBaseUtils.closeConnection(entity.getConnection());
}
initSize=0;
maxSize=0;
idelSize=new AtomicInteger(0);
activeSize=new AtomicInteger(0);
}
public int getIdelSize(){
return this.idelSize.get();
}
public int getActiveSize(){
return this.activeSize.get();
}
public static class HBaseConnectionEntity{
private String id;
private Connection connection;
private HBase_Connection_Status status;
public Connection getConnection() {
return connection;
}
public void setConnection(Connection connection) {
this.connection = connection;
}
public HBase_Connection_Status getStatus() {
return status;
}
public void setStatus(HBase_Connection_Status status) {
this.status = status;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
}
enum HBase_Connection_Status{
idel,active,close
}
class HBaseDetectFailConnection extends Thread{
@Override
public void run() {
Iterator<String> itIdel=idelConnections.keySet().iterator();
while(itIdel.hasNext()){
String key =itIdel.next();
HBaseConnectionEntity entity=idelConnections.get(key);
if(entity.getConnection().isClosed()){
idelConnections.remove(key);
idelSize.decrementAndGet();
}
}
Iterator<String> itActive=activeConnections.keySet().iterator();
while(itActive.hasNext()){
String key=itActive.next();
HBaseConnectionEntity entity=activeConnections.get(key);
if(entity.getConnection().isClosed()){
activeConnections.remove(key);
activeSize.decrementAndGet();
}
}
try {
Thread.sleep(10000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}
public class HBaseUtils {
private static Configuration conf = HBaseConfiguration.create();
private static ExecutorService poolx=Executors.newFixedThreadPool(300);
public static Connection getConnection(){
int i=0;
Connection conn=null;
do{
try {
conn=ConnectionFactory.createConnection(conf, poolx);
if(conn!=null){
break;
}
Thread.sleep(100);
i++;
} catch(InterruptedException e){
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}while(conn==null&&i<5);
return conn;
}
public static void closeConnection(Connection connection){
try {
connection.close();
poolx.shutdownNow();
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* 创建表
* @throws IOException
*/
public static void createTable(Connection connection,String tableName, String[] columns) throws IOException {
Admin admin =null;
try{
admin= connection.getAdmin();
TableName name=TableName.valueOf(tableName);
if (admin.tableExists(name)) {
admin.disableTable(name);
admin.deleteTable(name);
} else {
HTableDescriptor desc =new HTableDescriptor();
desc.setName(TableName.valueOf(tableName));
for (String column : columns) {
desc.addFamily(new HColumnDescriptor(column));
}
admin.createTable(desc);
}
}finally{
admin.close();
}
}
public static void createTable(Connection connection,Class entityClass){
String hbase_name=EntityUtils.getTargetTableName(entityClass);
Field[]fields=entityClass.getDeclaredFields();
Set<String> fimalySet=new HashSet<String>();
for(Field field:fields){
Column column=field.getAnnotation(Column.class);
fimalySet.add(column.family());
}
String [] familys=fimalySet.toArray(new String[]{});
try {
createTable(connection,hbase_name,familys);
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* 插入一行记录
* @param tablename
* 表名
* @param row
* 行名称
* @param columnFamily
* 列族名
* @param columns
* （列族名：column）组合成列名
* @param values
* 行与列确定的值
*/
public static void insertRecord(Connection connection,String tableName, String row, String columnFamily, String[] columns, String[] values) {
Table table=null;
try {
TableName name=TableName.valueOf(tableName);
table = connection.getTable(name);
Put put = new Put(Bytes.toBytes(row));
for (int i = 0; i < columns.length; i++) {
put.addColumn(Bytes.toBytes(columnFamily),
Bytes.toBytes(String.valueOf(columns[i])),
Bytes.toBytes(values[i]));
table.put(put);
}
} catch (IOException e) {
e.printStackTrace();
}finally {
try {
if(table!=null){
table.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
* 將对象转化为Put
* @param obj
* @return
* @throws IllegalAccessException
* @throws IllegalArgumentException
*/
private static Put convertEntityToPut(BaseEntity obj) throws IllegalArgumentException, IllegalAccessException{
Field[]fields=obj.getClass().getDeclaredFields();
Field rowKeyField=null;
for(Field field:fields){
if(field.isAnnotationPresent(RowKey.class)){
rowKeyField=field;
rowKeyField.setAccessible(true);
}
}
if(rowKeyField==null){
throw new RuntimeException("rowKey注解不存在");
}
Put p = new Put(Bytes.toBytes(String.valueOf(rowKeyField.get(obj))));
for(Field field:fields){
if(field!=rowKeyField){
Column column=field.getAnnotation(Column.class);
String family=column.family();
String col=column.target_column();
if(col.equals("")){
col=field.getName();
}
field.setAccessible(true);
Object value=field.get(obj);
String type=field.getType().toString();
type=type.substring(type.lastIndexOf(".")+1);
if(type.equals("Date")){
String rdbType=column.rdb_type();
Date tmpValue=(Date)value;
if(tmpValue==null){
p.addColumn(Bytes.toBytes(family), Bytes.toBytes(col), null);
continue;
}
String transValue=null;
if(rdbType.equals("date")){
SimpleDateFormat sdf=new SimpleDateFormat("yyyy-MM-dd");
transValue=sdf.format(tmpValue);
}else if(rdbType.equals("datetime")){
SimpleDateFormat sdf=new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
transValue=sdf.format(tmpValue);
}else if(rdbType.equals("time")){
SimpleDateFormat sdf=new SimpleDateFormat("HH:mm:ss");
transValue=sdf.format(tmpValue);
}else{
throw new RuntimeException("rdb_type not supported");
}
p.addColumn(Bytes.toBytes(family), Bytes.toBytes(col), Bytes.toBytes(transValue));
}else if(type.equals("class [B")){
p.addColumn(Bytes.toBytes(family), Bytes.toBytes(col), (byte[])value);
}else{
p.addColumn(Bytes.toBytes(family), Bytes.toBytes(col), EntityUtils.convertJavaTypeToByteArray(value));
}
}
}
return p;
}
public static List<BaseEntity> convertResultToEntity(List<Result> resultList,Class cls) throws Exception, IllegalAccessException{
List<BaseEntity> entityList=new ArrayList<BaseEntity>();
for(Result result : resultList){
BaseEntity entity=(BaseEntity)cls.newInstance();
EntityUtils.setRowKeyToEntity(result.getRow(),entity);
for (Cell cell : result.rawCells()) {
EntityUtils.setHBaseCellToEntity(cell,entity);
}
entityList.add(entity);
}
return entityList;
}
/**
* 批量插入
* @param tableName
* @param entityList
* @throws IOException
* @throws IllegalAccessException
* @throws IllegalArgumentException
*/
public static void batchInsertRecord(Connection connection,String tableName,List<BaseEntity> entityList) throws IOException, IllegalArgumentException, IllegalAccessException{
BufferedMutator table=null;
try{
table = connection.getBufferedMutator(TableName.valueOf(tableName));
List<Mutation> mutations = new ArrayList<Mutation>();
for(BaseEntity entity:entityList){
mutations.add(convertEntityToPut(entity));
}
table.mutate(mutations);
table.flush();
}finally {
try {
if(table!=null){
table.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
* 删除一行记录
* @param tablename
* 表名
* @param rowkey
* 行名
* @throws IOException
*/
public static void deleteRow(Connection connection,String tablename, String rowkey) throws IOException {
Table table=null;
try{
TableName name=TableName.valueOf(tablename);
table = connection.getTable(name);
List<Delete> list = new ArrayList<Delete>();
Delete d1 = new Delete(rowkey.getBytes());
list.add(d1);
table.delete(list);
}finally {
try {
if(table!=null){
table.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
* 查找一行记录
* @param tablename
* 表名
* @param rowkey
* 行名
*/
public static BaseEntity selectRow(Connection connection,String tablename, String rowKey,Class cls)
throws IOException {
TableName name=TableName.valueOf(tablename);
Table table = connection.getTable(name);
Get g = new Get(rowKey.getBytes());
Result rs = table.get(g);
BaseEntity entity=null;
try {
EntityUtils.setRowKeyToEntity(rs.getRow(),entity);
entity = (BaseEntity)cls.newInstance();
for (Cell cell : rs.rawCells()) {
EntityUtils.setHBaseCellToEntity(cell,entity);
}
return entity;
} catch (InstantiationException e) {
e.printStackTrace();
} catch (IllegalAccessException e) {
e.printStackTrace();
}finally {
try {
if(table!=null){
table.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
return entity;
}
/**
* 查询表中所有行
* @param tablename
*/
public static List<BaseEntity> scanAllRecord(Connection connection,String tablename,Class cls) {
Table table=null;
try {
List<BaseEntity> entityList=new ArrayList<BaseEntity>();
TableName name=TableName.valueOf(tablename);
table= connection.getTable(name);
Scan s = new Scan();
ResultScanner rs = table.getScanner(s);
for(Result result : rs){
BaseEntity entity=(BaseEntity)cls.newInstance();
EntityUtils.setRowKeyToEntity(result.getRow(),entity);
for (Cell cell : result.rawCells()) {
EntityUtils.setHBaseCellToEntity(cell,entity);
entityList.add(entity);
}
}
return entityList;
} catch (IOException e) {
e.printStackTrace();
} catch (InstantiationException e) {
e.printStackTrace();
} catch (IllegalAccessException e) {
e.printStackTrace();
}finally {
try {
if(table!=null){
table.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
return null;
}
/**
* 删除表操作
* @param tablename
* @throws IOException
*/
public static void deleteTable(Connection connection,Class cls) {
Admin admin=null;
try {
String tablename=EntityUtils.getTargetTableName(cls);
TableName name=TableName.valueOf(tablename);
admin= connection.getAdmin();
if(admin.tableExists(name)){
admin.disableTable(name);
admin.deleteTable(name);
}
} catch (MasterNotRunningException e) {
e.printStackTrace();
} catch (ZooKeeperConnectionException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}finally {
try {
if(admin!=null){
admin.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
* 分页检索表数据。<br>
* （如果在创建表时为此表指定了非默认的命名空间，则需拼写上命名空间名称，格式为【namespace:tablename】）。
* @param tableName 表名称(*)。
* @param startRowKey 起始行键(可以为空，如果为空，则从表中第一行开始检索)。
* @param endRowKey 结束行键(可以为空)。
* @param filterList 检索条件过滤器集合(不包含分页过滤器；可以为空)。
* @param maxVersions 指定最大版本数【如果为最大整数值，则检索所有版本；如果为最小整数值，则检索最新版本；否则只检索指定的版本数】。
* @param pageModel 分页模型(*)。
* @return 返回HBasePageModel分页对象。
*/
public static HBasePageModel scanResultByPageFilter(Connection connection,String tableName, byte[] startRowKey, byte[] endRowKey, FilterList filterList, int maxVersions, HBasePageModel pageModel) {
if(pageModel == null) {
pageModel = new HBasePageModel(10);
}
if(maxVersions <= 0 ) {
//默认只检索数据的最新版本
maxVersions = Integer.MIN_VALUE;
}
pageModel.initStartTime();
pageModel.initEndTime();
if(StringUtils.isBlank(tableName)) {
return pageModel;
}
Table table = null;
try {
//根据HBase表名称，得到HTable表对象，这里用到了笔者本人自己构建的一个表信息管理类。
table = connection.getTable(TableName.valueOf(tableName));
int tempPageSize = pageModel.getPageSize();
boolean isEmptyStartRowKey = false;
if(startRowKey == null) {
//则读取表的第一行记录，这里用到了笔者本人自己构建的一个表数据操作类。
Result firstResult =selectFirstResultRow(connection,tableName, filterList);
if(firstResult.isEmpty()) {
return pageModel;
}
startRowKey = firstResult.getRow();
}
if(pageModel.getPageStartRowKey() == null) {
isEmptyStartRowKey = true;
pageModel.setPageStartRowKey(startRowKey);
} else {
if(pageModel.getPageEndRowKey() != null) {
pageModel.setPageStartRowKey(pageModel.getPageEndRowKey());
}
//从第二页开始，每次都多取一条记录，因为第一条记录是要删除的。
tempPageSize += 1;
}
Scan scan = new Scan();
scan.setStartRow(pageModel.getPageStartRowKey());
if(endRowKey != null) {
scan.setStopRow(endRowKey);
}
PageFilter pageFilter = new PageFilter(pageModel.getPageSize() + 1);
if(filterList != null) {
filterList.addFilter(pageFilter);
scan.setFilter(filterList);
} else {
scan.setFilter(pageFilter);
}
if(maxVersions == Integer.MAX_VALUE) {
scan.setMaxVersions();
} else if(maxVersions == Integer.MIN_VALUE) {
} else {
scan.setMaxVersions(maxVersions);
}
ResultScanner scanner = table.getScanner(scan);
List<Result> resultList = new ArrayList<Result>();
int index = 0;
for(Result rs : scanner.next(tempPageSize)) {
if(isEmptyStartRowKey == false && index == 0) {
index += 1;
continue;
}
if(!rs.isEmpty()) {
resultList.add(rs);
}
index += 1;
}
scanner.close();
pageModel.setResultList(resultList);
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
if(table!=null){
table.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
int pageIndex = pageModel.getPageIndex() + 1;
pageModel.setPageIndex(pageIndex);
if(pageModel.getResultList().size() > 0) {
//获取本次分页数据首行和末行的行键信息
byte[] pageStartRowKey = pageModel.getResultList().get(0).getRow();
byte[] pageEndRowKey = pageModel.getResultList().get(pageModel.getResultList().size() - 1).getRow();
pageModel.setPageStartRowKey(pageStartRowKey);
pageModel.setPageEndRowKey(pageEndRowKey);
}
int queryTotalCount = pageModel.getQueryTotalCount() + pageModel.getResultList().size();
pageModel.setQueryTotalCount(queryTotalCount);
pageModel.initEndTime();
pageModel.printTimeInfo();
return pageModel;
}
/**
* 检索指定表的第一行记录。<br>
* （如果在创建表时为此表指定了非默认的命名空间，则需拼写上命名空间名称，格式为【namespace:tablename】）。
* @param tableName 表名称(*)。
* @param filterList 过滤器集合，可以为null。
* @return
*/
public static Result selectFirstResultRow(Connection connection,String tableName,FilterList filterList) {
if(StringUtils.isBlank(tableName)) return null;
Table table = null;
try {
table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
if(filterList != null) {
scan.setFilter(filterList);
}
ResultScanner scanner = table.getScanner(scan);
Iterator<Result> iterator = scanner.iterator();
int index = 0;
while(iterator.hasNext()) {
Result rs = iterator.next();
if(index == 0) {
scanner.close();
return rs;
}
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if(table!=null){
table.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
return null;
}
public static long rowCountByFilter(Connection connection,Class cls ) {
String tableName=EntityUtils.getSourceTableName(cls);
long rowCount = 0;
Table table = null;
try {
table= connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
scan.setFilter(new FirstKeyOnlyFilter());
ResultScanner resultScanner = table.getScanner(scan);
for (Result result : resultScanner) {
rowCount += result.size();
}
} catch (IOException e) {
e.printStackTrace();
}finally{
if(table!=null){
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return rowCount;
}
public static void addTableCoprocessor(Connection connection,String table, String coprocessorClassName) {
Admin admin=null;
try {
TableName tableName=TableName.valueOf(table);
admin= connection.getAdmin();
admin.disableTable(tableName);
HTableDescriptor htd = admin.getTableDescriptor(tableName);
htd.addCoprocessor(coprocessorClassName);
admin.modifyTable(tableName, htd);
admin.enableTable(tableName);
} catch (IOException e) {
e.printStackTrace();
}finally{
if(admin!=null){
try {
admin.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
public static long rowCountByAggregation(String tableName, String family) {
AggregationClient ac=null;
long rowCount = 0;
try {
Scan scan = new Scan();
scan.addFamily(Bytes.toBytes(family));
ac= new AggregationClient(HBaseConfiguration.create());
rowCount = ac.rowCount(TableName.valueOf(tableName), new LongColumnInterpreter(), scan);
} catch (Throwable e) {
e.printStackTrace();
}finally{
if(ac!=null){
try {
ac.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return rowCount;
}
public long testTableRowCount() {
HBaseConnectionEntity entity=null;
try{
entity=HBaseConnectionPool.getInstance().getConnection();
Connection conn=entity.getConnection();
String coprocessorClassName = "org.apache.hadoop.hbase.coprocessor.AggregateImplementation";
addTableCoprocessor(conn,"user", coprocessorClassName);
return rowCountByAggregation("user", "basic");
}finally{
HBaseConnectionPool.getInstance().releaseConnection(entity.getId());
}
}
}
扫一扫
2万+
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
