ResultSet is from UPDATE. No Data.

本文介绍了MySQL数据库通过读写分离来提升系统性能的方法。详细解释了读写分离的基本原理及其实现方式,并分享了一些实际应用的经验。

mysql 读写分离引起

转载于:https://www.cnblogs.com/cocoat/p/6018098.html

// BoxDBManager.ets import Box from '../bean/Box'; import relationalStore from '@ohos.data.relationalStore'; import common from '@ohos.app.ability.common'; import emitter from '@ohos.events.emitter'; import { Record } from '../bean/Record'; import { BusinessError } from '@ohos.base'; import { Logger } from './Logger'; import { uartUtils } from './UartUtils'; import { Member } from '../bean/Member'; // 定义锁信息接口 interface LockInfo { lockBoard: number; lockNumber: number; } interface SizeLockMap { size: string; start: number; } interface SizeLockMigrationMap { sizeChar: string; fullSize: string; start: number; } interface MemberInfo { id: number; name: string; phone: string; password: string; } const TAG = "UartUtils"; class BoxDBManager { private static instance: BoxDBManager; private rdbStore: relationalStore.RdbStore | null = null; private readonly DB_NAME = 'BoxDB.db'; private readonly TABLE_NAME = 'BOX'; public readonly DATA_CHANGED_EVENT_ID = 1001; private readonly DB_CONFIG: relationalStore.StoreConfig = { name: this.DB_NAME, securityLevel: relationalStore.SecurityLevel.S1 }; private readonly CREATE_TABLE_SQL = ` CREATE TABLE IF NOT EXISTS ${this.TABLE_NAME} ( ID INTEGER PRIMARY KEY AUTOINCREMENT, PHONE TEXT DEFAULT '', PASSWORD TEXT DEFAULT '', BOX_ID TEXT UNIQUE, SIZE TEXT, STATUS INTEGER DEFAULT 0, ENABLED INTEGER DEFAULT 1, LOCK_BOARD INTEGER DEFAULT 1, LOCK_NUMBER INTEGER DEFAULT 1 )`; private readonly CREATE_LOG_TABLE_SQL = ` CREATE TABLE IF NOT EXISTS BOX_OPERATION_LOG ( ID INTEGER PRIMARY KEY AUTOINCREMENT, BOX_ID INTEGER DEFAULT 0, OPERATION_TIME TEXT NOT NULL, OPERATOR TEXT NOT NULL, ACTION_TYPE TEXT NOT NULL, ACCESS_METHOD TEXT NOT NULL )`; private readonly CREATE_MEMBER_TABLE_SQL = ` CREATE TABLE IF NOT EXISTS MEMBER ( ID INTEGER PRIMARY KEY AUTOINCREMENT, MEMBER_ID TEXT NOT NULL, NAME TEXT NOT NULL, PASSWORD TEXT NOT NULL )`; private constructor() {} public static getInstance(): BoxDBManager { if (!BoxDBManager.instance) { BoxDBManager.instance = new BoxDBManager(); } return BoxDBManager.instance; } // 初始化数据库(使用Promise) async initDatabase(context: common.Context): Promise<boolean> { try { this.rdbStore = await relationalStore.getRdbStore(context, this.DB_CONFIG); await this.rdbStore.executeSql(this.CREATE_TABLE_SQL); await this.rdbStore.executeSql(this.CREATE_LOG_TABLE_SQL); await this.rdbStore.executeSql(this.CREATE_MEMBER_TABLE_SQL); await this.initDefaultBoxes(); console.info('Database initialized successfully'); return true; } catch (err) { console.error(`Database init failed: ${JSON.stringify(err)}`); return false; } } // 新增:添加会员(使用 Member 类) async addMember(member: Member): Promise<boolean> { if (!this.rdbStore){return false} try { const valueBucket: relationalStore.ValuesBucket = { 'MEMBER_ID': member.id, // 会员编号(字符串) 'NAME': member.name, // 姓名 'PASSWORD': member.password // 密码(已拼接好的完整密码) }; // 插入会员表 await this.rdbStore.insert('MEMBER', valueBucket); console.info(`会员 ${member.name} 添加成功`); return true; } catch (err) { console.error(`添加会员失败: ${JSON.stringify(err)}`); return false; } } // 验证会员(通过密码) async verifyMember(password: string): Promise<boolean> { if (!this.rdbStore) return false; try { // 使用RegExp构造函数替代正则表达式字面量 const passwordRegex = new RegExp('^\\d{8}$'); if (!passwordRegex.test(password)) { console.error("密码格式错误,必须是8位数字"); return false; } const predicates = new relationalStore.RdbPredicates('MEMBER'); predicates.equalTo('PASSWORD', password); const columns = ['ID']; const resultSet = await this.rdbStore.query(predicates, columns); const isMember = resultSet.rowCount > 0; resultSet.close(); return isMember; } catch (err) { console.error(`验证会员失败: ${JSON.stringify(err)}`); return false; } } // 删除会员 async deleteMember(memberId: string): Promise<boolean> { if (!this.rdbStore) return false; try { const predicates = new relationalStore.RdbPredicates('MEMBER'); predicates.equalTo('MEMBER_ID', memberId); const rowsDeleted = await this.rdbStore.delete(predicates); console.info(`删除会员成功,ID: ${memberId},删除行数: ${rowsDeleted}`); this.notifyDataChanged(); // 通知数据变化 return rowsDeleted > 0; } catch (err) { console.error(`删除会员失败: ${JSON.stringify(err)}`); return false; } } // 获取所有会员(用于后台管理页面展示) async getAllMembers(): Promise<Member[]> { if (!this.rdbStore) return []; try { const predicates = new relationalStore.RdbPredicates('MEMBER'); const columns = ['MEMBER_ID', 'NAME', 'PASSWORD']; const resultSet = await this.rdbStore.query(predicates, columns); const members: Member[] = []; while (resultSet.goToNextRow()) { members.push(new Member( resultSet.getLong(resultSet.getColumnIndex('MEMBER_ID')).toString(), resultSet.getString(resultSet.getColumnIndex('NAME')), resultSet.getString(resultSet.getColumnIndex('PASSWORD')) // 返回存储的密码 )); } resultSet.close(); return members; } catch (err) { console.error(`查询会员列表失败: ${JSON.stringify(err)}`); return []; } } // 在BoxDBManager类中添加以下方法 // 更新会员信息(支持ID修改) async updateMember(oldId: string, newMember: Member): Promise<boolean> { if (!this.rdbStore) return false; try { // 检查新ID是否已存在(如果修改了ID) if (oldId !== newMember.id) { // 直接查询新ID是否已存在 const checkPredicates = new relationalStore.RdbPredicates('MEMBER'); checkPredicates.equalTo('MEMBER_ID', newMember.id); const resultSet = await this.rdbStore.query(checkPredicates, ['MEMBER_ID']); const exists = resultSet.rowCount > 0; resultSet.close(); if (exists) { console.error(`更新失败:会员编号 ${newMember.id} 已存在`); return false; } } // 构建更新条件 const predicates = new relationalStore.RdbPredicates('MEMBER'); predicates.equalTo('MEMBER_ID', oldId); // 构建更新内容 const valueBucket: relationalStore.ValuesBucket = { 'MEMBER_ID': newMember.id, 'NAME': newMember.name, 'PASSWORD': newMember.password }; // 执行更新 const rowsUpdated = await this.rdbStore.update(valueBucket, predicates); if (rowsUpdated === 0) { console.error(`未找到会员编号 ${oldId} 的记录`); return false; } console.info(`更新会员成功,从ID: ${oldId} 更新为 ${newMember.id}`); this.notifyDataChanged(); // 通知数据变化 return true; } catch (err) { console.error(`更新会员失败: ${JSON.stringify(err)}`); return false; } } // 初始化默认箱子 private async initDefaultBoxes(): Promise<void> { const count = await this.getBoxCount(); if (count > 0) return; const sizeLockMapping:SizeLockMap[] = [ { size: "特大", start: 1 }, { size: "大", start: 4 }, { size: "中", start: 7 }, { size: "小", start: 10 } ]; const boxes: relationalStore.ValuesBucket[] = []; for (const mapping of sizeLockMapping) { for (let i = 1; i <= 3; i++) { boxes.push({ 'BOX_ID': `${mapping.size.charAt(0)}${i}`, 'SIZE': mapping.size, 'STATUS': 0, 'ENABLED': 1, 'LOCK_BOARD': 1, 'LOCK_NUMBER': mapping.start + (i - 1) // 计算正确的锁号 }); } } await this.rdbStore?.batchInsert(this.TABLE_NAME, boxes); console.info(`${boxes.length} default boxes initialized`); } async migrateLockNumbers(): Promise<void> { if (!this.rdbStore) return; try { // 获取所有箱子 const allBoxes = await this.getAllBoxes(); // 使用类型安全的映射定义 const sizeLockMapping:SizeLockMigrationMap[] = [ { sizeChar: "特", fullSize: "特大", start: 1 }, { sizeChar: "大", fullSize: "大", start: 4 }, { sizeChar: "中", fullSize: "中", start: 7 }, { sizeChar: "小", fullSize: "小", start: 10 } ]; for (const box of allBoxes) { // 提取尺寸和序号 const regex = new RegExp("^(\\D+)(\\d+)$"); const match = regex.exec(box.boxId); if (!match) continue; const sizeChar = match[1]; const index = parseInt(match[2]); // 查找匹配的尺寸映射 const mapping = sizeLockMapping.find(m => m.sizeChar === sizeChar); if (!mapping) continue; // 计算新锁号 const newLockNumber = mapping.start + (index - 1); // 更新数据库 const predicates = new relationalStore.RdbPredicates(this.TABLE_NAME); predicates.equalTo('ID', box.id); const valueBucket: relationalStore.ValuesBucket = { 'LOCK_NUMBER': newLockNumber }; await this.rdbStore.update(valueBucket, predicates); } console.info("Lock numbers migrated successfully"); } catch (err) { console.error("Lock number migration failed:", err); } } // 获取箱子数量 private async getBoxCount(): Promise<number> { if (!this.rdbStore) return 0; const predicates = new relationalStore.RdbPredicates(this.TABLE_NAME); const result = await this.rdbStore.query(predicates, ['count(*) as count']); if (result.rowCount > 0 && result.goToFirstRow()) { return result.getLong(result.getColumnIndex('count')); } return 0; } // 获取所有箱子 async getAllBoxes(): Promise<Box[]> { if (!this.rdbStore) return []; try { const predicates = new relationalStore.RdbPredicates(this.TABLE_NAME); const columns = ['ID', 'PHONE', 'PASSWORD', 'BOX_ID', 'SIZE', 'STATUS', 'ENABLED', 'LOCK_BOARD', 'LOCK_NUMBER']; // 直接使用await获取结果集 const resultSet = await this.rdbStore.query(predicates, columns); const boxes: Box[] = []; if (resultSet.rowCount > 0) { while (resultSet.goToNextRow()) { const box = new Box( resultSet.getString(resultSet.getColumnIndex('BOX_ID')), resultSet.getString(resultSet.getColumnIndex('SIZE')), resultSet.getLong(resultSet.getColumnIndex('STATUS')), resultSet.getLong(resultSet.getColumnIndex('ENABLED')) === 1, resultSet.getLong(resultSet.getColumnIndex('LOCK_BOARD')), resultSet.getLong(resultSet.getColumnIndex('LOCK_NUMBER')) ); box.id = resultSet.getLong(resultSet.getColumnIndex('ID')); box.phone = resultSet.getString(resultSet.getColumnIndex('PHONE')); box.password = resultSet.getString(resultSet.getColumnIndex('PASSWORD')); boxes.push(box); } } resultSet.close(); return boxes; } catch (err) { console.error(`Query failed: ${JSON.stringify(err)}`); return []; } } // 更新箱子状态 async updateBoxStatus(id: number, status: number): Promise<boolean> { if (!this.rdbStore) return false; const predicates = new relationalStore.RdbPredicates(this.TABLE_NAME); predicates.equalTo('ID', id); const valueBucket: relationalStore.ValuesBucket = { 'STATUS': status }; await this.rdbStore.update(valueBucket, predicates); this.notifyDataChanged(); return true; } // 占用箱子 async occupyBox(boxId: string, phone: string, password: string): Promise<boolean> { if (!this.rdbStore) return false; const predicates = new relationalStore.RdbPredicates(this.TABLE_NAME); predicates.equalTo('BOX_ID', boxId); predicates.equalTo('STATUS', 0); const valueBucket: relationalStore.ValuesBucket = { 'PHONE': phone, 'PASSWORD': password, 'STATUS': 1 }; await this.rdbStore.update(valueBucket, predicates); this.notifyDataChanged(); return true; } // 释放箱子 async releaseBox(phoneSuffix: string, password: string): Promise<string[]> { if (!this.rdbStore) return []; // 查询匹配的箱子 const predicates = new relationalStore.RdbPredicates(this.TABLE_NAME); predicates.endsWith('PHONE', phoneSuffix); predicates.equalTo('PASSWORD', password); predicates.equalTo('STATUS', 1); predicates.equalTo('ENABLED', 1); const result = await this.rdbStore.query(predicates, ['ID', 'BOX_ID']); const boxIds: string[] = []; while (result.goToNextRow()) { boxIds.push(result.getString(result.getColumnIndex('BOX_ID'))); } result.close(); if (boxIds.length === 0) return []; // 更新匹配的箱子 const updatePredicates = new relationalStore.RdbPredicates(this.TABLE_NAME); updatePredicates.endsWith('PHONE', phoneSuffix); updatePredicates.equalTo('PASSWORD', password); updatePredicates.equalTo('STATUS', 1); updatePredicates.equalTo('ENABLED', 1); const valueBucket: relationalStore.ValuesBucket = { 'PHONE': '', 'PASSWORD': '', 'STATUS': 0 }; await this.rdbStore.update(valueBucket, updatePredicates); this.notifyDataChanged(); return boxIds; } // 更新箱子信息 public async updateBoxInfo( id: number, boxId: string, size: string, status: number, // 新增状态参数 enabled: boolean, lockBoard: number, lockNumber: number ): Promise<boolean> { if (!this.rdbStore) return false; try { const predicates = new relationalStore.RdbPredicates(this.TABLE_NAME); predicates.equalTo('ID', id); const valueBucket: relationalStore.ValuesBucket = { 'BOX_ID': boxId, 'SIZE': size, 'STATUS': status, // 保存状态值 'ENABLED': enabled ? 1 : 0, 'LOCK_BOARD': lockBoard, 'LOCK_NUMBER': lockNumber }; const changedRows = await this.rdbStore.update(valueBucket, predicates); console.info(`Updated ${changedRows} rows for box ID: ${id}, new status: ${status}`); this.notifyDataChanged(); return changedRows > 0; } catch (err) { console.error(`Update box info failed: ${JSON.stringify(err)}`); return false; } } // 删除箱子 public async deleteBox(id: number): Promise<number> { if (!this.rdbStore) return 0; try { const predicates = new relationalStore.RdbPredicates(this.TABLE_NAME); predicates.equalTo('ID', id); const rows = await this.rdbStore.delete(predicates); this.notifyDataChanged(); return rows; } catch (err) { console.error(`Delete box failed: ${JSON.stringify(err)}`); return 0; } } // 添加新箱子 public async addBox(box: Box): Promise<number> { if (!this.rdbStore) return -1; try { const valueBucket: relationalStore.ValuesBucket = { 'BOX_ID': box.boxId, 'SIZE': box.size, 'ENABLED': box.enabled ? 1 : 0, 'LOCK_BOARD': box.lockBoard, 'LOCK_NUMBER': box.lockNumber }; const id = await this.rdbStore.insert(this.TABLE_NAME, valueBucket); this.notifyDataChanged(); return id; } catch (err) { console.error(`Add box failed: ${JSON.stringify(err)}`); return -1; } } // 清空箱子的手机号和密码 public async clearBoxCredentials(id: number): Promise<boolean> { if (!this.rdbStore) return false; try { const predicates = new relationalStore.RdbPredicates(this.TABLE_NAME); predicates.equalTo('ID', id); const valueBucket: relationalStore.ValuesBucket = { 'PHONE': '', 'PASSWORD': '' }; await this.rdbStore.update(valueBucket, predicates); return true; } catch (err) { console.error(`Clear credentials failed: ${JSON.stringify(err)}`); return false; } } // 通知数据变化 private notifyDataChanged() { try { console.info("Emitting DATA_CHANGED event"); emitter.emit({ eventId: this.DATA_CHANGED_EVENT_ID }, { data: { action: "DATA_CHANGED" } }); } catch (err) { console.error(`Emit event failed: ${err.message}`); } } addOperationLog(record:Record, callback: (success: boolean) => void) { const rdbStore = this.rdbStore; if (!rdbStore) { console.error('RdbStore is not initialized'); callback(false); return; } const valueBucket: relationalStore.ValuesBucket = { 'BOX_ID': record.Box_ID, 'OPERATION_TIME': record.Operation_Time, // 当前时间戳 'OPERATOR': record.Operator, 'ACTION_TYPE': record.Action_Type, 'ACCESS_METHOD': record.Access_Method }; rdbStore.insert('BOX_OPERATION_LOG', valueBucket, (err: BusinessError) => { if (err) { console.error(`Insert log failed. Code:${err.code}, message:${err.message}`); callback(false); return; } callback(true); }); } getLogsByTime(time: string, callback: (logs: Record[]) => void) { const rdbStore = this.rdbStore; if (!rdbStore) { console.error('RdbStore is not initialized'); callback([]); return; } const predicates = new relationalStore.RdbPredicates('BOX_OPERATION_LOG'); // 只查询Record类需要的字段 const columns = ['BOX_ID', 'OPERATION_TIME', 'OPERATOR', 'ACTION_TYPE', 'ACCESS_METHOD']; // 构建时间查询条件 if (time.length === 10) { // YYYY-MM-DD 格式 predicates.greaterThanOrEqualTo('OPERATION_TIME', `${time}T00:00:00Z`); predicates.lessThan('OPERATION_TIME', `${time}T23:59:59.999Z`); } else if (time.length === 7) { // YYYY-MM 格式 predicates.like('OPERATION_TIME', `${time}-%`); } else { console.error('Invalid time format. Use YYYY-MM or YYYY-MM-DD'); callback([]); return; } rdbStore.query(predicates, columns, (err: BusinessError, resultSet: relationalStore.ResultSet) => { if (err) { console.error(`Query failed. Code:${err.code}, message:${err.message}`); callback([]); return; } const logs: Record[] = []; if (resultSet.rowCount > 0) { while (resultSet.goToNextRow()) { logs.push(new Record( resultSet.getLong(resultSet.getColumnIndex('BOX_ID')), resultSet.getString(resultSet.getColumnIndex('OPERATION_TIME')), resultSet.getString(resultSet.getColumnIndex('OPERATOR')), resultSet.getString(resultSet.getColumnIndex('ACTION_TYPE')), resultSet.getString(resultSet.getColumnIndex('ACCESS_METHOD')) )); } } resultSet.close(); callback(logs); }); } // 新增方法:根据箱子ID获取锁板编号和锁编号 public async getLockInfoById(id: number): Promise<LockInfo | null> { if (!this.rdbStore) return null; try { const predicates = new relationalStore.RdbPredicates(this.TABLE_NAME); predicates.equalTo('ID', id); const columns = ['LOCK_BOARD', 'LOCK_NUMBER']; const resultSet = await this.rdbStore.query(predicates, columns); if (resultSet.rowCount > 0 && resultSet.goToFirstRow()) { return { lockBoard: resultSet.getLong(resultSet.getColumnIndex('LOCK_BOARD')), lockNumber: resultSet.getLong(resultSet.getColumnIndex('LOCK_NUMBER')) }; } return null; } catch (err) { console.error(`Get lock info failed: ${JSON.stringify(err)}`); return null; } } // 新增方法:根据箱子ID直接开锁 public async openLockById(id: number): Promise<boolean> { try { // 获取锁信息 const lockInfo = await this.getLockInfoById(id); if (!lockInfo) { Logger.e(TAG, `No lock info found for box ID: ${id}`); return false; } // 检查锁编号是否在有效范围内 (1-12) if (lockInfo.lockNumber < 1 || lockInfo.lockNumber > 12) { Logger.e(TAG, `Invalid lock number ${lockInfo.lockNumber} for box ID: ${id}`); return false; } // 开锁 uartUtils.openSpecificLock(lockInfo.lockNumber); return true; } catch (error) { Logger.e(TAG, `Failed to open lock for box ID ${id}: ${error.message}`); return false; } } } export default BoxDBManager.getInstance(); 根据box类将上面的代码修改一下 deveco studio4.1版本API11// Box.ets export default class Box { id: number = 0; phone: string = ''; password: string = ''; boxId: string = ''; size: string = ''; status: number = 0; // 0: 空闲, 1: 占用, 2: 打开 enabled: boolean = true; // 新增:能用状态 lockBoard: number = 1; // 新增:锁板 lockNumber: number = 1; // 新增:锁号 constructor( boxId: string, size: string, status: number, enabled: boolean = true, lockBoard: number = 1, lockNumber: number = 1 ) { this.boxId = boxId; this.size = size; this.status = status; this.enabled = enabled; this.lockBoard = lockBoard; this.lockNumber = lockNumber; } }
最新发布
07-22
package com.alibaba.otter.canal.server.embedded.handle.oracle; import com.alibaba.otter.canal.instance.core.CanalInstance; import com.alibaba.otter.canal.meta.FileMixedMetaManager; import com.alibaba.otter.canal.parse.CanalEventParser; import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser; import com.alibaba.otter.canal.parse.support.AuthenticationInfo; import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.server.embedded.handle.Constant; import com.alibaba.otter.canal.server.embedded.handle.NotMysqlHandler; import com.alibaba.otter.canal.server.exception.CanalServerException; import com.alibaba.otter.canal.utils.OracleUtil; import com.google.common.collect.Lists; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.CollectionUtils; import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; import java.sql.*; import java.text.SimpleDateFormat; import java.util.Date; import java.util.*; import java.util.concurrent.atomic.AtomicLong; /** * @author : yangpeng * create at: 2022/5/21 13:09 * @description: oracle连接实现 */ public class OracleHandler implements NotMysqlHandler { private static final Logger logger = LoggerFactory.getLogger(OracleHandler.class); private final static String ORACLE_DRIVER_URL = "jdbc:oracle:thin:@"; private final static String DATA_DICTIONARY_PATH = "/oracle/oradata/orcl/logmnr"; private Connection connection; private AuthenticationInfo authInfo; private AtomicLong currentScn = new AtomicLong(0); private Map<String, List<String>> pkMaps; private List<String> whiteList; //归档日志本文件最后一个点位-1, 下一个文件的开始点位 private Long nextChange = null; private Long firstChange = null; private String currentModel; private String currentRedoGroups; private String lastRedoGroups; private String opt = ">="; private List<String> action = new ArrayList<>(); static { //加载驱动 try { Class.forName("oracle.jdbc.OracleDriver"); } catch (ClassNotFoundException e) { e.printStackTrace(); } } @Override public void subscribe(CanalInstance canalInstance) { //数据库基本信息 CanalEventParser eventParser = canalInstance.getEventParser(); MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser; this.authInfo = mysqlEventParser.getMasterInfo(); connect(); String filterRegex = authInfo.getFilterRegex(); if(!org.springframework.util.StringUtils.isEmpty(filterRegex) && !filterRegex.endsWith(".*")){ whiteList = Arrays.asList(filterRegex.split(",")); } String index = FileMixedMetaManager.getScnLocal(canalInstance.getDestination()); //首次启动, 文件中的index是0, 同步类型: 全量+增量时使用衔接点位作为开始点位, //增量时衔接点位可能为空, 为空时获取系统最新点位作为开始点位 if("0".equals(index)){ String scn = authInfo.getTimestamp(); if(StringUtils.isEmpty(scn)) scn = getStartPosition(); try { this.currentScn.set(Long.valueOf(scn)); FileMixedMetaManager.saveScnLocal(canalInstance.getDestination(), Long.valueOf(scn)); } catch (IOException e) { e.printStackTrace(); } }else{ //index.scn文件中是非0点位, 说明当前实例运行被停止了或者被删除后实例重建了, 再次启动时比较文件点位和衔接点位 //衔接点位大于文件点位说明是删除后实例重建了, 删除后重建实例应使用最新的衔接点位开始增量同步 //衔接点位小于等于文件点位说明是暂停后重启了, 暂停后重启应使用文件中点位继续增量同步 String scn = authInfo.getTimestamp(); if(!StringUtils.isEmpty(scn) && Long.valueOf(scn) > Long.valueOf(index)){ try { this.currentScn.set(Long.valueOf(scn)); FileMixedMetaManager.saveScnLocal(canalInstance.getDestination(), Long.valueOf(scn)); } catch (IOException e) { e.printStackTrace(); } }else{ opt = ">"; this.currentScn.set(Long.valueOf(index)); } } if(StringUtils.isEmpty(authInfo.getAction())){ action.add("INSERT"); action.add("UPDATE"); action.add("DELETE"); }else{ action = Arrays.asList(authInfo.getAction().split(",")); } logger.warn("开始分析的点位:"+this.currentScn.get()); currentRedoGroups = queryCurrentRedoGroups(); lastRedoGroups = currentRedoGroups; queryLastArchivedLogScn(); authInfo.setArchived(true); updateSettings(); //获取主键 pkMaps = new HashMap<>(); for (String table : whiteList) { List<String> pkNames = getPkNames(authInfo.getDefaultDatabaseName(), table); pkMaps.put(authInfo.getDefaultDatabaseName()+"."+table, pkNames); } } private String queryCurrentRedoGroups(){ //获取当前正在使用的redo日志文件的相关组信息 Statement statement = null; ResultSet resultSet = null; try { statement = connection.createStatement(); resultSet = statement.executeQuery("SELECT GROUP# FROM V$LOG WHERE STATUS = 'CURRENT'"); resultSet.next(); return resultSet.getString(1); } catch (SQLException e) { logger.warn("Query Current Redo Group failed!"); } finally { close(statement, resultSet); } return ""; } private void queryLastArchivedLogScn(){ Statement statement = null; ResultSet resultSet = null; try { logger.warn("获取最后一个归档日志的开始点位和结束点位..."); statement = connection.createStatement(); String sql = "SELECT FIRST_CHANGE#, NEXT_CHANGE# FROM v$archived_log WHERE DEST_ID = 1 ORDER BY RECID DESC"; logger.warn(sql); resultSet = statement.executeQuery(sql); if(resultSet.next()){ firstChange = resultSet.getLong(1); nextChange = resultSet.getLong(2); } } catch (Exception e) { logger.warn("query last one Archive log startScn and endScn failed!"); } finally { close(statement, resultSet); } } private void isArchived(ArrayList<String> logFiles){ Statement statement = null; ResultSet resultSet = null; try { logger.warn("分析归档日志..."); currentModel = "archive"; statement = connection.createStatement(); String sql = "SELECT NAME, FIRST_CHANGE#, NEXT_CHANGE# FROM v$archived_log WHERE DEST_ID = 1 AND " + "FIRST_CHANGE# <= "+currentScn.get()+" AND NEXT_CHANGE# > "+currentScn.get(); logger.warn(sql); resultSet = statement.executeQuery(sql); if(resultSet.next()){ String NAME = resultSet.getString(1); firstChange = resultSet.getLong(2); nextChange = resultSet.getLong(3); //添加归档日志 logFiles.add(NAME); if(!Objects.equals(currentRedoGroups, lastRedoGroups)){ lastRedoGroups = currentRedoGroups; } } if(CollectionUtils.isEmpty(logFiles) && !Objects.equals(currentRedoGroups, lastRedoGroups)){ int num = 0; while (true){ resultSet = statement.executeQuery(sql); if(resultSet.next()){ String NAME = resultSet.getString(1); firstChange = resultSet.getLong(2); nextChange = resultSet.getLong(3); //添加归档日志 logFiles.add(NAME); } //在线日志写入到归档异步进行, 可能刚切换归档日志还没有落地, 若没获取到日志文件就阻塞, 最大60秒 if(CollectionUtils.isEmpty(logFiles)){ Thread.sleep(1000L); num++; logger.warn("在线日志写入到归档未完成, 线程已等待: "+num+"秒"); if(num >= 60) { logger.warn("在线日志写入到归档未完成等待60秒未完成, 默认放弃分析本次归档日志, 可能造成数据丢失, 丢失可能发生的点位:"+currentScn.get()); break; } } else{ lastRedoGroups = currentRedoGroups; break; } } } } catch (Exception e) { logger.warn("Archive log parsing failed!"); } finally { close(statement, resultSet); } } private void addLogfile(ArrayList<String> logFiles){ if(CollectionUtils.isEmpty(logFiles)){ Statement statement = null; ResultSet resultSet = null; try { logger.warn("分析在线日志..."); currentModel = "online"; statement = connection.createStatement(); //分析前校验日志组是否发生了变更, 发生变更的话从分析归档开始 String group = queryCurrentRedoGroups(); if(!currentRedoGroups.equals(group)){ currentRedoGroups = group; isArchived(logFiles); }else{ String sql = "SELECT f.member,g.group#,g.FIRST_CHANGE# FROM v$log g left join v$logfile f on g.group# = f.group# " + "where g.FIRST_CHANGE# <= " + currentScn.get() + " AND g.NEXT_CHANGE# > " + currentScn.get(); resultSet = statement.executeQuery(sql); logger.warn(sql); while (resultSet.next()) { String fileName = resultSet.getString(1); if(!fileName.contains("sredo")){ logFiles.add(fileName); firstChange = resultSet.getLong(3); } } } } catch (SQLException e) { logger.warn("Description Failed to query online logs!"); } finally { close(statement, resultSet); } } } private void offsetLogfile(ArrayList<String> logFiles){ if(CollectionUtils.isEmpty(logFiles)){ Statement statement = null; ResultSet resultSet = null; try { logger.warn("补偿措施, 分析最新的在线日志..."); currentModel = "online"; statement = connection.createStatement(); String sql = "SELECT f.member,g.group#,g.FIRST_CHANGE# FROM v$log g left join v$logfile f on g.group# = f.group# where g.status = 'CURRENT'"; resultSet = statement.executeQuery(sql); logger.warn(sql); while (resultSet.next()) { String fileName = resultSet.getString(1); if(!fileName.contains("sredo")){ logFiles.add(fileName); firstChange = resultSet.getLong(3); } } if(!CollectionUtils.isEmpty(logFiles)){ currentScn.set(firstChange); opt = ">="; } } catch (SQLException e) { logger.warn("Description Failed to query online logs!"); } finally { close(statement, resultSet); } } } private void setSqlTrace() { String setSqlTraceSql = "BEGIN\n" + " DBMS_SESSION.SET_SQL_TRACE(FALSE);\n" + " END;"; logger.warn(setSqlTraceSql); getCallableStatement(setSqlTraceSql); } private void setLogAnalyseDir(){ String createDictSql = "BEGIN dbms_logmnr_d.build(dictionary_filename =>'" + authInfo.getOraName() + "', dictionary_location =>'" + DATA_DICTIONARY_PATH + "'); END;"; logger.warn(createDictSql); getCallableStatement(createDictSql); } private void addAllfile(ArrayList<String> logFiles){ StringBuilder sbSQL = new StringBuilder(); //logmnr最大分析文件个数3 for (int i = 0; i < logFiles.size(); i++) { if (i == 0) { sbSQL.append("BEGIN\n"); } sbSQL.append("dbms_logmnr.add_logfile(logfilename=>'").append(logFiles.get(i)).append("',options=>dbms_logmnr.") .append(i == 0 ? "new" : "addfile").append(");\n"); } logFiles.clear(); sbSQL.append("END;\n"); logger.warn(sbSQL.toString()); getCallableStatement(sbSQL.toString()); } public void updateSettings() { ArrayList<String> logFiles = new ArrayList<>(); // 禁用当前会话的SQL跟踪功能 setSqlTrace(); // 设置日志分析目录 // setLogAnalyseDir(); // 是否分析归档日志 isArchived(logFiles); // 分析在线日志 addLogfile(logFiles); // 归档和在线都没有符合点位的文件, 可能是停机久了再次启动时归档文件被迁移或删除了, 从最新的在线日志文件开始点位开始分析 if (logFiles.isEmpty()) offsetLogfile(logFiles); // 归档丢失, 在线不存在, 数据库可能出现故障 if (logFiles.isEmpty()) throw new RuntimeException("The Oracle log file was not read"); // 添加所有日志文件 addAllfile(logFiles); try { // 开启logmnr视图 String startsql = "BEGIN\n" + "dbms_logmnr.start_logmnr(startScn=>'" + currentScn.get() + "'" //开始的scn号 // + ",dictfilename=>'" + DATA_DICTIONARY_PATH + "/" + authInfo.getOraName() + "'" //字典路径 + ",options=>" + " dbms_logmnr.dict_from_online_catalog" + //使用在线数据字典来查看相关的元数据信息 " + dbms_logmnr.skip_corruption" + //跳过损坏的日志快 " + dbms_logmnr.no_sql_delimiter" + //去掉SQL分隔符 " + dbms_logmnr.no_rowid_in_stmt" + //去掉SQL中的ROWID // " + dbms_logmnr.committed_data_only" + //去掉未提交事务的数据 " + dbms_logmnr.string_literals_in_stmt) ;\n" //输出SQL语句中的字符串常量 + "END;"; logger.warn(startsql); getCallableStatement(startsql); logger.warn("dbms_logmnr successfully! 当前分析模式: " + currentModel); } catch (Exception e) { logger.error(e.getMessage(), e); } } private void logSwitchVerify(){ String crgs = queryCurrentRedoGroups(); if(StringUtils.isNotEmpty(crgs) && !crgs.equals(currentRedoGroups)){ currentRedoGroups = crgs; unsubscribe(); updateSettings(); } } @Override public synchronized Message getWithoutAck(CanalInstance canalInstance) { try { if (connection.isClosed()) { reconnect(); } } catch (Exception e) { logger.error(e.getMessage(), e); } logSwitchVerify(); List<FlatMessage> data = new ArrayList<>(); Statement statement = null; ResultSet resultSet = null; String sql =""; try { statement = connection.createStatement(); long endScn = currentScn.get(); String queryLogSql = "SELECT scn,operation,sql_redo,table_space,seg_owner,table_name,timestamp,csf FROM v$logmnr_contents " + "where scn %s %d and table_name = '%s' and operation in('INSERT','DELETE','UPDATE') order by scn asc"; //logger.warn(String.format(queryLogSql, opt, currentScn.get(), whiteList.get(0))); resultSet = statement.executeQuery(String.format(queryLogSql, opt, currentScn.get(), whiteList.get(0))); opt = ">"; long lastScn = currentScn.get(); while (resultSet.next()) { endScn = resultSet.getLong("scn"); String operation = resultSet.getString(Constant.operation); if(!action.contains(operation.toUpperCase())) continue; // String segOwner = resultSet.getString(Constant.seg_owner); // String tableSpace = resultSet.getString(Constant.table_space); String tableName = resultSet.getString(Constant.table_name); sql = resultSet.getString(Constant.sql_redo); logger.info("解析到 Oracle 日志中的 SQL,操作类型:{},SQL 内容:{}", operation, sql); if(StringUtils.isEmpty(sql)){ logger.error(operation + "->" + tableName+", sql为空默认跳过"); continue; } FlatMessage flatMessage = new FlatMessage(); String timestamp = resultSet.getString(Constant.timestamp); if(!StringUtils.isEmpty(timestamp)){ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); long es = sdf.parse(timestamp).getTime() / 1000; flatMessage.setEs(es); }else{ long es = new Date().getTime() / 1000; flatMessage.setEs(es); } flatMessage.setDatabase(authInfo.getDefaultDatabaseName()); flatMessage.setTable(tableName); flatMessage.setIsDdl(false); flatMessage.setType(operation); flatMessage.setTs(System.currentTimeMillis()); flatMessage.setSql(sql); OracleUtil.toMqformat(flatMessage);//转换sql为mq所需json格式 if(null == flatMessage.getData()) { logger.warn("异常SQL: " + sql); continue; } //主键设置 List<String> list = pkMaps.get(authInfo.getDefaultDatabaseName() + "." + tableName); if(!CollectionUtils.isEmpty(list)){ flatMessage.setPkNames(list); }else{ flatMessage.setPkNames(new ArrayList<>()); } data.add(flatMessage); } if(endScn != currentScn.get()){ currentScn.set(endScn); FileMixedMetaManager.saveScnLocal(canalInstance.getDestination(), currentScn.get()); } //本次的归档日志已分析结束, 结束本次会话, 开始下次分析 if("archive".equals(currentModel) && CollectionUtils.isEmpty(data)){ currentScn.set(nextChange); opt = ">="; FileMixedMetaManager.saveScnLocal(canalInstance.getDestination(), currentScn.get()); logger.warn("本轮归档日志已分析结束, 结束本次会话, 开始下次分析, 开始点位: "+nextChange); unsubscribe(); updateSettings(); } if (!CollectionUtils.isEmpty(data)) { return new Message(lastScn, data, Constant.NOT_MYSQL); } } catch (Exception e) { logger.error("oracle logminer select v$logmnr_contents failed"); logger.warn(sql); StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); // 将出错的栈信息输出到printWriter中 e.printStackTrace(pw); pw.flush(); sw.flush(); try { sw.close(); } catch (IOException ex) { throw new RuntimeException(ex); } pw.close(); unsubscribe(); updateSettings(); } finally { close(statement, resultSet); } return new Message(-1, true, null); } //TRIGGER private String getStartPosition() { String sql = "SELECT MAX(CURRENT_SCN) CURRENT_SCN FROM GV$DATABASE"; Statement statement = null; ResultSet resultSet = null; try { statement = connection.createStatement(); resultSet = statement.executeQuery(sql); while (resultSet.next()) { long scn = resultSet.getLong(1); return String.valueOf(scn); } } catch (SQLException e) { e.printStackTrace(); } finally { close(statement, resultSet); } return "0"; } public List<String> getPkNames(String datasourceName, String tableName) { List<String> pkNames = Lists.newArrayList(); Statement statement = null; ResultSet resultSet = null; try { DatabaseMetaData metaData = connection.getMetaData(); resultSet = metaData.getPrimaryKeys(datasourceName, null, tableName); while (resultSet.next()) { pkNames.add(resultSet.getString("COLUMN_NAME")); } return pkNames; } catch (Exception e) { logger.error(datasourceName+"."+tableName+"oracle get table primary key returns null", e); } finally { close(statement, resultSet); } return null; } private void close(Statement statement, ResultSet resultSet){ try { if (resultSet != null) resultSet.close(); } catch (SQLException e1) { e1.printStackTrace(); } finally { try { if (statement != null) statement.close(); } catch (SQLException e) { e.printStackTrace(); } } } @Override public void connect() { String jdbcUrl = ORACLE_DRIVER_URL + authInfo.getAddress().getHostString() + ":" + authInfo.getAddress().getPort() + ":" + authInfo.getServerName(); try { this.connection = DriverManager.getConnection(jdbcUrl, authInfo.getUsername(), authInfo.getPassword()); } catch (SQLException e) { e.printStackTrace(); } } @Override public void reconnect() { disconnect(); connect(); } @Override public void disconnect() { if (connection != null) { try { unsubscribe(); connection.close(); } catch (SQLException e) { e.printStackTrace(); } } } @Override public void ack(CanalInstance canalInstance) { return; } @Override public boolean checkConsistent(CanalInstance canalInstance) { CanalEventParser eventParser = canalInstance.getEventParser(); MysqlEventParser parser = (MysqlEventParser) eventParser; AuthenticationInfo newAuthInfo = parser.getMasterInfo(); String newUrl = newAuthInfo.getAddress().getHostName() + newAuthInfo.getAddress().getPort(); String oldUrl = this.authInfo.getAddress().getHostName() + this.authInfo.getAddress().getPort(); if (Objects.equals(newUrl, oldUrl)) { return true; } return false; } @Override //回退到指定位置 public void rollback(CanalInstance canalInstance, Long batchId) { try { FileMixedMetaManager.saveScnLocal(canalInstance.getDestination(), batchId); currentScn.set(batchId); } catch (IOException e) { e.printStackTrace(); } } public void getCallableStatement(String sql) { CallableStatement callableStatement = null; try { callableStatement = connection.prepareCall(sql); callableStatement.execute(); } catch (SQLException e) { logger.error(e.getMessage(), e); e.printStackTrace(); StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); // 将出错的栈信息输出到printWriter中 e.printStackTrace(pw); pw.flush(); sw.flush(); try { sw.close(); } catch (IOException ex) { throw new RuntimeException(ex); } pw.close(); if (e.getErrorCode() == 1291) { throw new CanalServerException("1291"); } } finally { close(callableStatement, null); } } /** * description: 关闭查询会话 * * @param {} * @return void */ public void unsubscribe() { String endLogmnrSql = "BEGIN\n" + " dbms_logmnr.end_logmnr;\n" + " END;"; logger.warn(endLogmnrSql); getCallableStatement(endLogmnrSql); logger.warn("end_logmnr succeeded !"); } } 看下这个代码怎么加日志来排查问题
06-06
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值