otter自由门完成全量数据同步方法:
一、先参考otter官方文档完成channel的配置
二、涉及的部分数据库操作
(1)数据库账户授权
CREATE USER retl@'%' IDENTIFIED BY 'retl';
GRANT USAGE ON *.* TO `retl`@'%';
GRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO `retl`@'%';
GRANT SELECT, INSERT, UPDATE, DELETE, EXECUTE ON `retl`.* TO `retl`@'%';
/* 业务表授权,这里可以限定只授权同步业务的表 */
GRANT SELECT, INSERT, UPDATE, DELETE ON *.* TO `retl`@'%';
(2)删除现有的系统表
DROP TABLE IF EXISTS retl.retl_buffer;
DROP TABLE IF EXISTS retl.retl_mark;
DROP TABLE IF EXISTS retl.xdual;
(3)创建系统表
CREATE TABLE retl_buffer
(
ID BIGINT AUTO_INCREMENT, ## 无意义,自增即可
TABLE_ID INT(11) NOT NULL, ## tableId, 可通过该链接查询:http://otter.alibaba-inc.com/data_media_list.htm,即序号这一列,如果配置的是正则,需要指定full_name,当前table_id设置为0.
FULL_NAME varchar(512), ## schemaName + '.' + tableName (如果明确指定了table_id,可以不用指定full_name)
TYPE CHAR(1) NOT NULL, ## I/U/D ,分别对应于insert/update/delete
PK_DATA VARCHAR(256) NOT NULL, ## 多个pk之间使用char(1)进行分隔
GMT_CREATE TIMESTAMP NOT NULL, ## 无意义,系统时间即可
GMT_MODIFIED TIMESTAMP NOT NULL, ## 无意义,系统时间即可
CONSTRAINT RETL_BUFFER_ID PRIMARY KEY (ID)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE retl_mark
(
ID BIGINT AUTO_INCREMENT,
CHANNEL_ID INT(11),
CHANNEL_INFO varchar(128),
CONSTRAINT RETL_MARK_ID PRIMARY KEY (ID)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE xdual (
ID BIGINT(20) NOT NULL AUTO_INCREMENT,
X timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (ID)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=UTF8;
(4)插入初始化数据
INSERT INTO retl.xdual(id, x) VALUES (1,now()) ON DUPLICATE KEY UPDATE x = NOW();
(5)同步一条数据
insert into `retl`.`retl_buffer` ( `TABLE_ID`, `FULL_NAME`, `TYPE`, `PK_DATA`, `GMT_CREATE`, `GMT_MODIFIED`) values ('4', 'db_user.t_user_test', 'I', '1', now(), NOW());
(6)全量数据同步
insert into retl.retl_buffer(ID,TABLE_ID, FULL_NAME,TYPE,PK_DATA,GMT_CREATE,GMT_MODIFIED)
(select null,4,'db_user.t_user_test','I',id,now(),now() from db_user.t_user_test);
(7)按照条件数据同步
insert into retl.retl_buffer(ID,TABLE_ID, FULL_NAME,TYPE,PK_DATA,GMT_CREATE,GMT_MODIFIED)(select null,4,'db_user.t_user_test','I',id,now(),now() from db_user.t_user_test WHERE id > 12);
三、涉及的参考文献:
https://www.modb.pro/db/562955