由于之前项目需要,了解到了阿里优秀框架canal,使用起来真是很不错很方便,分库分表情况下用来缓存或是ES数据的同步很便捷。在尝试过程中,走了点弯路,总结此文,希望能够对观者有所帮助。
Canal官方下载地址
https://github.com/alibaba/canal/releases
下载的canal.deployer启动canal服务端,然后编写客户端程序消费数据,此方式对于应用部署和维护显然不是很方便。为此结合canal.deployer代码编写与spring整合方式,使web应用启动时即能启动服务端,又能消费数据。
canal消费方采用策略模式,定义BaseProcess抽象类,包含processInsert、processUpdate、processDelete三个抽象方法,分别用于处理三种类型的数据操作。封装processConvert方法将RowChage中的数据反射为javaBean对象。
1. Canal服务端代码CanalServer
package com.scy.canal.server;
import java.io.FileInputStream;
import java.util.List;
import java.util.Properties;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.alibaba.otter.canal.deployer.CanalController;
/**
* canal服务端代码
* @author suicy
*
*/
public class CanalServer {
private static final Log logger = LogFactory.getLog(CanalServer.class);
private static final String CLASSPATH_URL_PREFIX = "classpath:";
private CanalController controller;
private List<String> configs;
public void startup() {
logger.debug("CanalServer startup 准备启动canal服务端...");
try {
Properties properties = new Properties();
for (String conf : configs) {
if (conf.startsWith(CLASSPATH_URL_PREFIX)) {
conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX);
properties.load(CanalServer.class.getClassLoader().getResourceAsStream(conf));
} else {
properties.load(new FileInputStream(conf));
}
}
logger.debug("CanalController创建开始...");
controller = new CanalController(properties);
logger.debug("CanalController创建结束...");
controller.start();
logger.debug("CanalServer startup 启动canal服务端成功!");
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
try {
logger.info("## stop the canal server");
controller.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping canal Server:\n{}", e);
} finally {
logger.info("## canal server is down.");
}
}
});
} catch (Throwable e) {
logger.error("CanalServer startup 启动canal服务端失败,", e);
System.exit(0);
}
}
public void shutdown() {
try {
controller.stop();
} catch (Throwable e) {
logger.error("CanalServer shutdown canal服务端异常,", e);
}
}
public void setConfigs(List<String> configs) {
this.configs = configs;
}
}
2. 服务端配置文件spring-config-canal-server.xml
<?xml version="1.0" encoding="utf-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
<bean class="com.scy.canal.server.CanalServer" init-method="startup" destroy-method="shutdown">
<property name="configs">
<list>
<value>classpath:props/canal.properties</value>
</list>
</property>
</bean>
</beans>
3. Canal数据变更封装类CanalRowChange
package com.scy.canal.entity;
import java.io.Serializable;
import java.util.List;
import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
import com.alibaba.otter.canal.protocol.CanalEntry.RowData;
public class CanalRowChange implements Serializable{
private static final long serialVersionUID = -90027012566550680L;
private String schemaName;
private String tableName;
private List<RowData> rowData;
private EventType eventType;
public String getSchemaName() {
return schemaName;
}
public void setSchemaName(String schemaName) {
this.schemaName = schemaName;
}
public String getTableName() {
return tableName;
}
public void setTableName(String tableName) {
this.tableName = tableName;
}
public List<RowData> getRowData() {
return rowData;
}
public void setRowData(List<RowData> rowData) {
this.rowData = rowData;
}
public EventType getEventType() {
return eventType;
}
public void setEventType(EventType eventType) {
this.eventType = eventType;
}
}
4. Canal消费端代码CanalConsumer
package com.scy.canal.client;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.CanalConnectors;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.protocol.CanalEntry.Column;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
import com.alibaba.otter.canal.protocol.CanalEntry.RowData;
import com.scy.canal.entity.CanalRowChange;
import com.scy.canal.process.BaseProcess;
/**
* 数据消费
* @author suicy
*
*/
public class CanalConsumer {
private static final Logger logger = LoggerFactory.getLogger(CanalConsumer.class);
//数据处理类key为表名,value为对应的处理类
private Map<String, BaseProcess> processor;
public static volatile boolean running = true;
private String destination;
private String zkServers;
public void init(){
// 创建链接
logger.error("--------CanalConsumer destination:"+destination+" start------------");
Thread thread = new Thread(new Runnable() {
public void run() {
CanalConnector connector = CanalConnectors.newClusterConnector(zkServers, destination, null, null);
int batchSize = 1000;
try
{
connector.connect();
connector.subscribe();
while ( running ) {
Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据
long batchId = message.getId();
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
} else {
boolean res = parseData(message.getEntries(),destination);
if(res){
connector.ack(batchId);
}else{
connector.rollback(batchId);
}
}
}
} finally {
connector.disconnect();
}
}
});
thread.start();
}
/**
* 数据处理
* @param entrys
* @param destination2
* @return
*/
private boolean parseData(List<Entry> entrys, String destination2) {
for (Entry entry : entrys) {
if ( EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType()) || EntryType.TRANSACTIONEND.equals(entry.getEntryType()) ) {
continue;
}
RowChange rowChage = null;
try {
rowChage = RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) {
throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), e);
}
EventType eventType = rowChage.getEventType();
String schemaName = entry.getHeader().getSchemaName();
String tableName = entry.getHeader().getTableName();
/*1. 输出数据变更日志*/
if( logger.isErrorEnabled() ){
if(eventType == EventType.INSERT || eventType == EventType.DELETE || eventType == EventType.UPDATE ){
logger.error("================> binlog["+entry.getHeader().getLogfileName()+":"+ entry.getHeader().getLogfileOffset()+"] , "
+ "name["+schemaName+":"+tableName+"] , eventType : "+eventType);
for (RowData rowData : rowChage.getRowDatasList()) {
if (eventType == EventType.DELETE) {
printColumn(rowData.getBeforeColumnsList());
} else if (eventType == EventType.INSERT) {
printColumn(rowData.getAfterColumnsList());
} else if (eventType == EventType.UPDATE){
logger.error("-------> before");
printColumn(rowData.getBeforeColumnsList());
logger.error("-------> after");
printColumn(rowData.getAfterColumnsList());
}
}
}
}
/*2. 数据处理*/
if( EventType.INSERT.equals(eventType)||EventType.DELETE.equals(eventType)||EventType.UPDATE.equals(eventType)){
//构造CanalRowChange对象
CanalRowChange rowChange = bulidCanalRowChange(schemaName, tableName, eventType, rowChage.getRowDatasList());
/*根据表明获取数据处理类*/
if(processor.containsKey(tableName.toLowerCase())){
boolean res = false;
//根据事件类型调用相应的数据处理方法
if (EventType.UPDATE.equals(eventType)) {
res = processor.get(tableName).processUpdate(rowChange);
} else if (EventType.INSERT.equals(eventType)) {
res = processor.get(tableName).processInsert(rowChange);
} else if (EventType.DELETE.equals(eventType)) {
res = processor.get(tableName).processDelete(rowChange);
}
if(!res){
logger.error("================> binlog["+entry.getHeader().getLogfileName()+":"+ entry.getHeader().getLogfileOffset()+"] , "
+ "name["+schemaName+":"+tableName+"] , eventType : "+eventType);
return res;
}
}
}
}
return true;
}
private CanalRowChange bulidCanalRowChange(String schemaName, String tableName, EventType eventType, List<RowData> rowData){
CanalRowChange change = new CanalRowChange();
change.setSchemaName(schemaName);
change.setEventType(eventType);
change.setRowData(rowData);
change.setTableName(tableName);
return change;
}
private void printColumn(List<Column> columns) {
for (Column column : columns) {
logger.error(column.getName() + " : " + column.getValue() + " update=" + column.getUpdated());
}
}
protected void stop() {
if (!running) {
return;
}
running = false;
}
public Map<String, BaseProcess> getProcessor() {
return processor;
}
public void setProcessor(Map<String, BaseProcess> processor) {
this.processor = processor;
}
public String getDestination() {
return destination;
}
public void setDestination(String destination) {
this.destination = destination;
}
public String getZkServers() {
return zkServers;
}
public void setZkServers(String zkServers) {
this.zkServers = zkServers;
}
}
5. 消费端配置文件spring-config-canal-client.xml
<?xml version="1.0" encoding="utf-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd"
default-autowire="byName">
<bean id="canalConsumer_A1" class="com.scy.canal.client.CanalConsumer" init-method="init" destroy-method="stop" >
<property name="destination" value="A1" />
<property name="zkServers" value="${canal.zkServers}" />
<property name="processor">
<!-- 配置数据处理类key为表明,value为处理类 为BaseProcess子类-->
<map>
<entry key="goods" value-ref="goodsProcess"></entry>
</map>
</property>
</bean>
<bean id="goodsProcess" class="com.scy.canal.process.GoodsProcess"></bean>
</beans>
6. 数据消费父类BaseProcess
package com.scy.canal.process;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.springframework.core.convert.support.DefaultConversionService;
import com.alibaba.otter.canal.protocol.CanalEntry.Column;
import com.alibaba.otter.canal.protocol.CanalEntry.RowData;
import com.scy.canal.entity.CanalRowChange;
import com.scy.canal.entity.DateConverter;
/**
* 数据消费基类
*
* @author suicy
*
*/
public abstract class BaseProcess {
private DefaultConversionService convertor = new DefaultConversionService() {
{
addConverter(new DateConverter());
}
};
private Map<String, Map<String, Field>> clzFieldsCached;
/**
* 处理数据添加方法
* @param rowChange
* @return
*/
public abstract boolean processInsert(CanalRowChange rowChange);
/**
* 处理修改数据方法
* @param rowChange
* @return
*/
public abstract boolean processUpdate(CanalRowChange rowChange);
/**
* 处理删除数据方法
* @param rowChange
* @return
*/
public abstract boolean processDelete(CanalRowChange rowChange);
/**
* 数据转换方法
*
* @param rowChange
* @param clz
* @param isAfter
* @return
* @throws IllegalAccessException
* @throws InstantiationException
*/
public <T> List<T> processConvert(CanalRowChange rowChange, Class<T> clz, boolean isAfter)
throws InstantiationException, IllegalAccessException {
if (rowChange == null || clz == null) {
return null;
}
if (clzFieldsCached == null) {
clzFieldsCached = new HashMap<String, Map<String, Field>>();
}
Map<String, Field> fieldscached = clzFieldsCached.get(clz.getName());
if (fieldscached == null || fieldscached.size() <= 0) {
fieldscached = new HashMap<String, Field>();
for (Field field : clz.getDeclaredFields()) {
field.setAccessible(true);
fieldscached.put(field.getName().toLowerCase(), field);
}
clzFieldsCached.put(clz.getName(), fieldscached);
}
List<RowData> rowDatas = rowChange.getRowData();
if (rowDatas == null || rowDatas.size() <= 0) {
return null;
}
List<T> beans = new ArrayList<T>();
for (RowData rowData : rowDatas) {
T bean = clz.newInstance();
List<Column> cols = isAfter ? rowData.getAfterColumnsList() : rowData.getBeforeColumnsList();
if (cols == null || cols.size() <= 0) {
return null;
}
int count = 0;
for (Column col : cols) {
String name = col.getName();
String value = col.getValue();
Field field = fieldscached.get(name.toLowerCase());
if (field == null) {
continue;
}
if (StringUtils.isNotBlank(value)) {
Object nvalue = convertor.convert(value, field.getType());
field.set(bean, nvalue);
}
count++;
}
if (count != fieldscached.size()) {
return null;
}
beans.add(bean);
}
if (beans.size() != rowDatas.size()) {
return null;
}
return beans;
}
}
7. goods消费代码实例GoodsProcess
package com.scy.canal.process;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.scy.canal.entity.CanalRowChange;
import com.scy.canal.entity.Goods;
public class GoodsProcess extends BaseProcess{
private static final Log logger = LogFactory.getLog(GoodsProcess.class);
@Override
public boolean processInsert(CanalRowChange rowChange) {
try {
List<Goods> data = super.processConvert(rowChange, Goods.class, true);
logger.info("添加前"+data);
} catch (InstantiationException e) {
e.printStackTrace();
} catch (IllegalAccessException e) {
e.printStackTrace();
}
return true;
}
@Override
public boolean processUpdate(CanalRowChange rowChange) {
try {
List<Goods> data = super.processConvert(rowChange, Goods.class, false);
List<Goods> data2 = super.processConvert(rowChange, Goods.class, true);
logger.info("修改前:"+data);
logger.info("修改后:"+data2);
return true;
} catch (InstantiationException e) {
e.printStackTrace();
} catch (IllegalAccessException e) {
e.printStackTrace();
}
return false;
}
@Override
public boolean processDelete(CanalRowChange rowChange) {
try {
List<Goods> data = super.processConvert(rowChange, Goods.class, false);
logger.info("删除前:"+data);
return true;
} catch (InstantiationException e) {
e.printStackTrace();
} catch (IllegalAccessException e) {
e.printStackTrace();
}
return false;
}
}
8. Pom文件
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.scy</groupId>
<artifactId>canalDemo</artifactId>
<packaging>war</packaging>
<version>0.0.1-SNAPSHOT</version>
<name>canalDemo Maven Webapp</name>
<url>http://maven.apache.org</url>
<properties>
<jdk.version>1.6</jdk.version>
<project.build.sourceEncoding>GBK</project.build.sourceEncoding>
<org.springframework.version>3.1.1.RELEASE</org.springframework.version>
<mybatis.version>3.2.0</mybatis.version>
<jdbc.mysql.version>5.1.8</jdbc.mysql.version>
<junit.version>4.10</junit.version>
<velocity.version>1.7</velocity.version>
<velocity.tools.version>2.0</velocity.tools.version>
<canal.version>1.0.24</canal.version>
</properties>
<dependencies>
<!-- spring -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-beans</artifactId>
<version>${org.springframework.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jdbc</artifactId>
<version>${org.springframework.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-web</artifactId>
<version>${org.springframework.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-webmvc</artifactId>
<version>${org.springframework.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-orm</artifactId>
<version>${org.springframework.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-tx</artifactId>
<version>${org.springframework.version}</version>
</dependency>
<dependency>
<groupId>org.quartz-scheduler</groupId>
<artifactId>quartz</artifactId>
<version>1.8.4</version>
</dependency>
<!-- mybatis -->
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis</artifactId>
<version>${mybatis.version}</version>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis-spring</artifactId>
<version>1.1.0</version>
</dependency>
<!-- mySql -->
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>${jdbc.mysql.version}</version>
<type>jar</type>
<scope>runtime</scope>
</dependency>
<!-- log -->
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.14</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.6.0</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.6.0</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
<version>2.5</version>
</dependency>
<!-- ump 依赖包 begin -->
<dependency>
<groupId>org.aspectj</groupId>
<artifactId>aspectjweaver</artifactId>
<version>1.6.6</version>
</dependency>
<dependency>
<groupId>cglib</groupId>
<artifactId>cglib-nodep</artifactId>
<version>2.2</version>
</dependency>
<dependency>
<groupId>ant</groupId>
<artifactId>ant</artifactId>
<version>1.6.2</version>
</dependency>
<!-- net client -->
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>3.3</version>
</dependency>
<!-- chain -->
<dependency>
<groupId>commons-chain</groupId>
<artifactId>commons-chain</artifactId>
<version>1.2</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
<version>0.2.10</version>
</dependency>
<!-- test begin -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
<version>${org.springframework.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.deployer</artifactId>
<version>${canal.version}</version>
<exclusions>
<exclusion>
<groupId>org.springframework</groupId>
<artifactId>spring</artifactId>
</exclusion>
<exclusion>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</exclusion>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
</exclusion>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.client</artifactId>
<version>${canal.version}</version>
<exclusions>
<exclusion>
<groupId>org.springframework</groupId>
<artifactId>spring</artifactId>
</exclusion>
<exclusion>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</exclusion>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
</exclusion>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.9</version>
</dependency>
</dependencies>
<build>
<finalName>canalDemo</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding><!-- 指定编码格式,否则在DOS下运行mvn compile命令时会出现莫名的错误,因为系统默认使用GBK编码 -->
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>2.6</version>
<configuration>
<encoding>UTF-8</encoding><!-- 指定编码格式,否则在DOS下运行mvn命令时当发生文件资源copy时将使用系统默认使用GBK编码 -->
</configuration>
</plugin>
</plugins>
</build>
</project>
实例代码下载链接
http://download.youkuaiyun.com/download/suijiarui/9988995