使用canal同步mysql数据到es

// 1. POM Dependencies (pom.xml)
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.7.5</version>
    </parent>

    <groupId>com.example</groupId>
    <artifactId>binlog-es-sync</artifactId>
    <version>1.0.0</version>

    <dependencies>
        <!-- Spring Boot Starter -->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter</artifactId>
        </dependency>

        <!-- MySQL Connector -->
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>8.0.30</version>
        </dependency>

        <!-- Elasticsearch -->
        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-high-level-client</artifactId>
            <version>7.17.6</version>
        </dependency>

        <!-- Canal Client -->
        <dependency>
            <groupId>com.alibaba.otter</groupId>
            <artifactId>canal.client</artifactId>
            <version>1.1.6</version>
        </dependency>

        <!-- Lombok -->
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <optional>true</optional>
        </dependency>
    </dependencies>
</project>

// 2. User Entity (User.java)
package com.example.entity;

import lombok.Data;
import java.io.Serializable;

@Data
public class User implements Serializable {
    private Long id;
    private String username;
    private String email;
    private Integer age;
}

// 3. Elasticsearch Configuration (ElasticsearchConfig.java)
package com.example.config;

import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.elasticsearch.client.RestClient;
import org.springframework.beans.factory.annotation.Value;
import org.apache.http.HttpHost;

@Configuration
public class ElasticsearchConfig {
    @Value("${elasticsearch.host}")
    private String host;

    @Value("${elasticsearch.port}")
    private int port;

    @Bean
    public RestHighLevelClient restHighLevelClient() {
        return new RestHighLevelClient(
            RestClient.builder(
                new HttpHost(host, port, "http")
            )
        );
    }
}

// 4. Binlog Sync Service (BinlogSyncService.java)
package com.example.service;

import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.CanalConnectors;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.Message;
import com.example.entity.User;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.stereotype.Service;

import java.net.InetSocketAddress;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

@Service
@Slf4j
@RequiredArgsConstructor
public class BinlogSyncService implements InitializingBean {
    private final RestHighLevelClient restHighLevelClient;

    private static final String CANAL_HOST = "localhost";
    private static final int CANAL_PORT = 11111;
    private static final String DESTINATION = "example";

    @Override
    public void afterPropertiesSet() throws Exception {
        startBinlogSync();
    }

    private void startBinlogSync() {
        ExecutorService executorService = Executors.newSingleThreadExecutor();
        executorService.submit(() -> {
            CanalConnector connector = CanalConnectors.newSingleConnector(
                new InetSocketAddress(CANAL_HOST, CANAL_PORT), 
                DESTINATION, "", ""
            );

            try {
                connector.connect();
                connector.subscribe(".*\\..*");
                connector.rollback();

                while (true) {
                    Message message = connector.getWithoutAck(100);
                    long batchId = message.getId();
                    
                    try {
                        if (batchId == -1 || message.getEntries().isEmpty()) {
                            Thread.sleep(1000);
                            continue;
                        }

                        processEntries(message.getEntries());
                        connector.ack(batchId);
                    } catch (Exception e) {
                        connector.rollback(batchId);
                        log.error("Binlog sync error", e);
                    }
                }
            } catch (Exception e) {
                log.error("Canal connection error", e);
            } finally {
                connector.disconnect();
            }
        });
    }

    private void processEntries(List<CanalEntry.Entry> entries) throws Exception {
        for (CanalEntry.Entry entry : entries) {
            if (entry.getEntryType() == CanalEntry.EntryType.ROWDATA) {
                CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
                String tableName = entry.getHeader().getTableName();

                if ("user".equals(tableName)) {
                    processUserTableEntries(rowChange);
                }
            }
        }
    }

    private void processUserTableEntries(CanalEntry.RowChange rowChange) throws Exception {
        for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
            Map<String, Object> userData = new HashMap<>();

            // 根据操作类型处理数据
            switch (rowChange.getEventType()) {
                case INSERT:
                    userData = parseColumns(rowData.getAfterColumnsList());
                    syncToElasticsearch("user", userData);
                    break;
                case UPDATE:
                    userData = parseColumns(rowData.getAfterColumnsList());
                    syncToElasticsearch("user", userData);
                    break;
                case DELETE:
                    userData = parseColumns(rowData.getBeforeColumnsList());
                    deleteFromElasticsearch("user", userData);
                    break;
                default:
                    break;
            }
        }
    }

    private Map<String, Object> parseColumns(List<CanalEntry.Column> columns) {
        Map<String, Object> data = new HashMap<>();
        for (CanalEntry.Column column : columns) {
            data.put(column.getName(), column.getValue());
        }
        return data;
    }

    private void syncToElasticsearch(String index, Map<String, Object> data) throws Exception {
        String id = data.get("id").toString();
        IndexRequest request = new IndexRequest(index)
            .id(id)
            .source(data);

        restHighLevelClient.index(request, RequestOptions.DEFAULT);
        log.info("Synced data to Elasticsearch: {}", data);
    }

    private void deleteFromElasticsearch(String index, Map<String, Object> data) throws Exception {
        String id = data.get("id").toString();
        restHighLevelClient.delete(
            new org.elasticsearch.action.delete.DeleteRequest(index, id), 
            RequestOptions.DEFAULT
        );
        log.info("Deleted data from Elasticsearch: {}", data);
    }
}

// 5. Main Application (BinlogEsSyncApplication.java)
package com.example;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

@SpringBootApplication
public class BinlogEsSyncApplication {
    public static void main(String[] args) {
        SpringApplication.run(BinlogEsSyncApplication.class, args);
    }
}

// 6. application.properties
elasticsearch.host=localhost
elasticsearch.port=9200