canal同步mysql数据到kafka (depolyer client-adapter)

阶段一 ： 搭建zookeeper和kafka

https://blog.youkuaiyun.com/woshixiazaizhe/article/details/80610432
zookeeper：D:\Software\zookeeper\zookeeper-3.4.13\bin zkServer.cmd

kafka:
step1:进入 D:\Software\kafka\kafka_2.12-2.0.0

step2：启动kafka

    .\bin\windows\kafka-server-start.bat    .\config\server.properties

step3：创建kafka topic

λ bin\windows\kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic canal-01

topic canal-01创建成功查看所有topic

   λ bin\windows\kafka-topics.bat --list --zookeeper localhost:2181
__consumer_offsets
canal-01

****删除topic bin\kafka-topics.sh --delete --topic canal_01 --zookeeper localhost:2181

查看topic列表

bin\windows\kafka-topics.bat --list --zookeeper localhost:2181

step4 :启动生产者 producer

 bin\windows\kafka-console-producer.bat --broker-list localhost:9092 --topic canal-01

step5 :启动消费者 customer
此命令作废 可能由于版本原因
bin\windows\kafka-console-consumer.bat --zookeeper localhost:2181 --topic canal-01 --from-beginning

   bin\windows\kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic canal-01

阶段二 ： SpringBoot创建kafka生产和消费
springboot配置kafka生产者和消费者详解

1.加粗样式消费者配置

 <!--kafka-->
    <dependency>
      <groupId>org.springframework.kafka</groupId>
      <artifactId>spring-kafka</artifactId>
      <version>1.1.1.RELEASE</version>
    </dependency>
    <dependency>
      <groupId>org.apache.kafka</groupId>
      <artifactId>kafka_2.10</artifactId>
      <version>0.10.0.1</version>
    </dependency>

spring:
    kafka:
        bootstrap-servers: localhost:2181
        consumer:
            bootstrap-servers: localhost:9092
            auto-commit-interval: 100
            auto-offset-reset: latest
            enable-auto-commit: true
            group-id: test-consumer-group
            key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
            value-deserializer: org.apache.kafka.common.serialization.StringDeserializer

启动类添加注解

@EnableKafka

@Component
@Slf4j
public class KafkaConsumer {


    @KafkaListener(topics = {"canal-01"})
    public void listen(ConsumerRecord<String, String> record) {
        log.info(record.toString());
        System.out.println(record);
    }

}

2.生产方配置 直接启动canal client-adapter





启动canal -depolyer canal-client-adapter

{
    "data": [
        {
            "id": "300",
            "create_by": "fuckoff300",
            "create_date": "2017-12-19 13:18:06",
            "update_by": "jfeng300",
            "update_date": "2017-12-19 13:18:06",
            "del_flag": "0",
            "batch_no": "",
            "cost": "0",
            "weight": "0.0",
            "mileage": "0.0",
            "volume": "0.0",
            "load_number": "0",
            "unload_number": "0",
            "bat_status": "",
            "trucker_id": "",
            "confirm_date": "2019-03-14 15:04:56",
            "sign_date": "2019-03-14 15:04:56",
            "is_allow_edit": "",
            "dis_man_phone": "",
            "city_id": "",
            "remarks": "",
            "latest_time": "2019-03-14 15:04:56",
            "org_member_id": "",
            "settlement_user_id": "",
            "channel_bill_id": "",
            "channel_bill_no": ""
        }
    ],
    "database": "kxtx",
    "es": 1552547096000,
    "id": 2,
    "isDdl": false,
    "mysqlType": {
        "id": "varchar(36)",
        "create_by": "varchar(36)",
        "create_date": "datetime",
        "update_by": "varchar(36)",
        "update_date": "datetime",
        "del_flag": "tinyint(4)",
        "batch_no": "varchar(36)",
        "cost": "bigint(20)",
        "weight": "decimal(14,3)",
        "mileage": "decimal(9,3)",
        "volume": "decimal(9,4)",
        "load_number": "tinyint(4)",
        "unload_number": "tinyint(4)",
        "bat_status": "varchar(25)",
        "trucker_id": "varchar(36)",
        "confirm_date": "datetime",
        "sign_date": "datetime",
        "is_allow_edit": "char(1)",
        "dis_man_phone": "varchar(16)",
        "city_id": "varchar(36)",
        "remarks": "varchar(255)",
        "latest_time": "timestamp",
        "org_member_id": "varchar(36)",
        "settlement_user_id": "varchar(36)",
        "channel_bill_id": "varchar(50)",
        "channel_bill_no": "varchar(50)"
    },
    "old": null,
    "pkNames": [
        "id"
    ],
    "sql": "",
    "sqlType": {
        "id": 12,
        "create_by": 12,
        "create_date": 93,
        "update_by": 12,
        "update_date": 93,
        "del_flag": -6,
        "batch_no": 12,
        "cost": -5,
        "weight": 3,
        "mileage": 3,
        "volume": 3,
        "load_number": -6,
        "unload_number": -6,
        "bat_status": 12,
        "trucker_id": 12,
        "confirm_date": 93,
        "sign_date": 93,
        "is_allow_edit": 1,
        "dis_man_phone": 12,
        "city_id": 12,
        "remarks": 12,
        "latest_time": 93,
        "org_member_id": 12,
        "settlement_user_id": 12,
        "channel_bill_id": 12,
        "channel_bill_no": 12
    },
    "table": "dis_batch",
    "ts": 1552547097223,
    "type": "INSERT"
}