Kafka Connect是如何保证数据精准一次

最新推荐文章于 2025-02-15 14:58:31 发布

原创最新推荐文章于 2025-02-15 14:58:31 发布 · 466 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#Kafak Connect

Kafka 专栏收录该内容

27 篇文章

订阅专栏

本文深入剖析了KafkaConnect中生产者与消费者的默认配置细节，包括串行化与反串行化类的选择、重试策略、超时设置及客户端ID配置等关键参数，为开发者提供实践指导。

通过阅读源码可以看到Kafka Connect在对生产者以及消费者的默认配置对我们在编写生成者与消费者的启发。

    static Map<String, Object> producerConfigs(ConnectorTaskId id,
                                               String defaultClientId,
                                               WorkerConfig config,
                                               ConnectorConfig connConfig,
                                               Class<? extends Connector> connectorClass,
                                               ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy) {
        Map<String, Object> producerProps = new HashMap<>();
        producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, Utils.join(config.getList(WorkerConfig.BOOTSTRAP_SERVERS_CONFIG), ","));
        producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer");
        producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer");
        // These settings will execute infinite retries on retriable exceptions. They *may* be overridden via configs passed to the worker,
        // but this may compromise the delivery guarantees of Kafka Connect.
        // 这些设置将对可重试的异常执行无限重试。它们*可能*通过传递给工作程序的配置被覆盖，但这可能会损害Kafka Connect的交付保证。
        producerProps.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, Integer.toString(Integer.MAX_VALUE));
        producerProps.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, Long.toString(Long.MAX_VALUE));
        producerProps.put(ProducerConfig.ACKS_CONFIG, "all");
        producerProps.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "1");
        producerProps.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, Integer.toString(Integer.MAX_VALUE));
        producerProps.put(ProducerConfig.CLIENT_ID_CONFIG, defaultClientId);
        // User-specified overrides
        // 用户指定的替代
        producerProps.putAll(config.originalsWithPrefix("producer."));

        // Connector-specified overrides
        // 连接器指定的替代
        Map<String, Object> producerOverrides =
                connectorClientConfigOverrides(id, connConfig, connectorClass, ConnectorConfig.CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX,
                        ConnectorType.SOURCE, ConnectorClientConfigRequest.ClientType.PRODUCER,
                        connectorClientConfigOverridePolicy);
        producerProps.putAll(producerOverrides);

        return producerProps;
    }

    static Map<String, Object> consumerConfigs(ConnectorTaskId id,
                                               WorkerConfig config,
                                               ConnectorConfig connConfig,
                                               Class<? extends Connector> connectorClass,
                                               ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy) {
        // Include any unknown worker configs so consumer configs can be set globally on the worker
        // and through to the task
        Map<String, Object> consumerProps = new HashMap<>();

        consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, SinkUtils.consumerGroupId(id.connector()));
        consumerProps.put(ConsumerConfig.CLIENT_ID_CONFIG, "connector-consumer-" + id);
        consumerProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,
                Utils.join(config.getList(WorkerConfig.BOOTSTRAP_SERVERS_CONFIG), ","));
        consumerProps.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
        consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
        consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer");
        consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer");

        consumerProps.putAll(config.originalsWithPrefix("consumer."));
        // Connector-specified overrides
        Map<String, Object> consumerOverrides =
                connectorClientConfigOverrides(id, connConfig, connectorClass, ConnectorConfig.CONNECTOR_CLIENT_CONSUMER_OVERRIDES_PREFIX,
                        ConnectorType.SINK, ConnectorClientConfigRequest.ClientType.CONSUMER,
                        connectorClientConfigOverridePolicy);
        consumerProps.putAll(consumerOverrides);

        return consumerProps;
    }