springboot集成kafka

1. 添加依赖

在 pom.xml 中添加 Kafka 的依赖：

<dependencies>
    <!-- Spring Boot Starter for Kafka -->
    <dependency>
        <groupId>org.springframework.kafka</groupId>
        <artifactId>spring-kafka</artifactId>
    </dependency>

    <!-- Spring Boot Starter Web (可选，如果需要Web支持) -->
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-web</artifactId>
    </dependency>
</dependencies>

2. 配置 Kafka

在 application.properties 或 application.yml 中配置 Kafka 的相关属性：

application.properties 配置示例：

# Kafka 服务器地址
spring.kafka.bootstrap-servers=localhost:9092

# 生产者配置
spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer
spring.kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer

# 消费者配置
spring.kafka.consumer.group-id=my-group
spring.kafka.consumer.auto-offset-reset=earliest
spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer
spring.kafka.consumer.value-deserializer=org.apache.kafka.common.serialization.StringDeserializer

application.yml 配置示例：

spring:
  kafka:
    bootstrap-servers: localhost:9092
    producer:
      key-serializer: org.apache.kafka.common.serialization.StringSerializer
      value-serializer: org.apache.kafka.common.serialization.StringSerializer
    consumer:
      group-id: my-group
      auto-offset-reset: earliest
      key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
      value-deserializer: org.apache.kafka.common.serialization.StringDeserializer

3. 创建 Kafka 生产者

在 Spring Boot 中，可以使用 KafkaTemplate 来发送消息。

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Service;

@Service
public class KafkaProducerService {

    private static final String TOPIC = "my-topic";

    @Autowired
    private KafkaTemplate<String, String> kafkaTemplate;

    public void sendMessage(String message) {
        kafkaTemplate.send(TOPIC, message);
        System.out.println("Produced message: " + message);
    }
}

4. 创建 Kafka 消费者

使用 @KafkaListener 注解来监听 Kafka 消息。

import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.stereotype.Service;

@Service
public class KafkaConsumerService {

    @KafkaListener(topics = "my-topic", groupId = "my-group")
    public void consume(String message) {
        System.out.println("Consumed message: " + message);
    }
}

5. 创建 Controller（可选）

如果需要通过 HTTP 接口发送消息，可以创建一个 Controller。

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;

@RestController
public class KafkaController {

    @Autowired
    private KafkaProducerService kafkaProducerService;

    @GetMapping("/send")
    public String sendMessage(@RequestParam String message) {
        kafkaProducerService.sendMessage(message);
        return "Message sent: " + message;
    }
}

6. 启动 Kafka

确保 Kafka 服务已经启动。可以使用以下命令启动 Kafka（需要提前安装 Kafka 和 Zookeeper）：

# 启动 Zookeeper
zookeeper-server-start.sh config/zookeeper.properties

# 启动 Kafka
kafka-server-start.sh config/server.properties

7. 运行 Spring Boot 项目

启动 Spring Boot 项目后，可以通过以下方式测试 Kafka 集成：

1. 访问 http://localhost:8080/send?message=HelloKafka 发送消息。

2. 查看控制台日志，确认消费者是否接收到消息。

8. 高级配置（可选）

8.1 自定义序列化器

如果需要发送复杂对象，可以自定义序列化器和反序列化器。

 

import org.apache.kafka.common.serialization.Serializer;
import com.fasterxml.jackson.databind.ObjectMapper;

public class JsonSerializer<T> implements Serializer<T> {

    private final ObjectMapper objectMapper = new ObjectMapper();

    @Override
    public byte[] serialize(String topic, T data) {
        try {
            return objectMapper.writeValueAsBytes(data);
        } catch (Exception e) {
            throw new RuntimeException("Error serializing JSON", e);
        }
    }
}

在配置中指定自定义序列化器：

spring.kafka.producer.value-serializer=com.example.JsonSerializer
spring.kafka.consumer.value-deserializer=com.example.JsonDeserializer

8.2 批量消费

如果需要批量消费消息，可以配置 ConcurrentKafkaListenerContainerFactory：

import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.core.ConsumerFactory;

@Configuration
public class KafkaConfig {

    @Bean
    public ConcurrentKafkaListenerContainerFactory<String, String> batchFactory(
            ConsumerFactory<String, String> consumerFactory) {
        ConcurrentKafkaListenerContainerFactory<String, String> factory =
                new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory);
        factory.setBatchListener(true); // 启用批量消费
        return factory;
    }
}

在消费者中使用批量消费：

@KafkaListener(topics = "my-topic", groupId = "my-group", containerFactory = "batchFactory")
public void consume(List<String> messages) {
    System.out.println("Consumed messages: " + messages);
}