springboot整合kafka

1. maven项目依赖包

<!-- kafka -->
<dependency>
	<groupId>org.apache.kafka</groupId>
	<artifactId>kafka-clients</artifactId>
	<version>1.0.0</version>
</dependency>
<dependency>
	<groupId>org.apache.kafka</groupId>
	<artifactId>kafka-streams</artifactId>
	<version>1.0.0</version>
</dependency>
<dependency>
	<groupId>org.springframework.kafka</groupId>
	<artifactId>spring-kafka</artifactId>
	<version>1.3.0.RELEASE</version>
</dependency>

2. 生产者

配置文件producer.xml

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
       xmlns:context="http://www.springframework.org/schema/context"
       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">

    <bean id="kafkaProducerProperites" class="java.util.HashMap">
        <constructor-arg>
            <map>
                <entry key="bootstrap.servers" value="localhost:9092" />
                <entry key="group.id" value="0"/>
                <entry key="retries" value="3"/>
                <entry key="batch.size" value="16384"/>
                <entry key="producer.type" value="async"/>
                <entry key="batch.num.messages" value="500"/>
                <entry key="linger.ms" value="1"/>
                <entry key="queue.buffering.max.ms" value="1000"/>
                <entry key="send.buffer.bytes" value="104857600"/>
                <entry key="key.serializer" value="org.apache.kafka.common.serialization.StringSerializer" />
                <entry key="value.serializer" value="org.apache.kafka.common.serialization.StringSerializer" />
            </map>
        </constructor-arg>
    </bean>

    <bean id="kafkaProducerFactory" class="org.springframework.kafka.core.DefaultKafkaProducerFactory">
        <constructor-arg ref="kafkaProducerProperites"/>
    </bean>

    <bean id="kafkaTemplate" class="org.springframework.kafka.core.KafkaTemplate">
        <constructor-arg ref="kafkaProducerFactory" />
        <constructor-arg name="autoFlush" value="true"/>
        <property name="defaultTopic" value="defaultTopic"/>
    </bean>
</beans>

KafkaService类（起名随意），需要注入kafkaTemplate类用来发送消息。

import net.sf.json.JSONObject;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

@Service
@Transactional
public class KafkaService {

    @Autowired
    private KafkaTemplate kafkaTemplate;

    /**
     * 向kafka里传入数据
     * @param message 要传入的数据
     */
    public void sendMes(String message){
        //第一个参数是自己定义的topic的名字，第二个参数是你要发送的消息
        kafkaTemplate.send("TestTopic", message);
    }
}

主程序类

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.kafka.core.KafkaTemplate;

@SpringBootApplication
public class Application {

    public static void main(String[] args) {
	SpringApplication.run(Application.class, args);
    }

    @Bean
    public KafkaTemplate getKafkaTemplate(){
        ApplicationContext ctx = new ClassPathXmlApplicationContext("classpath:producer.xml");
        return ctx.getBean("kafkaTemplate", KafkaTemplate.class);
    }
}

 

3. 消费者

配置文件consumer.xml

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
       xmlns:context="http://www.springframework.org/schema/context"
       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">

    <!-- 包路径扫描，为监听类所在的地址，本例中为KafkaConsumerListener.java -->
    <context:component-scan base-package="com.xxx.consumer" />

    <bean id="kafkaConsumerProperites" class="java.util.HashMap">
        <constructor-arg>
            <map>
                <entry key="bootstrap.servers" value="localhost:9092" />
                <entry key="group.id" value="0"/>
                <entry key="enable.auto.commit" value="true"/>
                <entry key="auto.commit.interval.ms" value="1000"/>
                <entry key="session.timeout.ms" value="30000"/>
                <entry key="num.consumer.fetchers" value="10"/>
                <entry key="socket.receive.buffer.bytes" value="67108864"/>
                <entry key="key.deserializer" value="org.apache.kafka.common.serialization.StringDeserializer" />
                <entry key="value.deserializer" value="org.apache.kafka.common.serialization.StringDeserializer" />
            </map>
        </constructor-arg>
    </bean>

    <bean id="kafkaConsumerFactory" class="org.springframework.kafka.core.DefaultKafkaConsumerFactory">
        <constructor-arg ref="kafkaConsumerProperites"/>
    </bean>

    <!-- 可配置多个topic -->
    <bean id="consumerContainerProperties" class="org.springframework.kafka.listener.config.ContainerProperties">
        <constructor-arg name="topics">
            <list>
                <value>TestTopic</value>
                <value>TestTopic2</value>
            </list>
        </constructor-arg>
        <property name="messageListener" ref="kafkaConsumerListenerReqJson" />
    </bean>

    <bean id="conusmerContainer" class="org.springframework.kafka.listener.KafkaMessageListenerContainer">
        <constructor-arg ref="kafkaConsumerFactory"/>
        <constructor-arg ref="consumerContainerProperties"/>
    </bean>
</beans>

监听类KafkaConsumerListener

需要配置多个topic时，只需在onMessage方法上的KafkaListener中的topics注解中加入其他的（topics = {"test1","test2"}），也可以再创建一个监听类监听另一个注解（确认在配置文件中包扫描路径下）。

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.listener.MessageListener;
import org.springframework.stereotype.Component;

/**
 * kafka的消费者监听类，topic为TestTopic
 */
@Component("kafkaConsumerListener")
public class KafkaConsumerListener implements MessageListener<String, String> {

    @Override
    @KafkaListener(topics = "TestTopic")
    public void onMessage(ConsumerRecord<String, String> record) {
        
        String topic = record.topic();
        String key = record.key();
        //拿到从kafka里传回来的数据
        String val = record.value();
        long offset = record.offset();
        int partition = record.partition();
        //System.out.println("------------------------------------------------------------");
        System.out.printf("receive msg -- topic:%s   key:%s  val:%s  offset:%s  partition:%s \r\n",topic,key,val,offset,partition);
    }
}

主程序类

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.web.servlet.ServletComponentScan;
import org.springframework.context.annotation.ImportResource;

@ServletComponentScan
@SpringBootApplication
@ImportResource("classpath:consumer.xml")
public class Application {

	public static void main(String[] args) {
		SpringApplication.run(Application.class, args);
    }
}