kafka API操作topic

最新推荐文章于 2023-12-05 08:10:48 发布

原创最新推荐文章于 2023-12-05 08:10:48 发布 · 3.3k 阅读

6 ·

CC 4.0 BY-SA版权

kafka 专栏收录该内容

3 篇文章

订阅专栏

本文介绍了在Kafka 2.1.0版本中如何通过API进行Topic的管理，包括连接初始化、查询Topic信息、删除与创建Topic、列出所有Topic及发送与消费消息的实践操作。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

kafka 版本：2.1.0

具体更多的明细API用法见官网：kafka官网API

pom信息：

<dependency>
        <groupId>org.apache.kafka</groupId>
        <artifactId>kafka_2.11</artifactId>
        <version>2.1.0</version>
</dependency>
<!-- kafka Producer&Consumer API  -->
<dependency>
      <groupId>org.apache.kafka</groupId>
      <artifactId>kafka-clients</artifactId>
       <version>2.1.0</version>
</dependency>

连接初始化：

private static final String topicName = "test";
    private static final String broker = "host:port";
    public static AdminClient client;

    static void initClient() {
        if (client != null) {
            client.close();
            client = null;
        }
        try {
            Properties admainProp = new Properties();
            admainProp.put("bootstrap.servers", broker);
            client = AdminClient.create(admainProp);
            System.out.println("初始化KAFKA-Client成功!");
        } catch (Exception e) {
            System.out.println("初始化Client失败！");
        }
    }

查询Topic信息：

public  void getTopicInfo(String topicName) {
        DescribeTopicsResult topicsResult = client.describeTopics(Arrays.asList(topicName.split(",")));
        KafkaFuture<Map<String, TopicDescription>> allInfo = topicsResult.all();
        try {
            Map<String, TopicDescription> all = allInfo.get();
            System.out.println(JSON.toJSONString(all));
            TopicDescription t1 = all.get("topicName");
            System.out.println(t1.toString()+":"+t1.isInternal());
        } catch (Exception e) {
            System.out.println("查询topic信息失败！：" + e.getMessage());
        }

    }

删除Topic：

 public  void deleteTopic(String topicName){
        DeleteTopicsResult topicsResult= client.deleteTopics(Arrays.asList("topicName"));
            System.out.println("topic删除成功！");
    }

创建Topic:

public  void createTopics() {
        try {
            NewTopic newTopic = new NewTopic("topicName", 1, (short) 1);
            CreateTopicsOptions createTopicsOptions = new CreateTopicsOptions();
//        createTopicsOptions.validateOnly(true);
            createTopicsOptions.timeoutMs(Integer.valueOf("1000"));
            client.createTopics(Arrays.asList(newTopic), createTopicsOptions);
            System.out.println("Topic创建成功！");
        } catch (Exception e) {
            System.out.println("创建失败：" + e.getMessage());
        }
    }

列出所有Topic:

System.out.println("topics:"+client.listTopics().listings().get().toString());

测试发送消息：

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.errors.ProducerFencedException;
import org.apache.kafka.common.serialization.StringSerializer;

public class KafkaProductTest {
    private static Log logger = LogFactory.getLog(KafkaProductTest.class);
    private static final String topic = "tttt";
    private static final String broker = "host:port";
    public static Producer<String, String> producer;

    //producer初始化
    static synchronized void initProducer() {
        if (producer != null) {
            producer.close();
            producer = null;
        }
        Properties props = new Properties();
        props.put("bootstrap.servers", broker);
        props.put("acks", "all");
        // props.put("delivery.timeout.ms", 3000);
        props.put("batch.size", 16384);
        props.put("linger.ms", 1);
        props.put("buffer.memory", 33554432);
        props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        //定义生产者对象，该类指定了两个参数的泛型，第一个参数表示分区键值的类型，第二参数表示消息类型
        producer = new KafkaProducer<String, String>(props);
    }

//批量发送数据
    public void testBachProfucer() {
        try {
            producer.beginTransaction();
            for (int i = 0; i < 100; i++) {
                producer.send(new ProducerRecord<String, String>("ttt", Integer.toString(i), Integer.toString(i)));
            }
            producer.commitTransaction();
        } catch (ProducerFencedException e) {
            producer.close();
        }
        producer.close();
    }

    public static void main(String[] args) {
        initProducer();
        System.out.println("开始发送"+System.currentTimeMillis());
        for(int i=0;i<1000000;i++) {
            producer.send(new ProducerRecord<String, String>("ttt", "1", " {\"all\":\"all"+i+"\"}"));
        }System.out.println("发送完成"+System.currentTimeMillis());
        producer.close();
    }

}

消费数据：

public class KafkaConsumeTest {
    private  static Log logger = LogFactory.getLog(KafkaConsumeTest.class);
    private static  final String  topic ="test";
    private static  final  String broker="host:port";
    public  static  KafkaConsumer<String,String> consumer ;

    static  synchronized  void initConsumer(){
        if(consumer!=null){
            consumer.close();
            consumer=null;
        }
        Properties props = new Properties();
        props.put("bootstrap.servers", broker);
        props.put("group.id", "test-id");
        //自动维护offset
//        props.put("enable.auto.commit", "true");
        //手动管理
        props.put("enable.auto.commit", "false");
        props.put("auto.commit.interval.ms", "100");
        props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        consumer = new KafkaConsumer<String, String>(props);
        System.out.println("初始化完成");
//        按照topic的 partition消费配置，代替上面的配置，配一个就行了，指定partition消费
//        TopicPartition partition0 = new TopicPartition(topic, 0);
//        TopicPartition partition1 = new TopicPartition(topic, 1);
//        consumer.assign(Arrays.asList(partition0, partition1));
    }

    public static void main(String[] args) {
        initConsumer();
        //可订阅多个topic消费
        consumer.subscribe(Arrays.asList(topic));
        while (true) {
            ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(100));//.poll(100);//连接timeout
            //手动提交offset
            //consumer.commitAsync();
            for (ConsumerRecord<String, String> record : records) {
                 System.out.println(record.value());
            }
        }
    }

}