Go 操作kafka

最新推荐文章于 2025-05-14 21:14:00 发布

lisus2007

最新推荐文章于 2025-05-14 21:14:00 发布

阅读量480

点赞数

CC 4.0 BY-SA版权

分类专栏： GO语言文章标签： golang kafka

本文链接：https://blog.youkuaiyun.com/lisus2007/article/details/132885501

GO语言专栏收录该内容

23 篇文章

订阅专栏

本文详细介绍了如何使用Go语言的Sarama库实现Kafka生产者，包括配置参数、发送消息，以及连接Kafka并消费消息的过程。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

1.下载和安装

go get github.com/IBM/sarama

2.连接

func main() {
   config := sarama.NewConfig()
   config.Producer.RequiredAcks = sarama.WaitForAll //发送完数据需要leader和follow都确认
   config.Producer.Partitioner = sarama.NewRandomPartitioner
   config.Producer.Return.Successes = true         // 成功交付的消息将在success channel返回
   config.Producer.Retry.Max = 0                   //重新发送的次数
   config.Producer.Timeout = time.Microsecond * 10 //等待 WaitForAck的时间
   config.Producer.Idempotent = true               //开启幂等性
   //config.Producer.Transaction =

   //构造一个消息

   //连接kafka
   client, err := sarama.NewSyncProducer([]string{"192.168.0.4:9092"}, config)
   if err != nil {
      fmt.Println("producer closed,err:", err)
      return
   }
   defer client.Close()

   //发送消息
   for i := 0; i < 5; i++ {
      msg := &sarama.ProducerMessage{}
      msg.Topic = "user-top"
      msg.Value = sarama.StringEncoder("this is test log:" + strconv.Itoa(i))
      pid, offset, err := client.SendMessage(msg)
      if err != nil {
         fmt.Println("send msg failed,err:", err)
         return
      }
      fmt.Printf("pid:%v offset:%v\n", pid, offset)
   }

}

3.连接kafka消费消息

func main() {
   consumer, err := sarama.NewConsumer([]string{"192.168.0.4:9092"}, nil)
   if err != nil {
      fmt.Println("err1", err)
      fmt.Printf("fail to start consumer,err:%v\n", err)
      return
   }
   partitionList, err := consumer.Partitions("user-topic")
   if err != nil {
      fmt.Println("err2", err)
      fmt.Printf("fail to get list of partition:err%v\n", err)
      return
   }
   for _, partition := range partitionList {
      // 针对每个分区创建一个对应的分区消费者
      pc, err := consumer.ConsumePartition("user-topic", int32(partition), sarama.OffsetNewest)
      if err != nil {
         fmt.Println("err3", err)
         fmt.Printf("failed to start consumer for partition %d,err:%v\n", partition, err)
         return
      }
      defer pc.AsyncClose()
      // 异步从每个分区消费信息
      go func(sarama.PartitionConsumer) {
         for msg := range pc.Messages() {
            fmt.Printf("Partition:%d Offset:%d Key:%v Value:%v", msg.Partition, msg.Offset, msg.Key, msg.Value)
         }
      }(pc)
   }
}