go与kafka(github.com/Shopify/sarama)

最新推荐文章于 2025-07-16 09:23:54 发布

我的自学笔记

最新推荐文章于 2025-07-16 09:23:54 发布

阅读量7.6k

点赞数

CC 4.0 BY-SA版权

分类专栏： go

本文链接：https://blog.youkuaiyun.com/sinat_26682309/article/details/90300367

go 专栏收录该内容

21 篇文章

订阅专栏

本文介绍了如何使用Go语言的Sarama库与Kafka进行交互，包括启动Zookeeper和Kafka服务，以及创建Producer和Consumer进行消息生产和消费的操作流程。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

启动kafka

分别命令行启动zookeeper和kafka

.\bin\windows\zookeeper-server-start.bat .\config\zookeeper.properties
.\bin\windows\kafka-server-start.bat .\config\server.properties

producer

import (
	"fmt"
	"github.com/Shopify/sarama"
)

func main() {
	config := sarama.NewConfig()
	config.Producer.RequiredAcks = sarama.WaitForAll  //赋值为-1：这意味着producer在follower副本确认接收到数据后才算一次发送完成。
	config.Producer.Partitioner = sarama.NewRandomPartitioner  //写到随机分区中，默认设置8个分区
	config.Producer.Return.Successes = true
	msg := &sarama.ProducerMessage{}
	msg.Topic = `nginx_log`
	msg.Value = sarama.StringEncoder("this is a good test")
	client,err := sarama.NewSyncProducer([]string{"127.0.0.1:9092"},config)
	if err != nil{
		fmt.Println("producer close err, ",err)
		return
	}
	defer client.Close()
	pid, offset, err := client.SendMessage(msg)

	if err != nil{
		fmt.Println("send message failed, ", err)
		return
	}
	fmt.Printf("分区ID:%v, offset:%v \n", pid, offset)
	}

consumer

func main() {
	var wg sync.WaitGroup
	consumer, err := sarama.NewConsumer([]string{"127.0.0.1:9092"}, nil)
	if err != nil{
		fmt.Println("Failed to start consumer: %s", err)
		return
	}
	partitionList, err := consumer.Partitions("nginx_log")  //获得该topic所有的分区
	if err != nil{
		fmt.Println("Failed to get the list of partition:, ",err)
		return
	}
	fmt.Println(partitionList)

	for partition := range partitionList{
		pc, err := consumer.ConsumePartition("nginx_log", int32(partition), sarama.OffsetNewest)
		if err != nil{
			fmt.Println("Failed to start consumer for partition %d: %s\n", partition, err)
			return
		}
                wg.Add(1)
		go func(sarama.PartitionConsumer) { //为每个分区开一个go协程去取值
			for msg := range pc.Messages(){  //阻塞直到有值发送过来，然后再继续等待
				fmt.Printf("Partition:%d, Offset:%d, key:%s, value:%s\n", msg.Partition, msg.Offset, string(msg.Key),string(msg.Value))
				}
                        defer pc.AsyncClose()
			wg.Done()
			}(pc)
	}
	wg.Wait()
	consumer.Close()
}