有时候我们一个项目里面会处理多个mq队列，笔者这里举几个简单的例子，配置文件如下：
[rabbit_mq_xxx_test] queueName=queuexxx host=xxx port=5672 username=uname password=upass virtualHost=v_xxx_mq exchangeName=xxx_EX_TEST routingKey=xxx_RK_TEST
[rabbit_mq_aaa_test]
queueName=queueaaa
host=aaa
port=5672
username=uname
password=upass
virtualHost=v_aaa_mq
exchangeName=aaa_EX_TEST
routingKey=aaa_RK_TEST
连接端代码可以如下：
type QueueBind struct {
Queue string
Exchange string
RoutingKey string
Ch *amqp.Channel
Conn *amqp.Connection
Url string
HostIp string
Mutex sync.RWMutex
LastUsed time.Time
}
var (
queueMap map[string]*QueueBind
managersMutex sync.RWMutex
)
func InitconfigByPrefix(preFix string) *QueueBind {
preFix = preFix + configs.FACTORY
queueName := configs.GetServerProp(preFix, "queueName", "")
host := configs.GetServerProp(preFix, "host", "")
port := configs.GetServerProp(preFix, "port", "")
username := configs.GetServerProp(preFix, "username", "")
password := configs.GetServerProp(preFix, "password", "")
virtualHost := configs.GetServerProp(preFix, "virtualHost", "")
exchangeName := configs.GetServerProp(preFix, "exchangeName", "")
routingKey := configs.GetServerProp(preFix, "routingKey", "")
mqUrl := utils.Sprintf("amqp://%s:%s@%s:%s/%s", username, password, host, port, virtualHost)
return &QueueBind{
Queue: queueName,
Exchange: exchangeName,
RoutingKey: routingKey,
Url: mqUrl,
HostIp: host,
}
}
func initMQConn() {
queueMap = map[string]*QueueBind{
model.XXX: InitconfigByPrefix("rabbit_mq_xxx_"),
model.API_AAA: InitconfigByPrefix("rabbit_mq_aaa_"),
}
for k, v := range queueMap {
err := v.InitConnection()
if err != nil {
panic(err)
}
queueMap[k] = v
}
}
// 获取连接管理器
func GetMQManager(apiType string) (*QueueBind, error) {
managersMutex.RLock()
manager, exists := queueMap[apiType]
managersMutex.RUnlock()
if !exists {
return nil, fmt.Errorf("MQ manager not found for apiType: %s", apiType)
}
return manager, nil
}
// 初始化连接
func (m *QueueBind) InitConnection() error {
m.Mutex.Lock()
defer m.Mutex.Unlock()
// 关闭现有连接
if m.Ch != nil {
m.Ch.Close()
}
if m.Conn != nil {
m.Conn.Close()
}
// 创建新连接
conn, err := amqp.Dial(m.Url)
if err != nil {
return fmt.Errorf("failed to connect to RabbitMQ: %v", err)
}
ch, err := conn.Channel()
if err != nil {
conn.Close()
return fmt.Errorf("failed to open channel: %v", err)
}
// 声明队列
_, err = ch.QueueDeclare(
m.Queue,
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
if err != nil {
ch.Close()
conn.Close()
return fmt.Errorf("failed to declare queue: %v", err)
}
// 声明交换机
err = ch.ExchangeDeclare(
m.Exchange,
"direct",
true, // durable
false, // delete when unused
false, // internal
false, // no-wait
nil, // arguments
)
if err != nil {
ch.Close()
conn.Close()
return fmt.Errorf("failed to declare exchange: %v", err)
}
// 绑定队列
err = ch.QueueBind(
m.Queue,
m.RoutingKey,
m.Exchange,
false,
nil,
)
if err != nil {
ch.Close()
conn.Close()
return fmt.Errorf("failed to bind queue: %v", err)
}
m.Conn = conn
m.Ch = ch
m.LastUsed = time.Now()
return nil
}
// 重置连接
func (m *QueueBind) ResetConnection() {
m.Mutex.Lock()
defer m.Mutex.Unlock()
if m.Ch != nil {
m.Ch.Close()
m.Ch = nil
}
if m.Conn != nil {
m.Conn.Close()
m.Conn = nil
}
}
// 检查连接是否有效
func (m *QueueBind) isConnectionValid() bool {
m.Mutex.RLock()
defer m.Mutex.RUnlock()
if m.Conn == nil || m.Ch == nil {
return false
}
// 检查连接是否关闭
if m.Conn.IsClosed() {
return false
}
return true
}
// 发送消息（带重试和自动重连）
func (m *QueueBind) SendMessageWithRetry(msg []byte, maxRetries int) error {
var lastErr error
for i := 0; i < maxRetries; i++ {
// 尝试发送消息
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
err := m.Ch.PublishWithContext(
ctx,
m.Exchange,
m.RoutingKey,
false, // mandatory
false, // immediate
amqp.Publishing{
ContentType: "text/plain",
Body: msg,
DeliveryMode: amqp.Persistent,
})
cancel()
if err == nil {
m.LastUsed = time.Now()
return nil
}
lastErr = err
// 发送失败，检查错误类型决定是否重连
if shouldReconnect(err) {
m.ResetConnection()
// 重连后立即重试，不等待
if i < maxRetries-1 {
if reconnectErr := m.InitConnection(); reconnectErr != nil {
lastErr = reconnectErr
// 重连失败，等待后继续
time.Sleep(time.Duration(i+1) * 100 * time.Millisecond)
continue
}
// 重连成功，立即重试
continue
}
}
// 不需要重连或最后一次重试，等待后继续
if i < maxRetries-1 {
time.Sleep(time.Duration(i+1) * 100 * time.Millisecond)
}
}
return fmt.Errorf("failed to send message after %d retries, last error: %v", maxRetries, lastErr)
}
// 判断错误类型是否需要重新连接
func shouldReconnect(err error) bool {
if err == nil {
return false
}
errorStr := err.Error()
// 这些错误通常需要重新建立连接
reconnectErrors := []string{
"channel is not open",
"connection is not open",
"EOF",
"connection closed",
"channel/connection is not open",
}
for _, reconnectErr := range reconnectErrors {
if strings.Contains(errorStr, reconnectErr) {
return true
}
}
return false
}
// SendMs send msg to mq
func SendMs(apiType string, msg []byte) (hostIp string, err error) {
manager, err := GetMQManager(apiType)
if err != nil {
return "", err
}
return manager.HostIp, manager.SendMessageWithRetry(msg, configs.RETRY_COUNT)
}
使用的时候可以直接调用SendMs，msg可以根据自己的业务选择json或者protobuf字符串；
但是此处有一个bug，就是服务端把queue直接删除掉，客户端是不会报错的，最好是做一个定时任务，定时去检查queue是否存在并重新建立queue：
// 检查队列是否存在
func (m *QueueBind) checkQueueExists() (bool, error) {
if m.Ch == nil || m.Ch.IsClosed() {
return false, fmt.Errorf("channel is closed")
}
_, err := m.Ch.QueueDeclarePassive(
m.Queue, // name
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
if err != nil {
// 如果队列不存在，会返回404错误
if strings.Contains(err.Error(), "404") {
return false, nil
}
return false, err
}
return true, nil
}
// 重新创建队列和绑定
func (m *QueueBind) recreateQueueAndBind() error {
// 确保连接有效
if !m.isConnectionValid() {
if err := m.InitConnection(); err != nil {
return err
}
}
// 声明队列
_, err := m.Ch.QueueDeclare(
m.Queue, // name
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
if err != nil {
return fmt.Errorf("failed to declare queue: %v", err)
}
// 声明交换机
err = m.Ch.ExchangeDeclare(
m.Exchange,
"direct",
true, // durable
false, // delete when unused
false, // internal
false, // no-wait
nil, // arguments
)
if err != nil {
return fmt.Errorf("failed to declare exchange: %v", err)
}
// 绑定队列
err = m.Ch.QueueBind(
m.Queue,
m.RoutingKey,
m.Exchange,
false,
nil,
)
if err != nil {
return fmt.Errorf("failed to bind queue: %v", err)
}
return nil
}
// 判断是否是路由错误
func isRoutingError(err error) bool {
if err == nil {
return false
}
errorStr := err.Error()
routingErrors := []string{
"NO_ROUTE",
"no queue",
"returned",
"404",
}
for _, routingErr := range routingErrors {
if strings.Contains(errorStr, routingErr) {
return true
}
}
return false
}
// 启用Confirm模式
func (m *QueueBind) enableConfirmMode() error {
if m.Ch == nil {
return fmt.Errorf("channel is nil")
}
return m.Ch.Confirm(false) // false表示不等待旧的消息确认
}
// 等待消息确认
func (m *QueueBind) waitForConfirmation() bool {
select {
case confirmed := <-m.Ch.NotifyPublish(make(chan amqp.Confirmation, 1)):
return confirmed.Ack
case <-time.After(5 * time.Second):
return false
}
}
如果系统对时间要求比较高，新增加一个定时任务，每隔30分钟检查一下queue的情况：
// 添加队列存在性检查
if exists, err := m.checkQueueExists(); err != nil {
Log.Errorf("failed to check queue existence: %v", err)
} else if !exists {
configs.Log.Warn("Queue %s does not exist, recreating...", m.Queue)
if err := m.recreateQueueAndBind(); err != nil {
Log.Errorf("failed to recreate queue: %v", err)
}
}
如果系统对时间不敏感可以考虑下面这个方法：
func (m *QueueBind) SendMessageWithConfirm(msg []byte, maxRetries int) error {
var lastErr error
// 启用Confirm模式
if err := m.enableConfirmMode(); err != nil {
return err
}
for i := 0; i < maxRetries; i++ {
// 检查队列是否存在（只在第一次尝试时）
if i == 0 {
if exists, err := m.checkQueueExists(); err != nil || !exists {
if err := m.recreateQueueAndBind(); err != nil {
lastErr = err
time.Sleep(time.Duration(i+1) * 100 * time.Millisecond)
continue
}
}
}
// 发送消息
err := m.Ch.Publish(
m.Exchange,
m.RoutingKey,
true, // mandatory - 消息必须被路由到队列
false, // immediate
amqp.Publishing{
ContentType: "text/plain",
Body: msg,
DeliveryMode: amqp.Persistent,
})
if err != nil {
lastErr = err
m.ResetConnection()
time.Sleep(time.Duration(i+1) * 100 * time.Millisecond)
continue
}
// 等待确认
confirmed := m.waitForConfirmation()
if confirmed {
m.LastUsed = time.Now()
return nil
} else {
lastErr = fmt.Errorf("message not confirmed by broker")
// 消息未被确认，可能是队列不存在，尝试重新创建
if recreateErr := m.recreateQueueAndBind(); recreateErr != nil {
lastErr = recreateErr
}
m.ResetConnection()
time.Sleep(time.Duration(i+1) * 100 * time.Millisecond)
}
}
return fmt.Errorf("failed to send message after %d retries, last error: %v", maxRetries, lastErr)
}
以上是发送数据的操作，对于消费端的代码，此处提供一个简单的实现：
// Mq consumer Return
const (
RtFAIL = iota
RtSUCCESS
RtSKIP
RtTURN
)
// DealFunc consumer deal function type
type DealFunc func(msg []byte) int
// Consumer interface
type Consumer struct {
// Deal consumer deal function
Deal DealFunc
Type string
}
// ConsumerConfig 消费者配置
type ConsumerConfig struct {
MaxRetries int // 最大重试次数
RetryInterval time.Duration // 重试间隔
ReconnectWait time.Duration // 重连等待时间
PrefetchCount int // 预取数量
EnableQoS bool // 是否启用QoS
HealthCheckInterval time.Duration // 健康检查间隔
MaxInFlight int // 最大并发处理消息数
ProcessingTimeout time.Duration // 消息处理超时时间
}
var DefaultConsumerConfig = ConsumerConfig{
MaxRetries: configs.RETRY_COUNT,
RetryInterval: 2 * time.Second,
ReconnectWait: 3 * time.Second,
PrefetchCount: 1,
EnableQoS: true,
HealthCheckInterval: 30 * time.Second,
MaxInFlight: 10,
ProcessingTimeout: 5 * time.Minute,
}
// ConsumerMetrics 消费者指标
type ConsumerMetrics struct {
ProcessedCount int64 `json:"processed_count,omitempty"`
SuccessCount int64 `json:"success_count,omitempty"`
FailureCount int64 `json:"failure_count,omitempty"`
PanicCount int64 `json:"panic_count,omitempty"`
TotalDuration time.Duration `json:"total_duration,omitempty"`
LastError string `json:"last_error,omitempty"`
LastErrorTime time.Time `json:"last_error_time,omitempty"`
}
// ConsumerStatus 消费者状态
type ConsumerStatus struct {
Type string `json:"type,omitempty"`
WorkerID int `json:"worker_id,omitempty"`
Status string `json:"status,omitempty"`
StartTime time.Time `json:"start_time,omitempty"`
LastActivity time.Time `json:"last_activity,omitempty"`
ErrorCount int `json:"error_count,omitempty"`
Metrics ConsumerMetrics `json:"metrics,omitempty"`
}
// BackpressureController 背压控制器
type BackpressureController struct {
maxInFlight int
currentInFlight int32
semaphore chan struct{}
mutex sync.RWMutex
}
// 全局变量
var (
shutdownSignal chan struct{}
consumerStatusMap sync.Map
globalOnce sync.Once
)
func init() {
shutdownSignal = make(chan struct{})
}
// GetType return consumer type
func (consumer Consumer) GetType() string {
return consumer.Type
}
// NewBackpressureController 创建背压控制器
func NewBackpressureController(maxInFlight int) *BackpressureController {
if maxInFlight <= 0 {
maxInFlight = 10
}
return &BackpressureController{
maxInFlight: maxInFlight,
semaphore: make(chan struct{}, maxInFlight),
}
}
// Acquire 获取处理许可
func (b *BackpressureController) Acquire() bool {
select {
case b.semaphore <- struct{}{}:
atomic.AddInt32(&b.currentInFlight, 1)
return true
default:
return false
}
}
// Release 释放处理许可
func (b *BackpressureController) Release() {
select {
case <-b.semaphore:
atomic.AddInt32(&b.currentInFlight, -1)
default:
}
}
// CurrentInFlight 获取当前正在处理的消息数量
func (b *BackpressureController) CurrentInFlight() int {
return int(atomic.LoadInt32(&b.currentInFlight))
}
// Validate 配置验证
func (config *ConsumerConfig) Validate() error {
if config.MaxRetries < 0 {
return errors.New("MaxRetries must be non-negative")
}
if config.PrefetchCount < 0 {
return errors.New("PrefetchCount must be non-negative")
}
if config.RetryInterval <= 0 {
return errors.New("RetryInterval must be positive")
}
if config.HealthCheckInterval <= 0 {
return errors.New("HealthCheckInterval must be positive")
}
if config.ProcessingTimeout <= 0 {
return errors.New("ProcessingTimeout must be positive")
}
return nil
}
// ConsumerStart 开启n个消费者同时执行
func ConsumerStart(n int, queueType string, dealFunc DealFunc) error {
return ConsumerStartWithConfig(n, queueType, dealFunc, DefaultConsumerConfig)
}
// ConsumerStartWithConfig start consumer with custom config
func ConsumerStartWithConfig(n int, queueType string, dealFunc DealFunc, config ConsumerConfig) error {
if err := config.Validate(); err != nil {
return fmt.Errorf("invalid consumer config: %v", err)
}
defer func() {
if er := recover(); er != nil {
erLog := utils.Sprintf("%v panic:%v", utils.RunFuncName(), er)
configs.Log.Error(erLog)
}
}()
for i := 0; i < n; i++ {
consumer := Consumer{
Type: queueType,
Deal: dealFunc,
}
go startConsumerWithRetry(consumer, config, i)
}
configs.Log.Infof("Started %d consumers for type: %s", n, queueType)
return nil
}
// startConsumerWithRetry 带重试机制的消费者启动
func startConsumerWithRetry(consumer Consumer, config ConsumerConfig, workerID int) {
consumerTag := fmt.Sprintf("%s-worker-%d", consumer.Type, workerID)
// 初始化状态
status := ConsumerStatus{
Type: consumer.Type,
WorkerID: workerID,
Status: "starting",
StartTime: time.Now(),
LastActivity: time.Now(),
ErrorCount: 0,
}
updateConsumerStatus(consumerTag, status)
for retryCount := 0; retryCount < config.MaxRetries; retryCount++ {
configs.Log.Infof("Starting consumer type: %s, worker: %d, retry: %d",
consumer.Type, workerID, retryCount)
// 更新状态
status.Status = "running"
status.LastActivity = time.Now()
updateConsumerStatus(consumerTag, status)
err := startConsumer(consumer, config, consumerTag)
if err == nil {
// 正常退出，可能是主动关闭
configs.Log.Infof("Consumer stopped normally type: %s, worker: %d",
consumer.Type, workerID)
status.Status = "stopped"
status.LastActivity = time.Now()
updateConsumerStatus(consumerTag, status)
return
}
// 更新错误计数
status.ErrorCount++
status.Status = "error"
status.LastActivity = time.Now()
updateConsumerStatus(consumerTag, status)
configs.Log.Errorf("Consumer stopped with error, will retry type: %s, worker: %d, error: %v, retry: %d, maxRetries: %d",
consumer.Type,
workerID,
err,
retryCount+1,
config.MaxRetries)
if retryCount < config.MaxRetries-1 {
time.Sleep(config.RetryInterval)
// 尝试重新初始化连接
ReInitMQConn()
}
}
configs.Log.Errorf("Consumer failed after max retries type: %s, worker: %d, maxRetries: %d",
consumer.Type,
workerID,
config.MaxRetries)
status.Status = "failed"
status.LastActivity = time.Now()
updateConsumerStatus(consumerTag, status)
}
// startConsumer 启动单个消费者
func startConsumer(consumer Consumer, config ConsumerConfig, consumerTag string) error {
defer func() {
if err := recover(); err != nil {
configs.Log.Errorf("Consumer panic recovered type: %s, worker: %s, panic: %v",
consumer.Type,
consumerTag,
err)
}
}()
ch, err := createConsumerChannel(consumer.GetType(), config)
if err != nil {
return fmt.Errorf("failed to create channel: %v", err)
}
defer safelyCloseChannel(ch, consumer.GetType(), consumerTag)
// 设置关闭监听
closeChan := make(chan *amqp.Error, 1)
notifyClose := ch.NotifyClose(closeChan)
// 设置取消监听
cancelChan := make(chan string, 1)
notifyCancel := ch.NotifyCancel(cancelChan)
v, ok := queueMap[consumer.GetType()]
if !ok {
return errors.New("channel type not found in queueMap")
}
// 开始消费
msgs, err := startConsuming(ch, v.Queue, consumerTag)
if err != nil {
return fmt.Errorf("failed to start consuming: %v", err)
}
configs.Log.Infof("Consumer started successfully type: %s, worker: %s, queue: %s",
consumer.Type,
consumerTag,
v.Queue)
// 创建背压控制器
backpressure := NewBackpressureController(config.MaxInFlight)
return processMessages(consumer, msgs, notifyClose, notifyCancel, consumerTag, backpressure, config)
}
// createConsumerChannel 创建消费者channel
func createConsumerChannel(chType string, config ConsumerConfig) (*amqp.Channel, error) {
v, ok := queueMap[chType]
if !ok {
return nil, fmt.Errorf("channel type %s not found", chType)
}
// 检查连接是否有效
if !v.isConnectionValid() {
configs.Log.Warnf("Connection invalid, reinitializing type: %s", chType)
if err := v.InitConnection(); err != nil {
return nil, fmt.Errorf("failed to reinitialize connection: %v", err)
}
}
ch, err := v.Conn.Channel()
if err != nil {
return nil, fmt.Errorf("failed to open channel: %v", err)
}
// 设置心跳（如果配置了）
if config.HealthCheckInterval > 0 {
// RabbitMQ 连接时已经可以设置心跳，这里主要是应用层的心跳检查
}
// 声明队列
_, err = ch.QueueDeclare(
v.Queue,
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
if err != nil {
ch.Close()
return nil, fmt.Errorf("failed to declare queue %s: %v", v.Queue, err)
}
// 声明交换机
err = ch.ExchangeDeclare(
v.Exchange,
"direct",
true, // durable
false, // delete when unused
false, // internal
false, // no-wait
nil, // arguments
)
if err != nil {
ch.Close()
return nil, fmt.Errorf("failed to declare exchange %s: %v", v.Exchange, err)
}
// 绑定队列
err = ch.QueueBind(
v.Queue,
v.RoutingKey,
v.Exchange,
false,
nil,
)
if err != nil {
ch.Close()
return nil, fmt.Errorf("failed to bind queue: %v", err)
}
// 设置QoS
if config.EnableQoS {
err = ch.Qos(config.PrefetchCount, 0, false)
if err != nil {
ch.Close()
return nil, fmt.Errorf("failed to set QoS: %v", err)
}
}
return ch, nil
}
// startConsuming 开始消费消息
func startConsuming(ch *amqp.Channel, queueName, consumerTag string) (<-chan amqp.Delivery, error) {
msgs, err := ch.Consume(
queueName, // queue
consumerTag, // consumer tag
false, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
if err != nil {
return nil, fmt.Errorf("failed to consume from queue %s: %v", queueName, err)
}
return msgs, nil
}
// processMessages 处理消息循环
func processMessages(consumer Consumer, msgs <-chan amqp.Delivery,
notifyClose <-chan *amqp.Error, notifyCancel <-chan string, consumerTag string,
backpressure *BackpressureController, config ConsumerConfig) error {
// 健康检查ticker
healthTicker := time.NewTicker(config.HealthCheckInterval)
defer healthTicker.Stop()
for {
select {
case <-shutdownSignal:
configs.Log.Infof("Received shutdown signal, stopping consumer type: %s, worker: %s",
consumer.Type, consumerTag)
return nil
case <-healthTicker.C:
// 执行健康检查
if backpressure.CurrentInFlight() >= backpressure.maxInFlight {
configs.Log.Warnf("High backpressure detected type: %s, worker: %s, inFlight: %d",
consumer.Type, consumerTag, backpressure.CurrentInFlight())
}
case e, ok := <-notifyCancel:
if !ok {
return errors.New("cancel channel closed")
}
configs.Log.Errorf("Queue manually deleted type: %s, worker: %s, reason: %v",
consumer.Type,
consumerTag,
e)
return fmt.Errorf("queue canceled: %s", e)
case e, ok := <-notifyClose:
if !ok {
return errors.New("close channel closed")
}
configs.Log.Errorf("Channel closed type: %s, worker: %s, reason: %v",
consumer.Type,
consumerTag,
e)
return fmt.Errorf("channel closed: %v", e)
case msg, ok := <-msgs:
if !ok {
return errors.New("message channel closed")
}
// 背压控制
if !backpressure.Acquire() {
configs.Log.Warnf("Backpressure triggered, requeuing message type: %s, worker: %s, inFlight: %d",
consumer.Type, consumerTag, backpressure.CurrentInFlight())
// 拒绝消息并重新入队
msg.Nack(false, true)
continue
}
// 使用 goroutine 处理消息，避免阻塞主循环
go func(msg amqp.Delivery) {
defer backpressure.Release()
processSingleMessage(consumer, msg, consumerTag, config.ProcessingTimeout)
}(msg)
}
}
}
// processSingleMessage 处理单条消息
func processSingleMessage(consumer Consumer, msg amqp.Delivery, consumerTag string, timeout time.Duration) {
var result int
startTime := time.Now()
// 更新状态
updateConsumerActivity(consumerTag)
defer func() {
processingTime := time.Since(startTime)
recordMetrics(consumerTag, result, processingTime)
if err := recover(); err != nil {
configs.Log.Errorf("Message processing panic type: %s, worker: %s, panic: %v, messageId: %s",
consumer.Type,
consumerTag,
err,
msg.MessageId)
// 发生panic时拒绝消息并重新入队
msg.Nack(false, true)
recordPanic(consumerTag)
}
}()
configs.Log.Debugf("Processing message type: %s, worker: %s, messageId: %s, bodySize: %d",
consumer.Type,
consumerTag,
msg.MessageId,
len(msg.Body))
// 带超时处理消息
result = processWithTimeout(consumer, msg.Body, timeout)
processingTime := time.Since(startTime)
configs.Log.Debugf("Message processed type: %s, worker: %s, messageId: %s, result: %d, processingTime: %s",
consumer.Type,
consumerTag,
msg.MessageId,
result,
processingTime)
// 处理消息确认
handleMessageAck(consumer, msg, result, consumerTag)
}
// processWithTimeout 带超时处理消息
func processWithTimeout(consumer Consumer, msgBody []byte, timeout time.Duration) int {
if timeout <= 0 {
// 没有超时限制
return consumer.Deal(msgBody)
}
resultChan := make(chan int, 1)
go func() {
defer func() {
if err := recover(); err != nil {
configs.Log.Errorf("Deal function panic: %v", err)
resultChan <- RtFAIL
}
}()
resultChan <- consumer.Deal(msgBody)
}()
select {
case result := <-resultChan:
return result
case <-time.After(timeout):
configs.Log.Errorf("Message processing timeout after %v", timeout)
return RtFAIL
}
}
// handleMessageAck 处理消息确认
func handleMessageAck(consumer Consumer, msg amqp.Delivery, result int, consumerTag string) {
switch result {
case RtSUCCESS, RtSKIP, RtTURN:
// 确认消息
if err := msg.Ack(false); err != nil {
configs.Log.Errorf("Failed to ack message type: %s, worker: %s, messageId: %s, error: %v",
consumer.Type,
consumerTag,
msg.MessageId,
err)
} else {
recordSuccess(consumerTag)
}
case RtFAIL:
// 拒绝消息并重新入队
if err := msg.Nack(false, true); err != nil {
configs.Log.Errorf("Failed to nack message worker: %s, messageId: %s, error: %v",
consumerTag,
msg.MessageId,
err)
} else {
recordFailure(consumerTag)
}
default:
configs.Log.Warnf("Unknown result code, rejecting message type: %s, worker: %s, messageId: %s, result: %d",
consumer.Type,
consumerTag,
msg.MessageId,
result)
msg.Nack(false, true)
recordFailure(consumerTag)
}
}
// safelyCloseChannel 安全关闭channel
func safelyCloseChannel(ch *amqp.Channel, chType, consumerTag string) {
if ch != nil {
if err := ch.Close(); err != nil {
configs.Log.Errorf("Error closing channel type: %s, worker: %s, error: %v",
chType,
consumerTag,
err)
} else {
configs.Log.Debugf("Channel closed successfully type: %s, worker: %s",
chType,
consumerTag)
}
}
}
// 状态管理函数
func updateConsumerStatus(consumerTag string, status ConsumerStatus) {
consumerStatusMap.Store(consumerTag, status)
}
func updateConsumerActivity(consumerTag string) {
if value, ok := consumerStatusMap.Load(consumerTag); ok {
if status, ok := value.(ConsumerStatus); ok {
status.LastActivity = time.Now()
consumerStatusMap.Store(consumerTag, status)
}
}
}
// 指标记录函数
func recordMetrics(consumerTag string, result int, processingTime time.Duration) {
value, ok := consumerStatusMap.Load(consumerTag)
if !ok {
return
}
status, ok := value.(ConsumerStatus)
if !ok {
return
}
status.Metrics.ProcessedCount++
status.Metrics.TotalDuration += processingTime
switch result {
case RtSUCCESS, RtSKIP, RtTURN:
status.Metrics.SuccessCount++
case RtFAIL:
status.Metrics.FailureCount++
}
consumerStatusMap.Store(consumerTag, status)
}
func recordSuccess(consumerTag string) {
// 在 recordMetrics 中已经处理
}
func recordFailure(consumerTag string) {
// 在 recordMetrics 中已经处理
}
func recordPanic(consumerTag string) {
value, ok := consumerStatusMap.Load(consumerTag)
if !ok {
return
}
status, ok := value.(ConsumerStatus)
if !ok {
return
}
status.Metrics.PanicCount++
status.Metrics.LastError = "panic occurred"
status.Metrics.LastErrorTime = time.Now()
consumerStatusMap.Store(consumerTag, status)
}
// GetAllConsumerStatus 获取所有消费者状态
func GetAllConsumerStatus() []ConsumerStatus {
var statuses []ConsumerStatus
consumerStatusMap.Range(func(key, value interface{}) bool {
if status, ok := value.(ConsumerStatus); ok {
statuses = append(statuses, status)
}
return true
})
return statuses
}
// GetConsumerStatus 获取特定消费者状态
func GetConsumerStatus(consumerTag string) (ConsumerStatus, bool) {
value, ok := consumerStatusMap.Load(consumerTag)
if !ok {
return ConsumerStatus{}, false
}
status, ok := value.(ConsumerStatus)
return status, ok
}
// GracefulShutdown 优雅关闭所有消费者
func GracefulShutdown() {
globalOnce.Do(func() {
configs.Log.Info("Initiating graceful shutdown of all consumers")
close(shutdownSignal)
// 等待一段时间让消费者完成当前消息处理
time.Sleep(5 * time.Second)
configs.Log.Info("Graceful shutdown completed")
})
}
// ReInitMQConn 重新初始化MQ连接
func ReInitMQConn() {
configs.Log.Info("Reinitializing MQ connections")
managersMutex.Lock()
defer managersMutex.Unlock()
for name, manager := range queueMap {
configs.Log.Infof("Reinitializing MQ connection name: %s", name)
if err := manager.InitConnection(); err != nil {
configs.Log.Errorf("Failed to reinitialize MQ connection name: %s, error: %v", name, err)
} else {
configs.Log.Infof("Successfully reinitialized MQ connection name: %s", name)
}
}
}
具体使用的时候创建3个消费端线程进行消费：
//创建3个线程消费
err := rabbitmq.ConsumerStart(3, model.API_XXX, func(msg []byte) int {
return rabbitmq.RtSUCCESS
})
if nil != err {
panic(err)
}
扫一扫
472
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
