RocketMQ的负载均衡模式和广播模式消息消费方式

RocketMQ的负载均衡模式和广播模式消息消费方式

1 负载均衡模式（集群消费）

消费者采用负载均衡方式消费消息，一个分组(Group)下的多个消费者共同消费队列消息，每个消费者处理的消息不同。一个Consumer Group中的各个Consumer实例分摊去消费消息，即一条消息只会投递到一个Consumer Group下面的一个实例。例如某个Topic有3个队列，其中一个Consumer Group 有 3 个实例，那么每个实例只消费其中的1个队列。集群消费模式是消费者默认的消费方式。

代码演示消费者1

package com.neu.rocketmq.example.quickstart;

import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;

import java.util.List;

/**
 * Created with IntelliJ IDEA.
 *
 * @Author: yqq
 * @Date: 2023/06/19/19:52
 * @Description: 消费者1
 */
public class BalanceConsumer {
    public static void main(String[] args) throws Exception {
        //实例化消息消费者，并指定组名
        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("group-consumer");
        //指定NameServer地址
        consumer.setNamesrvAddr("node1:9876");
        //订阅Topic
        consumer.subscribe("TopicTest","*");
        //负载均衡消费模式（可以不设置，默认是负载均衡模式）
        consumer.setMessageModel(MessageModel.CLUSTERING);
        //注册回调函数，处理消息
        consumer.registerMessageListener(new MessageListenerConcurrently() {
            @Override
            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> list, ConsumeConcurrentlyContext context) {
                try {
                    for (MessageExt msg : list){
                        String topic = msg.getTopic();
                        String msgBody = new String(msg.getBody(),"utf-8");
                        String tags = msg.getTags();
                        System.out.println("收到消息：" + " topic:" + topic + " ,tags:" + tags + " ,msg" + msgBody);

                    }
                }catch (Exception e){
                    e.printStackTrace();
                    //消费失败，稍后尝试消费
                    return ConsumeConcurrentlyStatus.RECONSUME_LATER;
                }
                //成功消费
                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
            }
        });
        //启动消费者
        consumer.start();
        System.out.printf("Consumer started.%n");
    }
}

代码演示消费者2

package com.neu.rocketmq.example.quickstart;

import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;

import java.util.List;

/**
 * Created with IntelliJ IDEA.
 *
 * @Author: yqq
 * @Date: 2023/06/19/19:52
 * @Description:
 */
public class BalanceConsumer2 {
    public static void main(String[] args) throws Exception {
        //实例化消息消费者，并指定组名
        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("group-consumer");
        //指定NameServer地址
        consumer.setNamesrvAddr("node1:9876");
        //订阅Topic
        consumer.subscribe("TopicTest","*");
        //负载均衡消费模式（可以不设置，默认是负载均衡模式）
        consumer.setMessageModel(MessageModel.CLUSTERING);
        //注册回调函数，处理消息
        consumer.registerMessageListener(new MessageListenerConcurrently() {
            @Override
            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> list, ConsumeConcurrentlyContext context) {
                try {
                    for (MessageExt msg : list){
                        String topic = msg.getTopic();
                        String msgBody = new String(msg.getBody(),"utf-8");
                        String tags = msg.getTags();
                        System.out.println("收到消息：" + " topic:" + topic + " ,tags:" + tags + " ,msg" + msgBody);

                    }
                }catch (Exception e){
                    e.printStackTrace();
                    //消费失败，稍后尝试消费
                    return ConsumeConcurrentlyStatus.RECONSUME_LATER;
                }
                //成功消费
                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
            }
                });
                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
            }
        });
        //启动消费者
        consumer.start();
        System.out.printf("Consumer started.%n");
    }
}

2广播消费

广播消费模式中消息将对一个Consumer Group下的各个Consumer实例都投递一遍。即使这些 Consumer属于同一个Consumer Group，消息也会被Consumer Group 中的每个Consumer都消费一次。实际上，是一个消费组下的每个消费者实例都获取到了topic下面的每个Message Queue去拉取消费。所以消息会投递到每个消费者实例。

代码演示消费者1

package com.neu.rocketmq.example.quickstart;

import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;

import java.util.List;

/**
 * Created with IntelliJ IDEA.
 *
 * @Author: yqq
 * @Date: 2023/06/19/19:52
 * @Description:
 */
public class BroadcastConsumer {
    public static void main(String[] args) throws Exception {
        //实例化消息消费者，并指定组名
        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("group-consumer");
        //指定NameServer地址
        consumer.setNamesrvAddr("node1:9876");
        //订阅Topic
        consumer.subscribe("TopicTest","*");
        //负载均衡消费模式（可以不设置，默认是负载均衡模式）
        consumer.setMessageModel(MessageModel.BROADCASTING);
        //注册回调函数，处理消息
        consumer.registerMessageListener(new MessageListenerConcurrently() {
            @Override
            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> list, ConsumeConcurrentlyContext context) {
                try {
                    for (MessageExt msg : list){
                        String topic = msg.getTopic();
                        String msgBody = new String(msg.getBody(),"utf-8");
                        String tags = msg.getTags();
                        System.out.println("收到消息：" + " topic:" + topic + " ,tags:" + tags + " ,msg" + msgBody);

                    }
                }catch (Exception e){
                    e.printStackTrace();
                    //消费失败，稍后尝试消费
                    return ConsumeConcurrentlyStatus.RECONSUME_LATER;
                }
                //成功消费
                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
            }
        });
        //启动消费者
        consumer.start();
        System.out.printf("Consumer started.%n");
    }
}

代码演示消费者2

package com.neu.rocketmq.example.quickstart;

import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;

import java.util.List;

/**
 * Created with IntelliJ IDEA.
 *
 * @Author: yqq
 * @Date: 2023/06/19/19:52
 * @Description:
 */
public class BroadcastConsumer2 {
    public static void main(String[] args) throws Exception {
        //实例化消息消费者，并指定组名
        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("group-consumer");
        //指定NameServer地址
        consumer.setNamesrvAddr("node1:9876");
        //订阅Topic
        consumer.subscribe("TopicTest","*");
        //负载均衡消费模式（可以不设置，默认是负载均衡模式）
        consumer.setMessageModel(MessageModel.BROADCASTING);
        //注册回调函数，处理消息
        consumer.registerMessageListener(new MessageListenerConcurrently() {
            @Override
            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> list, ConsumeConcurrentlyContext context) {
                try {
                    for (MessageExt msg : list){
                        String topic = msg.getTopic();
                        String msgBody = new String(msg.getBody(),"utf-8");
                        String tags = msg.getTags();
                        System.out.println("收到消息：" + " topic:" + topic + " ,tags:" + tags + " ,msg" + msgBody);

                    }
                }catch (Exception e){
                    e.printStackTrace();
                    //消费失败，稍后尝试消费
                    return ConsumeConcurrentlyStatus.RECONSUME_LATER;
                }
                //成功消费
                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
            }
        });
        //启动消费者
        consumer.start();
        System.out.printf("Consumer started.%n");
    }
}

3 消息消费时的权衡

1. 负载均衡模式：适用场景&注意事项

• 消费端集群化部署，每条消息只需要被处理一次。

• 由于消费进度在服务端维护，可靠性更高

• 集群消费模式下，每一条消息都只会被分发到一台机器上处理。如果需要被集群下的每一台机器都处理，请使用广播模式

• 集群消费模式下，不保证每一次失败重投的消息路由到同一台机器上，因此处理消息时不应该做任何确定性假设

2. 广播模式：适用场景&注意事项

• 每条消息都需要被相同逻辑的多台机器处理

• 消费进度在客户端维护，出现重复的概率稍大于集群模式

• 广播模式下，消息队列 RocketMQ 保证每条消息至少被每台客户端消费一次，但是并不会对消费失败的消息进行失败重投，因此业务方需要关注消费失败的情况。

• 广播模式下，客户端每一次重启都会从最新消息消费。客户端在被停止期间发送至服务端的消息将会被自动跳过，请谨慎选择