Yahoo pulsar -1 Producer send和Consumer receive 过程源码剖析

1、同步发送消息  

 MessageId send(byte[] message) throws PulsarClientException;

   private static final AtomicLongFieldUpdater<ProducerImpl> msgIdGeneratorUpdater = AtomicLongFieldUpdater
            .newUpdater(ProducerImpl.class, "msgIdGenerator");

msgIdGeneratorUpdater 用于生成请求ID,每次发送请求ID都不一样,用来区分消息。

 private final BlockingQueue<OpSendMsg> pendingMessages  = Queues.newArrayBlockingQueue(conf.getMaxPendingMessages());   阻塞的消息队列,用于缓存发送消息。

从Channel所在的线程  eventLoop()来执行发送请求。避免线程切换。

// Read the connection before validating if it's still connected, so that we avoid reading a null
                    // value
                    ClientCnx cnx = cnx();
                    if (isConnected()) {
                        // If we do have a connection, the message is sent immediately, otherwise we'll try again once a
                        // new
                        // connection is established
                        cmd.retain();
                        cnx.ctx().channel().eventLoop().execute(WriteInEventLoopCallback.create(this, cnx, op));

                   }

    private static final class WriteInEventLoopCallback implements Runnable {
        private ProducerImpl producer;
        private ClientCnx cnx;
        private OpSendMsg op;

        static WriteInEventLoopCallback create(ProducerImpl producer, ClientCnx cnx, OpSendMsg op) {
            WriteInEventLoopCallback c = RECYCLER.get();
            c.producer = producer;
            c.cnx = cnx;
            c.op = op;
            return c;
        }

        @Override
        public void run() {
            if (log.isDebugEnabled()) {
                log.debug("[{}] [{}] Sending message cnx {}, sequenceId {}", producer.topic, producer.producerName, cnx,
                        op.sequenceId);
            }

            try {
                cnx.ctx().writeAndFlush(op.cmd, cnx.ctx().voidPromise());
            } finally {
                recycle();
            }
        }

        private void recycle() {
            producer = null;
            cnx = null;
            op = null;
            RECYCLER.recycle(this, recyclerHandle);
        }

        private final Handle recyclerHandle;

        private WriteInEventLoopCallback(Handle recyclerHandle) {
            this.recyclerHandle = recyclerHandle;
        }

        private static final Recycler<WriteInEventLoopCallback> RECYCLER = new Recycler<WriteInEventLoopCallback>() {
            @Override
            protected WriteInEventLoopCallback newObject(Handle handle) {
                return new WriteInEventLoopCallback(handle);
            }
        };
    }

发送消息后接受响应消息：

 @Override
    protected void handleSendReceipt(CommandSendReceipt sendReceipt) {
        checkArgument(state == State.Ready);

        long producerId = sendReceipt.getProducerId();
        long sequenceId = sendReceipt.getSequenceId();
        long ledgerId = -1;
        long entryId = -1;
        if (sendReceipt.hasMessageId()) {
            ledgerId = sendReceipt.getMessageId().getLedgerId();
            entryId = sendReceipt.getMessageId().getEntryId();
        }

        if (log.isDebugEnabled()) {
            log.debug("{} Got receipt for producer: {} -- msg: {} -- id: {}:{}", ctx.channel(), producerId, sequenceId,
                    ledgerId, entryId);
        }
        //CommandSendReceipt 把producerId 和sequenceId 返回,用来查询原来的消息
        producers.get(producerId).ackReceived(this, sequenceId, ledgerId, entryId);
    }

    void ackReceived(ClientCnx cnx, long sequenceId, long ledgerId, long entryId) {
        OpSendMsg op = null;
        boolean callback = false;
        synchronized (this) {
            op = pendingMessages.peek();  //获取队首消息
            if (op == null) {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] [{}] Got ack for timed out msg {}", topic, producerName, sequenceId);
                }
                return;
            }
            //根据队首消息中的seqId与返回消息的seqId做对比,并进行处理。比如:关闭连接或者忽略响应等
            long expectedSequenceId = op.sequenceId;
            if (sequenceId > expectedSequenceId) {
                log.warn("[{}] [{}] Got ack for msg. expecting: {} - got: {} - queue-size: {}", topic, producerName,
                        expectedSequenceId, sequenceId, pendingMessages.size());
                // Force connection closing so that messages can be retransmitted in a new connection
                cnx.channel().close();
            } else if (sequenceId < expectedSequenceId) {
                // Ignoring the ack since it's referring to a message that has already timed out.
                if (log.isDebugEnabled()) {
                    log.debug("[{}] [{}] Got ack for timed out msg {} last-seq: {}", topic, producerName, sequenceId,
                            expectedSequenceId);
                }
            } else {//如果SeqID序列化正确,则移出队首消息并对改消息继续处理
                // Message was persisted correctly
                if (log.isDebugEnabled()) {
                    log.debug("[{}] [{}] Received ack for msg {} ", topic, producerName, sequenceId);
                }
                pendingMessages.remove();
                semaphore.release(op.numMessagesInBatch);
                callback = true;
                pendingCallbacks.add(op);
            }
        }
        if (callback) {
            op = pendingCallbacks.poll();
            if (op != null) {
                op.setMessageId(ledgerId, entryId, partitionIndex);
                try {
                    // Need to protect ourselves from any exception being thrown in the future handler from the
                    // application
                    op.callback.sendComplete(null);//通知发送消息完成
                } catch (Throwable t) {
                    log.warn("[{}] [{}] Got exception while completing the callback for msg {}:", topic, producerName,
                            sequenceId, t);
                }
                ReferenceCountUtil.safeRelease(op.cmd);
                op.recycle();
            }
        }
    }

Consumer   recieve  消息过程：

    final BlockingQueue<Message> incomingMessages;     阻塞消息队列, consumer  不断轮训。

 if (maxReceiverQueueSize<= 1) {
            this.incomingMessages = Queues.newArrayBlockingQueue(1);
        } else {
            this.incomingMessages = new GrowableArrayBlockingQueue<>();
        }

//Consumer使用 while(true) 来轮询消息,等消息到来后进行处理。

 @Override
    protected Message internalReceive() throws PulsarClientException {
        if (conf.getReceiverQueueSize() == 0) {
            checkArgument(zeroQueueLock != null, "Receiver queue size can't be modified");
            zeroQueueLock.writeLock().lock();
            try {
                return fetchSingleMessageFromBroker();
            } finally {
                zeroQueueLock.writeLock().unlock();
            }
        }
        Message message;
        try {
            message = incomingMessages.take();
            messageProcessed(message);
            return message;
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            stats.incrementNumReceiveFailed();
            throw new PulsarClientException(e);
        }
    }

消息被放到unAckedMessageTracker 等待追踪。

    /**
     * Record the event that one message has been processed by the application.
     *
     * Periodically, it sends a Flow command to notify the broker that it can push more messages
     */
    private synchronized void messageProcessed(Message msg) {
        ClientCnx currentCnx = cnx();
        ClientCnx msgCnx = ((MessageImpl) msg).getCnx();

        if (msgCnx != currentCnx) {
            // The processed message did belong to the old queue that was cleared after reconnection.
            return;
        }

        // reset timer for messages that are received by the client
        MessageIdImpl id = (MessageIdImpl) msg.getMessageId();
        if (id instanceof BatchMessageIdImpl) {
            id = new MessageIdImpl(id.getLedgerId(), id.getEntryId(), getPartitionIndex());
        }
        unAckedMessageTracker.add(id);
        increaseAvailablePermits(currentCnx);
        stats.updateNumMsgsReceived(msg);
    }

    void messageReceived(MessageIdData messageId, ByteBuf headersAndPayload, ClientCnx cnx) {
        if (log.isDebugEnabled()) {
            log.debug("[{}][{}] Received message: {}/{}", topic, subscription, messageId.getLedgerId(),
                    messageId.getEntryId());
        }

        MessageMetadata msgMetadata = null;
        ByteBuf payload = headersAndPayload;

        if (!verifyChecksum(headersAndPayload, messageId)) {
            // discard message with checksum error
            discardCorruptedMessage(messageId, cnx, ValidationError.ChecksumMismatch);
            return;
        }

        try {
            msgMetadata = Commands.parseMessageMetadata(payload);
        } catch (Throwable t) {
            discardCorruptedMessage(messageId, cnx, ValidationError.ChecksumMismatch);
            return;
        }

        ByteBuf uncompressedPayload = uncompressPayloadIfNeeded(messageId, msgMetadata, payload, cnx);
        if (uncompressedPayload == null) {
            // Message was discarded on decompression error
            return;
        }

        final int numMessages = msgMetadata.getNumMessagesInBatch();

        if (numMessages == 1 && !msgMetadata.hasNumMessagesInBatch()) {
            final MessageImpl message = new MessageImpl(messageId, msgMetadata, uncompressedPayload,
                getPartitionIndex(), cnx);
            uncompressedPayload.release();
            msgMetadata.recycle();

            try {
                lock.readLock().lock();
                // Enqueue the message so that it can be retrieved when application calls receive()
                // if the conf.getReceiverQueueSize() is 0 then discard message if no one is waiting for it.
                // if asyncReceive is waiting then notify callback without adding to incomingMessages queue
                unAckedMessageTracker.add((MessageIdImpl) message.getMessageId());
                boolean asyncReceivedWaiting = !pendingReceives.isEmpty();
                if ((conf.getReceiverQueueSize() != 0 || waitingOnReceiveForZeroQueueSize) && !asyncReceivedWaiting) {
                    incomingMessages.add(message);  //接受消息后,投放到incomingMessages队列中。等待被轮训
                }
                if (asyncReceivedWaiting) {
                    notifyPendingReceivedCallback(message, null);
                }
            } finally {
                lock.readLock().unlock();
            }
        } else {
            if (conf.getReceiverQueueSize() == 0) {
                log.warn(
                    "Closing consumer [{}]-[{}] due to unsupported received batch-message with zero receiver queue size",
                    subscription, consumerName);
                // close connection
                closeAsync().handle((ok, e) -> {
                    // notify callback with failure result
                    notifyPendingReceivedCallback(null,
                        new PulsarClientException.InvalidMessageException(
                            format("Unsupported Batch message with 0 size receiver queue for [%s]-[%s] ",
                                subscription, consumerName)));
                    return null;
                });
            } else {
                // handle batch message enqueuing; uncompressed payload has all messages in batch
                receiveIndividualMessagesFromBatch(msgMetadata, uncompressedPayload, messageId, cnx);
            }
            uncompressedPayload.release();
            msgMetadata.recycle();
        }
        //当接受到消息时,消息被按照顺序添加到线程池中。线程池为单线程。所以,所有消息都被顺序处理
        if (listener != null) {
            // Trigger the notification on the message listener in a separate thread to avoid blocking the networking
            // thread while the message processing happens
            listenerExecutor.execute(() -> {
                for (int i = 0; i < numMessages; i++) {
                    try {
                        Message msg = internalReceive(0, TimeUnit.MILLISECONDS);
                        // complete the callback-loop in case queue is cleared up
                        if (msg == null) {
                            if (log.isDebugEnabled()) {
                                log.debug("[{}] [{}] Message has been cleared from the queue", topic, subscription);
                            }
                            break;
                        }
                        try {
                            if (log.isDebugEnabled()) {
                                log.debug("[{}][{}] Calling message listener for message {}", topic, subscription, msg.getMessageId());
                            }
                            listener.received(ConsumerImpl.this, msg);
                        } catch (Throwable t) {
                            log.error("[{}][{}] Message listener error in processing message: {}", topic, subscription,
                                    msg.getMessageId(), t);
                        }

                    } catch (PulsarClientException e) {
                        log.warn("[{}] [{}] Failed to dequeue the message for listener", topic, subscription, e);
                        return;
                    }
                }
            });
        }
    }

    protected final ExecutorService listenerExecutor;    业务处理线程池。

  // gets the next single threaded executor from the list of executors
            ExecutorService listenerThread = externalExecutorProvider.getExecutor();

//业务线程池提供者
public class ExecutorProvider {
    private final int numThreads;
    private final List<ExecutorService> executors;
    private final AtomicInteger currentThread = new AtomicInteger(0);

    public ExecutorProvider(int numThreads, String threadNamePrefix) {
        checkArgument(numThreads > 0);
        this.numThreads = numThreads;
        checkNotNull(threadNamePrefix);
        executors = Lists.newArrayListWithCapacity(numThreads);
        for (int i = 0; i < numThreads; i++) {
        	//这里都是单线程哦
            executors.add(Executors.newSingleThreadExecutor(new DefaultThreadFactory(threadNamePrefix)));
        }
    }

    public ExecutorService getExecutor() {
        return executors.get((currentThread.getAndIncrement() & Integer.MAX_VALUE) % numThreads);
    }

    public void shutdownNow() {
        executors.forEach(executor -> executor.shutdownNow());
    }
}