kafka消费组管理模块(二)-元数据管理器GroupMetadataManager

组管理器是管理多个GroupMetadata

一.核心字段

其中最核心的就是groupMetadataCache ,底层就是一个ConcurrentHashMap,消费组名,对应GroupMetadata

private val groupMetadataCache = new Pool[String, GroupMetadata]

二.消费者组管理

核心方法就是对groupMetadataCache操作

消费者组元数据管理分为查询获取组信息、添加组、移除组和加载组信息

方法分析

1.持久化Group信息storeGroup

def storeGroup(group: GroupMetadata,
               groupAssignment: Map[String, Array[Byte]],
               responseCallback: Errors => Unit): Unit = {
  getMagic(partitionFor(group.groupId)) match {
    case Some(magicValue) =>

      //生成groupMetadata消息开始
      val timestampType = TimestampType.CREATE_TIME
      val timestamp = time.milliseconds()
      val key = GroupMetadataManager.groupMetadataKey(group.groupId)
      val value = GroupMetadataManager.groupMetadataValue(group, groupAssignment, interBrokerProtocolVersion)

      val records = {
        val buffer = ByteBuffer.allocate(AbstractRecords.estimateSizeInBytes(magicValue, compressionType,
          Seq(new SimpleRecord(timestamp, key, value)).asJava))
        val builder = MemoryRecords.builder(buffer, magicValue, compressionType, timestampType, 0L)
        builder.append(timestamp, key, value)
        builder.build()
      }
     //生成groupMetadata消息结束


      //找到要保存的__consumer_offsets的分区
      val groupMetadataPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, partitionFor(group.groupId))
      val groupMetadataRecords = Map(groupMetadataPartition -> records)
      val generationId = group.generationId

      //处理错误码
      def putCacheCallback(responseStatus: Map[TopicPartition, PartitionResponse]): Unit = {
       ........
      }
     //向__consumer_offsets写入groupMetadata record
      appendForGroup(group, groupMetadataRecords, putCacheCallback)

    case None =>
      responseCallback(Errors.NOT_COORDINATOR)
      None
  }
}

2.接管或新创建Group Manager的时候,需要从__consumer_offsets里取出需要的group metadata和offsets matadata信息:doLoadGroupsAndOffsets

private def doLoadGroupsAndOffsets(topicPartition: TopicPartition, onGroupLoaded: GroupMetadata => Unit): Unit = {
  def logEndOffset: Long = replicaManager.getLogEndOffset(topicPartition).getOrElse(-1L)

  replicaManager.getLog(topicPartition) match {
 
    case Some(log) =>
      val loadedOffsets = mutable.Map[GroupTopicPartition, CommitRecordMetadataAndOffset]()
      val pendingOffsets = mutable.Map[Long, mutable.Map[GroupTopicPartition, CommitRecordMetadataAndOffset]]()
      val loadedGroups = mutable.Map[String, GroupMetadata]()
      val removedGroups = mutable.Set[String]()

      // buffer may not be needed if records are read from memory
      var buffer = ByteBuffer.allocate(0)

      // loop breaks if leader changes at any time during the load, since logEndOffset is -1
      var currOffset = log.logStartOffset

      // loop breaks if no records have been read, since the end of the log has been reached
      var readAtLeastOneRecord = true

      while (currOffset < logEndOffset && readAtLeastOneRecord && !shuttingDown.get()) {
       //从log读取一些消息(__consumer_offsets里读消息)
        val fetchDataInfo = log.read(currOffset,
          maxLength = config.loadBufferSize,
          isolation = FetchLogEnd,
          minOneMessage = true)

        readAtLeastOneRecord = fetchDataInfo.records.sizeInBytes > 0

       //把消息转化为MemoryRecords
        val memRecords = fetchDataInfo.records match {
          case records: MemoryRecords => records
          case fileRecords: FileRecords =>
            val sizeInBytes = fileRecords.sizeInBytes
            val bytesNeeded = Math.max(config.loadBufferSize, sizeInBytes)

            // minOneMessage = true in the above log.read means that the buffer may need to be grown to ensure progress can be made
            if (buffer.capacity < bytesNeeded) {
              if (config.loadBufferSize < bytesNeeded)
                warn(s"Loaded offsets and group metadata from $topicPartition with buffer larger ($bytesNeeded bytes) than " +
                  s"configured offsets.load.buffer.size (${config.loadBufferSize} bytes)")

              buffer = ByteBuffer.allocate(bytesNeeded)
            } else {
              buffer.clear()
            }

            fileRecords.readInto(buffer, 0)
            MemoryRecords.readableRecords(buffer)
        }
       //解析MemoryRecords
        memRecords.batches.forEach { batch =>
          val isTxnOffsetCommit = batch.isTransactional
       //事务相关的控制消息的处理,暂时不解读
          if (batch.isControlBatch) {
            ......
          } else {
            var batchBaseOffset: Option[Long] = None
            for (record <- batch.asScala) {
              require(record.hasKey, "Group metadata/offset entry key should not be null")
              if (batchBaseOffset.isEmpty)
                batchBaseOffset = Some(record.offset)
           //判断消息的key,OffsetKey和groupMetadataKey两种
              GroupMetadataManager.readMessageKey(record.key) match {
                //处理offsets消息
                case offsetKey: OffsetKey =>
                  // load offset
                  val groupTopicPartition = offsetKey.key
                  //没有value说明offset被删除了
                  if (!record.hasValue) {
                     loadedOffsets.remove(groupTopicPartition)
                  } else {
                   //有value取出offset保存起来
                    val offsetAndMetadata = GroupMetadataManager.readOffsetMessageValue(record.value)
                   //CommitRecordMetadataAndOffset保存了__cosumer_offsets 的offset和group消费的topic的offset的对应关系
                    loadedOffsets.put(groupTopicPartition, CommitRecordMetadataAndOffset(batchBaseOffset, offsetAndMetadata))
                  }
                //处理group metadata消息
                case groupMetadataKey: GroupMetadataKey =>
                  // load group metadata
                  val groupId = groupMetadataKey.key
                  val groupMetadata = GroupMetadataManager.readGroupMessageValue(groupId, record.value, time)
                //根据是否有value保存到loaded或者remove的map里
                  if (groupMetadata != null) {
                    removedGroups.remove(groupId)
                    loadedGroups.put(groupId, groupMetadata)
                  } else {
                    loadedGroups.remove(groupId)
                    removedGroups.add(groupId)
                  }

                case unknownKey =>
                  throw new IllegalStateException(s"Unexpected message key $unknownKey while loading offsets and group metadata")
              }
            }
          }
          currOffset = batch.nextOffset
        }
      }

      val (groupOffsets, emptyGroupOffsets) = loadedOffsets      // 解释:Map[GroupTopicPartition, CommitRecordMetadataAndOffset]
        .groupBy(_._1.group)       //解释:Map[group,Map[GroupTopicPartition, CommitRecordMetadataAndOffset]]
        .map { case (k, v) =>      //解释:(key=group,value=Map[GroupTopicPartition, CommitRecordMetadataAndOffset])
          k -> v.map { case (groupTopicPartition, offset) => (groupTopicPartition.topicPartition, offset) }
        }.partition { case (group, _) => loadedGroups.contains(group) }   //根据有offsets信息,有无group信息分组

      val pendingOffsetsByGroup = ....事务相关,不解读

      loadedGroups.values.foreach { group =>         //解释:Map[String, GroupMetadata]
        val offsets = groupOffsets.getOrElse(group.groupId, Map.empty[TopicPartition, CommitRecordMetadataAndOffset])
        val pendingOffsets = pendingGroupOffsets.getOrElse(group.groupId, Map.empty[Long, mutable.Map[TopicPartition, CommitRecordMetadataAndOffset]])
        loadGroup(group, offsets, pendingOffsets) //更新 offsets哪个map
        onGroupLoaded(group)//回调
      }

      // load groups which store offsets in kafka, but which have no active members and thus no group
      // metadata stored in the log
      (emptyGroupOffsets.keySet ++ pendingEmptyGroupOffsets.keySet).foreach { groupId =>
        val group = new GroupMetadata(groupId, Empty, time)
        val offsets = emptyGroupOffsets.getOrElse(groupId, Map.empty[TopicPartition, CommitRecordMetadataAndOffset])
        val pendingOffsets = pendingEmptyGroupOffsets.getOrElse(groupId, Map.empty[Long, mutable.Map[TopicPartition, CommitRecordMetadataAndOffset]])
        debug(s"Loaded group metadata $group with offsets $offsets and pending offsets $pendingOffsets")
        loadGroup(group, offsets, pendingOffsets)
        onGroupLoaded(group)
      }

      removedGroups.foreach { groupId =>
        // if the cache already contains a group which should be removed, raise an error. Note that it
        // is possible (however unlikely) for a consumer group to be removed, and then to be used only for
        // offset storage (i.e. by "simple" consumers)
        if (groupMetadataCache.contains(groupId) && !emptyGroupOffsets.contains(groupId))
          throw new IllegalStateException(s"Unexpected unload of active group $groupId while " +
            s"loading partition $topicPartition")
      }
  }
}

 

三.消费者组位移管理

1.保存位移storeOffsets

和stroeGroup类似 生成offsets的record,写入到对应的__consumer_offsets的partition里,更新offsets的map

2.查询位移getOffsets

def getOffsets(groupId: String, requireStable: Boolean, topicPartitionsOpt: Option[Seq[TopicPartition]]): Map[TopicPartition, PartitionData] = {
  val group = groupMetadataCache.get(groupId)
  if (group == null) {
    ...返回错误
  } else {
    group.inLock {
      if (group.is(Dead)) {
       ....返回错误
      } else {
        val topicPartitions = topicPartitionsOpt.getOrElse(group.allOffsets.keySet)

        topicPartitions.map { topicPartition =>
          if (requireStable && group.hasPendingOffsetCommitsForTopicPartition(topicPartition)) {
          ....返回错误
          } else {
            val partitionData = group.offset(topicPartition) match {
              case None =>
             ...返回错误
              case Some(offsetAndMetadata) =>
                new PartitionData(offsetAndMetadata.offset,
                  offsetAndMetadata.leaderEpoch, offsetAndMetadata.metadata, Errors.NONE)
            }
            topicPartition -> partitionData
          }
        }.toMap
      }
    }
  }
}