Kafka日志模块（二）：Log定义及初始化

在一个 log 目录下存在多个以“topic-partition”命名的分区目录，每个 topic 分区对应一个 Log 对象，用于管理名下的 LogSegment 对象集合，Log 类使用 SkipList 数据结构对 LogSegment 进行组织和管理。它的定义为

class Log(@volatile var dir: File, // dir 就是这个日志所在的文件夹路径，也就是主题分区的路径
          @volatile var config: LogConfig, // 配置信息
          @volatile var logStartOffset: Long, // log日志文件的当前最早位移
          @volatile var recoveryPoint: Long, // 恢复操作的起始 offset，即 HW 位置，之前的消息已经全部落盘
          scheduler: Scheduler, // 定时任务调度器
          brokerTopicStats: BrokerTopicStats,
          val time: Time,
          val maxProducerIdExpirationMs: Int,
          val producerIdExpirationCheckIntervalMs: Int,
          val topicPartition: TopicPartition,
          val producerStateManager: ProducerStateManager,
          logDirFailureChannel: LogDirFailureChannel) extends Logging with KafkaMetricsGroup {

    // nextOffsetMetadata封装了下一条待插入消息的位移值，基本上可以把这个属性和 Log End Offset 等同起来
    @volatile private var nextOffsetMetadata: LogOffsetMetadata = _ 
    // 取日志高水位值
    @volatile private var highWatermarkMetadata: LogOffsetMetadata = LogOffsetMetadata(logStartOffset)
    // 保存了分区日志下所有的日志段信息， Key 值是日志段的起始位移值，Value 则是日志段对象本身
    private val segments: ConcurrentNavigableMap[java.lang.Long, LogSegment] = new ConcurrentSkipListMap[java.lang.Long, LogSegment]
    // 判断出现 Failure 时是否执行日志截断操作（Truncation），之前靠高水位来判断的机制，可能会造成副本间数据不一致的情形。这里的 Leader Epoch Cache 是一个缓存类数据，里面保存了分区 Leader 的 Epoch 值与对应位移值的映射关系
    @volatile var leaderEpochCache: Option[LeaderEpochFileCache] = None
}

Log 类的初始化逻辑如下：

  locally {
    val startMs = time.milliseconds

    // create the log directory if it doesn't exist
    // 创建分区log路径
    Files.createDirectories(dir.toPath)
    // 初始化leader epoch cache，下面会作详细解释
    initializeLeaderEpochCache()
    // 加载所有日志段对象，返回下一条待出入的消息offset，下面会作详细解释
    val nextOffset = loadSegments()

    /* Calculate the offset of the next message */
    // 更新nextOffsetMetadata
    nextOffsetMetadata = LogOffsetMetadata(nextOffset, activeSegment.baseOffset, activeSegment.size)

    leaderEpochCache.foreach(_.truncateFromEnd(nextOffsetMetadata.messageOffset))
    // 更新Log Start Offset
    updateLogStartOffset(math.max(logStartOffset, segments.firstEntry.getValue.baseOffset))

    // The earliest leader epoch may not be flushed during a hard failure. Recover it here.
    leaderEpochCache.foreach(_.truncateFromStart(logStartOffset))

    // Any segment loading or recovery code must not use producerStateManager, so that we can build the full state here
    // from scratch.
    if (!producerStateManager.isEmpty)
      throw new IllegalStateException("Producer state must be empty during log initialization")
    loadProducerState(logEndOffset, reloadFromCleanShutdown = hasCleanShutdownFile)

    info(s"Completed load of log with ${segments.size} segments, log start offset $logStartOffset and " +
      s"log end offset $logEndOffset in ${time.milliseconds() - startMs} ms")
  }

下面介绍一下initializeLeaderEpochCache初始化leader epoch cache

  private def initializeLeaderEpochCache(): Unit = lock synchronized {
    val leaderEpochFile = LeaderEpochCheckpointFile.newFile(dir)

    // 生成leader epoch cache对象
    def newLeaderEpochFileCache(): LeaderEpochFileCache = {
      val checkpointFile = new LeaderEpochCheckpointFile(leaderEpochFile, logDirFailureChannel)
      new LeaderEpochFileCache(topicPartition, logEndOffset _, checkpointFile)
    }