spark task 任务状态管理

最新推荐文章于 2025-07-07 22:55:56 发布

gaoshui87

最新推荐文章于 2025-07-07 22:55:56 发布

阅读量3.8k

点赞数 1

CC 4.0 BY-SA版权

分类专栏： spark 文章标签： spark

本文链接：https://blog.youkuaiyun.com/gaoshui87/article/details/78253510

本文详细阐述了Spark Task任务状态管理，包括任务完成时的状态更新、结果的序列化与返回、大数据量时的Block拉取过程，以及如何从BlockManagerMasterEndpoint获取Block数据。总结了Spark Task状态更新的完整流程。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

spark task 任务状态管理

spark task 的任务状态经常进行更新，当任务完成后，这个任务是怎么取得结果的呢，看下面的代码流程

def statusUpdate(tid: Long, state: TaskState, serializedData: ByteBuffer) {
var failedExecutor: Option[String] = None
// 进行状态的更新 了
synchronized {
  try {
    if (state == TaskState.LOST && taskIdToExecutorId.contains(tid)) {
      // We lost this entire executor, so remember that it's gone
      val execId = taskIdToExecutorId(tid)

      if (executorIdToTaskCount.contains(execId)) {
        // 删除该进程相关
        removeExecutor(execId,
          SlaveLost(s"Task $tid was lost, so marking the executor as lost as well."))
        failedExecutor = Some(execId)
      }
    }
    taskIdToTaskSetManager.get(tid) match {
      case Some(taskSet) =>
        if (TaskState.isFinished(state)) {
          taskIdToTaskSetManager.remove(tid)
          taskIdToExecutorId.remove(tid).foreach { execId =>
            if (executorIdToTaskCount.contains(execId)) {
              executorIdToTaskCount(execId) -= 1
            }
          }
        }
        if (state == TaskState.FINISHED) {
          taskSet.removeRunningTask(tid)
          // 这个任务成功了
          taskResultGetter.enqueueSuccessfulTask(taskSet, tid, serializedData)
        } else if (Set(TaskState.FAILED, TaskState.KILLED, TaskState.LOST).contains(state)) {
          taskSet.removeRunningTask(tid)
          taskResultGetter.enqueueFailedTask(taskSet, tid, state, serializedData)
        }

可以看到上面的任务的状态的更新，当LOST状态时，就直接删除executor进程信息，当任务完成时，流程如下

def enqueueSuccessfulTask(
taskSetManager: TaskSetManager, tid: Long, serializedData: ByteBuffer) {
// 这个任务已经完成了
getTaskResultExecutor.execute(new Runnable {
  override def run(): Unit = Utils.logUncaughtExceptions {
    try {
      // 结果数据反序列化
      val (result, size) = serializer.get().deserialize[TaskResult[_]](serializedData) match {
        case directResult: DirectTaskResult[_] =>
          if (!taskSetManager.canFetchMoreResults(serializedData.limit())) {
            // 判断当前是否数据量比较大
            return
          }
          directResult.value()
          (directResult, serializedData.limit())
        case IndirectTaskResult(blockId, size) =>
          if (!taskSetManager.canFetchMoreResults(size)) {
            // dropped by executor if size is larger than maxResultSize
            sparkEnv.blockManager.master.removeBlock(blockId)
            return
          }
          logDebug("Fetching indirect task result for TID %s".format(tid))
          // 去拉取数据了
          scheduler.handleTaskGettingResult(taskSetManager, tid)
          // 拉取远程的block数据了
          val serializedTaskResult = sparkEnv.blockManager.getRemoteBytes(blockId)
          if (!serializedTaskResult.isDefined) {
            /* We won't be able to get the task result if the machine that ran the task failed
             * between when the task ended and when we tried to fetch the result, or if the
             * block manager had to flush the result. */
            // 拉取失败的情况下
            scheduler.handleFailedTask(