Hadoop源码分析25 JobInProgress 主要容器

1. JobInProgress中的容器

1.1maps、reduces、cleanup、setup

 TaskInProgress maps[]= new TaskInProgress[0];

 

 TaskInProgress reduces[]= new TaskInProgress[0];

 

 TaskInProgress cleanup[]= new TaskInProgress[0];

 

 TaskInProgress setup[]= new TaskInProgress[0];

 

在方法initTasks()中被初始化，

   maps= new TaskInProgress[numMapTasks];

   for(int i=0; i＜ numMapTasks;++i) {

     maps[i]= new TaskInProgress(jobId,jobFile,

                                  splits[i],

                                  jobtracker,conf,this,i, numSlotsPerMap);

   }

   this.reduces=new TaskInProgress[numReduceTasks];

   for(int i = 0;i ＜numReduceTasks;i++) {

     reduces[i]=  new TaskInProgress(jobId,jobFile,

                                  numMapTasks,i,

                                  jobtracker,conf,this,numSlotsPerReduce);

     nonRunningReduces.add(reduces[i]);

   }

   // createcleanup two cleanup tips, one map and one reduce.

   cleanup= new TaskInProgress[2];

 

   // cleanupmap tip. This map doesn't use any splits. Just assign anempty

   //split.

   TaskSplitMetaInfo emptySplit = JobSplit.EMPTY_TASK_SPLIT;

   cleanup[0]= new TaskInProgress(jobId,jobFile,emptySplit,

           jobtracker,conf,this,numMapTasks,1);

   cleanup[0].setJobCleanupTask();

 

   // cleanupreduce tip.

   cleanup[1]= newTaskInProgress(jobId,jobFile,numMapTasks,

                      numReduceTasks,jobtracker,conf,this,1);

   cleanup[1].setJobCleanupTask();

 

   // create twosetup tips, one map and one reduce.

   setup=new TaskInProgress[2];

 

   // setup maptip. This map doesn't use any split. Just assign anempty

   //split.

   setup[0]= new TaskInProgress(jobId,jobFile,emptySplit,

           jobtracker,conf,this,numMapTasks+ 1,1);

   setup[0].setJobSetupTask();

 

   // setupreduce tip.

   setup[1]= new TaskInProgress(jobId,jobFile,numMapTasks,

                      numReduceTasks+ 1,jobtracker,conf,this,1);

   setup[1].setJobSetupTask();

 

1.2nonRunningMapCache

//NetworkTopology Node to the set of TIPs

 Map＜Node,List＜TaskInProgress＞＞ nonRunningMapCache;

 

在方法 initTasks()中被初始化：

nonRunningMapCache = createCache(splits, maxLevel);

 

而createCache为

private Map＜Node,List＜TaskInProgress＞＞ createCache(

                                TaskSplitMetaInfo[] splits, int maxLevel)

                                throws UnknownHostException{

....

}

即根据 TaskSplitMetaInfo创建 Node 和 TaskInProgress的映射。

 

 

1.3runningMapCache

 

 // Map ofNetworkTopology Node to set of running TIPs

 Map＜Node,Set＜TaskInProgress＞＞ runningMapCache = newIdentityHashMap＜Node,Set＜TaskInProgress＞＞();

在scheduleMap(TaskInProgresstip)中被填充数据：

   for(Stringhost: splitLocations) {

     Node node = jobtracker.getNode(host);

 

     for(int j = 0;j ＜maxLevel;++j) {

       Set＜TaskInProgress＞hostMaps= runningMapCache.get(node);

       if(hostMaps== null){

         // create acache if needed

         hostMaps = new LinkedHashSet＜TaskInProgress＞();

         runningMapCache.put(node,hostMaps);

       }

       hostMaps.add(tip);

       node = node.getParent();

     }

   }

 

1.4nonLocalMaps

 // A list ofnon-local, non-running maps

 final List＜TaskInProgress＞ nonLocalMaps = new  LinkedList＜TaskInProgress＞();;

在createCache(TaskSplitMetaInfo[]splits, intmaxLevel)中被填充数据：

   for(int i = 0;i ＜splits.length;i++) {

     String[] splitLocations = splits[i].getLocations();

     if(splitLocations ==null||splitLocations.length== 0){

       nonLocalMaps.add(maps[i]);

       continue;

     }

   ......

  }

 

1.5failedMaps

 // Set offailed, non-running maps sorted by #failures

 final SortedSet＜TaskInProgress＞ failedMaps =  new TreeSet＜TaskInProgress＞(failComparator);;

在 failedTask(TaskInProgresstip, TaskAttemptID taskid,  TaskStatusstatus,  TaskTracker taskTracker, boolean wasRunning, boolean wasComplete, boolean wasAttemptRunning)中被填充

 

     if (!isComplete){

         retireMap(tip);

         failMap(tip);

       }

      

    

其中

private synchronized void failMap(TaskInProgress tip) {

    ......

   failedMaps.add(tip);

 }

 

 

1.6nonLocalRunningMaps

 

 // A set ofnon-local running maps

Set＜TaskInProgress＞ nonLocalRunningMaps =new LinkedHashSet＜TaskInProgress＞();

在scheduleMap(TaskInProgresstip) 中被填充

   String[] splitLocations = tip.getSplitLocations();

 

   // Add theTIP to the list of non-local running TIPs

   if(splitLocations ==null||splitLocations.length== 0){

     nonLocalRunningMaps.add(tip);

     return;

   }

 

1.6nonRunningReduces

 // A list ofnon-running reduce TIPs

Set＜TaskInProgress＞ nonRunningReduces= new TreeSet＜TaskInProgress＞(failComparator);

 

在 initTasks()被初始化

   this.reduces=new TaskInProgress[numReduceTasks];

   for(int i = 0;i ＜numReduceTasks;i++) {

     reduces[i]= new TaskInProgress(jobId,jobFile,

                                  numMapTasks,i,

                                  jobtracker,conf,this,numSlotsPerReduce);

     nonRunningReduces.add(reduces[i]);

   }

 

在 failedTask(TaskInProgresstip, TaskAttemptID taskid,  TaskStatusstatus,    TaskTracker taskTracker, boolean wasRunning, boolean wasComplete, boolean wasAttemptRunning)被填充

 

       

   

if(!isComplete){

         retireReduce(tip);

         failReduce(tip);

 }

 

 private synchronized void failReduce(TaskInProgress tip) {

   。。。。。。

   nonRunningReduces.remove(tip);

 }

 

 

1.7runningReduces

 

 // A set ofrunning reduce TIPs

 Set＜TaskInProgress＞ runningReduces = new LinkedHashSet＜TaskInProgress＞();

 

调用如下:

 protected synchronized void scheduleReduce(TaskInProgress tip) {

   。。。

   runningReduces.add(tip);

 }

 

 private synchronized void retireReduce(TaskInProgress tip) {

 ......

   runningReduces.remove(tip);

 }

 

1.8 mapCleanupTasks、reduceCleanupTasks

 // A list ofcleanup tasks for the map task attempts, to belaunched

 List＜TaskAttemptID＞ mapCleanupTasks=new LinkedList＜TaskAttemptID＞();

// A list ofcleanup tasks for the reduce task attempts, to belaunched

 List＜TaskAttemptID＞ reduceCleanupTasks =new  LinkedList＜TaskAttemptID＞();

 

 在updateTaskStatus(TaskInProgresstip, TaskStatus status) 中填充

 

  if(state ==TaskStatus.State.FAILED_UNCLEAN||

                state == TaskStatus.State.KILLED_UNCLEAN){

       tip.incompleteSubTask(taskid, this.status);

 

       if(tip.isMapTask()){

         mapCleanupTasks.add(taskid);

       } else{

         reduceCleanupTasks.add(taskid);

       }

       jobtracker.removeTaskEntry(taskid);

     }

 

 

 

1.9taskCompletionEvents

 List＜TaskCompletionEvent＞ taskCompletionEvents=

new ArrayList＜TaskCompletionEvent＞(numMapTasks+numReduceTasks+10);

 

在updateTaskStatus(TaskInProgress tip, TaskStatus status)中填充

 

taskEvent= new TaskCompletionEvent(taskCompletionEventTracker,

                                           taskid,

                                           tip.idWithinJob(),

                                           status.getIsMap()&&

                                           !tip.isJobCleanupTask() &&

                                           !tip.isJobSetupTask(),

                                           taskCompletionStatus,

                                           httpTaskLogLocation

                                          );

this.taskCompletionEvents.add(taskEvent);

taskCompletionEventTracker++;

 

1.10trackerToFailuresMap

 // Map oftrackerHostName -＞ no. of taskfailures

Map＜String,Integer＞trackerToFailuresMap= new TreeMap＜String,Integer＞();

 

在addTrackerTaskFailure中调用

synchronized void addTrackerTaskFailure(StringtrackerName TaskTracker taskTracker) {

   if(flakyTaskTrackers＜ (clusterSize*CLUSTER_BLACKLIST_PERCENT)){

     String trackerHostName =convertTrackerNameToHostName(trackerName);

 

     Integer trackerFailures = trackerToFailuresMap.get(trackerHostName);

     if(trackerFailures== null){

       trackerFailures = 0;

     }

     trackerToFailuresMap.put(trackerHostName,++trackerFailures);

   ......

1.11firstTaskLaunchTimes

 // First*task launch time

 final Map＜TaskType,Long＞firstTaskLaunchTimes=

     new EnumMap＜TaskType,Long＞(TaskType.class);

 

在setFirstTaskLaunchTime中调用

 void setFirstTaskLaunchTime(TaskInProgresstip) {

   TaskType key = tip.getFirstTaskType();

 

   synchronized(firstTaskLaunchTimes){

     // Could beoptimized to do only one lookup with a little more code

     if(!firstTaskLaunchTimes.containsKey(key)){

       firstTaskLaunchTimes.put(key,tip.getExecStartTime());

     }

   }

 }

 

1.12mapTaskIdToFetchFailuresMap

 // Map ofmapTaskId -＞ no. of fetchfailures

 private Map＜TaskAttemptID,Integer＞ mapTaskIdToFetchFailuresMap=

   new TreeMap＜TaskAttemptID,Integer＞();

 

在fetchFailureNotification(TaskInProgresstip, TaskAttemptID mapTaskId,  StringmapTrackerName, TaskAttemptID reduceTaskId, reduceTrackerName) 调用：

 

IntegerfetchFailures = mapTaskIdToFetchFailuresMap.get(mapTaskId);

fetchFailures= (fetchFailures == null)? 1 : (fetchFailures+1);

mapTaskIdToFetchFailuresMap.put(mapTaskId,fetchFailures);

1.13trackersReservedForMaps、trackersReservedForReduces

 private Map＜TaskTracker,FallowSlotInfo＞trackersReservedForMaps=

   new  HashMap＜TaskTracker,FallowSlotInfo＞();

 

 private Map＜TaskTracker,FallowSlotInfo＞trackersReservedForReduces=

   new HashMap＜TaskTracker,FallowSlotInfo＞();

在 reserveTaskTracker(TaskTrackertaskTracker, TaskType type, intnumSlots)中调用

Map＜TaskTracker,FallowSlotInfo＞map =(type ==TaskType.MAP) ?  trackersReservedForMaps : trackersReservedForReduces;

map.put(taskTracker,info);