The originalTrackingUrl changes in the process of application run

最新推荐文章于 2020-12-13 20:09:21 发布

原创最新推荐文章于 2020-12-13 20:09:21 发布 · 446 阅读

0 ·

CC 4.0 BY-SA版权

hadoop-yarn 专栏收录该内容

28 篇文章

订阅专栏

本文详细解析了 YARN 中 ApplicationMaster (AM) 的注册与注销过程，包括设置 Tracking URL、与 ResourceManager 交互的具体步骤等关键技术细节。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

Before am register, the originalTrackingUrl=”N/A”;
After am register, the originalTrackingUrl is the web url of application master, like ‘http://localhost:42298’

private static final class AMRegisteredTransition extends BaseTransition {
        @Override
        public void transition(RMAppAttemptImpl appAttempt,
                               RMAppAttemptEvent event) {
            long delay = System.currentTimeMillis() - appAttempt.launchAMEndTime;
            ClusterMetrics.getMetrics().addAMRegisterDelay(delay);
            RMAppAttemptRegistrationEvent registrationEvent
                    = (RMAppAttemptRegistrationEvent) event;
            appAttempt.host = registrationEvent.getHost();
            appAttempt.rpcPort = registrationEvent.getRpcport();
            appAttempt.originalTrackingUrl =
                    sanitizeTrackingUrl(registrationEvent.getTrackingurl());

updateInfoOnAMUnregister

At updateInfoOnAMUnregister, originalTrackingUrl=’The mapreduce history address’

 private void updateInfoOnAMUnregister(RMAppAttemptEvent event) {
        progress = 1.0f;
        RMAppAttemptUnregistrationEvent unregisterEvent =
                (RMAppAttemptUnregistrationEvent) event;
        diagnostics.append(unregisterEvent.getDiagnosticMsg());
        originalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl());
        finalStatus = unregisterEvent.getFinalApplicationStatus();
    }

ApplicationMasterService.finishApplicationMaster

set tracking url withrequest.getTrackingUrl()

 @Override
  public FinishApplicationMasterResponse finishApplicationMaster(
      FinishApplicationMasterRequest request) throws YarnException,
      IOException {
rmContext.getDispatcher().getEventHandler().handle(
          new RMAppAttemptUnregistrationEvent(applicationAttemptId, request
              .getTrackingUrl(), request.getFinalApplicationStatus(), request
              .getDiagnostics()));
}

MapReduce tracking url

MRAppMaster has LocalContainerAllocator, which extends class RMCommunicator.

 protected ContainerAllocator createContainerAllocator(
      final ClientService clientService, final AppContext context) {
    return new ContainerAllocatorRouter(clientService, context);
  }

ContainerAllocatorRouter

@Override
    protected void serviceStart() throws Exception {

        this.containerAllocator = new RMContainerAllocator(
            this.clientService, this.context);

    }

public class RMContainerAllocator extends RMContainerRequestor

RMContainerRequestor.serviceStart register resourcemanager

  @Override
  protected void serviceStart() throws Exception {
    scheduler= createSchedulerProxy();
    JobID id = TypeConverter.fromYarn(this.applicationId);
    JobId jobId = TypeConverter.toYarn(id);
    job = context.getJob(jobId);
    register();
    startAllocatorThread();
    super.serviceStart();
  }

register set tracking url to serviceAddr.httpAddress

 protected void register() {
    //Register
    InetSocketAddress serviceAddr = null;
    if (clientService != null ) {
      serviceAddr = clientService.getBindAddress();
    }
    try {
      RegisterApplicationMasterRequest request =
        recordFactory.newRecordInstance(RegisterApplicationMasterRequest.class);
      if (serviceAddr != null) {
        request.setHost(serviceAddr.getHostName());
        request.setRpcPort(serviceAddr.getPort());
        request.setTrackingUrl(MRWebAppUtil
            .getAMWebappScheme(getConfig())
            + serviceAddr.getHostName() + ":" + clientService.getHttpPort());
      }
      RegisterApplicationMasterResponse response =
        scheduler.registerApplicationMaster(request);

RMContainerRequestor.unregister

 protected void unregister() {
    try {
      doUnregistration();
    } catch(Exception are) {
      LOG.error("Exception while unregistering ", are);
      // if unregistration failed, isLastAMRetry needs to be recalculated
      // to see whether AM really has the chance to retry
      RunningAppContext raContext = (RunningAppContext) context;
      raContext.resetIsLastAMRetry();
    }
  }

doUnregistration set tracking url to historyUrl

@VisibleForTesting
  protected void doUnregistration()
      throws YarnException, IOException, InterruptedException {

    String historyUrl =
        MRWebAppUtil.getApplicationWebURLOnJHSWithScheme(getConfig(),
            context.getApplicationID());

    FinishApplicationMasterRequest request =
        FinishApplicationMasterRequest.newInstance(finishState,
          sb.toString(), historyUrl);

        FinishApplicationMasterResponse response =
            scheduler.finishApplicationMaster(request);

  }

MRWebAppUtil.getApplicationWebURLOnJHSWithScheme

public static String getApplicationWebURLOnJHSWithScheme(Configuration conf,
      ApplicationId appId) throws UnknownHostException {
    return getJHSWebappScheme()
        + getApplicationWebURLOnJHSWithoutScheme(conf, appId);
  }

MRWebAppUtil.getApplicationWebURLOnJHSWithoutScheme

public static String getApplicationWebURLOnJHSWithoutScheme(Configuration conf,
      ApplicationId appId)
      throws UnknownHostException {
    //construct the history url for job
    String addr = getJHSWebappURLWithoutScheme(conf);
    Iterator<String> it = ADDR_SPLITTER.split(addr).iterator();
    it.next(); // ignore the bind host
    String port = it.next();
    // Use hs address to figure out the host for webapp
    addr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS,
        JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS);
    String host = ADDR_SPLITTER.split(addr).iterator().next();
    String hsAddress = JOINER.join(host, ":", port);
    InetSocketAddress address = NetUtils.createSocketAddr(
      hsAddress, getDefaultJHSWebappPort(),
      getDefaultJHSWebappURLWithoutScheme());
    StringBuffer sb = new StringBuffer();
    if (address.getAddress().isAnyLocalAddress() || 
        address.getAddress().isLoopbackAddress()) {
      sb.append(InetAddress.getLocalHost().getCanonicalHostName());
    } else {
      sb.append(address.getHostName());
    }
    sb.append(":").append(address.getPort());
    sb.append("/jobhistory/job/");
    JobID jobId = TypeConverter.fromYarn(appId);
    sb.append(jobId.toString());
    return sb.toString();
  }