hbase 源代码分析 （12） Master和RegionService 启动过程

第9篇hbase shell启动过程时，最后说到HMaster和HRegionService启动入口。而本地模式只需要启动 HMaster就行，内部会启动zk和 HRegionService

主要流程。

1）HMaster 是HregionService的实现类。所以启动HMaster会先启动 HregionService，

2） HregionService里面启动了很多进程。主要有协处理器，zk，rpc，锁，状态追踪器，wal继承或者类。

3）HMaster会向zk里注册信息。写入当前ServerName，方便Client找到。

4） HMaster 会加载Meta信息到内存。然后等待RegionService的报告，如果没有收到报告，会去zk 询问已经启动但是没有报告的RegionService，

5） HMaster会启动均衡

6）会启动一下健康检测的机制。

HMaster.java

/**
 * @see org.apache.hadoop.hbase.master.HMasterCommandLine
 */
public static void main(String [] args) {
  VersionInfo.logVersion();
  new HMasterCommandLine(HMaster.class).doMain(args);
}

这里会调用HmasterCommandLine.java的run线程

   
 public int run(String args[]) throws Exception {
 
//此处有省略，
//主要是参数设置，
    String command = remainingArgs.get(0);
    if ("start".equals(command)) {
      return startMaster();
    } else if ("stop".equals(command)) {
      return stopMaster();
    } else if ("clear".equals(command)) {
      return (ZNodeClearer.clear(getConf()) ? 0 : 1);
    } else {
      usage("Invalid command: " + command);
      return 1;
    }
  }

startMaster

   
  private int startMaster() {
    Configuration conf = getConf();
    try {
    //本地模式。master和regionService 公用一个jVM
      if (LocalHBaseCluster.isLocal(conf)) {

      .... //省略配置zk信息

        LocalHBaseCluster cluster = new LocalHBaseCluster(conf, mastersCount, regionServersCount,
          LocalHMaster.class, HRegionServer.class);
        ((LocalHMaster)cluster.getMaster(0)).setZKCluster(zooKeeperCluster);
        cluster.startup();
        waitOnMasterThreads(cluster);
      } else {
//    集群模式
        logProcessInfo(getConf());
        CoordinatedStateManager csm =
          CoordinatedStateManagerFactory.getCoordinatedStateManager(conf);
        HMaster master = HMaster.constructMaster(masterClass, conf, csm);

        master.start();
        master.join();
        if(master.isAborted())
          throw new RuntimeException("HMaster Aborted");
      }
    }
  }

1）本地模式

本地模式主要是将new localMaster，new regionService .这里里面主要是启动协处理管理器 ZkCoordinatedStateManager ， MasterThread， RegionServerThread 启动了大量协处理。

@Override
public void initialize(Server server) {
  this.server = server;
  this.watcher = server.getZooKeeper();
  splitLogWorkerCoordination = new ZkSplitLogWorkerCoordination(this, watcher);
  splitLogManagerCoordination = new ZKSplitLogManagerCoordination(this, watcher);
  splitTransactionCoordination = new ZKSplitTransactionCoordination(this, watcher);
  closeRegionCoordination = new ZkCloseRegionCoordination(this, watcher);
  openRegionCoordination = new ZkOpenRegionCoordination(this, watcher);
  regionMergeCoordination = new ZkRegionMergeCoordination(this, watcher);
}

因为Hmaster继承RegionServicer，所以Hmaster和regionService都需要启动这些。

   

  public HRegionServer(Configuration conf, CoordinatedStateManager csm)
      throws IOException, InterruptedException {
    super("RegionServer");  // thread name
//此处有省略，
//主要配置信息
    rpcControllerFactory = RpcControllerFactory.instantiate(this.conf);
    rpcRetryingCallerFactory = RpcRetryingCallerFactory.instantiate(this.conf);
    // 设置了些jaas，keytab等文件
    ZKUtil.loginClient(this.conf, HConstants.ZK_CLIENT_KEYTAB_FILE,
      HConstants.ZK_CLIENT_KERBEROS_PRINCIPAL, hostName);
    // 登录
    login(userProvider, hostName);
   
//启动文件系统
    this.fs = new HFileSystem(this.conf, useHBaseChecksum)；
//加载meta信息
this.tableDescriptors = new FSTableDescriptors(
      this.conf, this.fs, this.rootDir, !canUpdateTableDescriptor(), false);
//执行服务器
    service = new ExecutorService(getServerName().toShortString());
//hbase-site.xml中读取span
    spanReceiverHost = SpanReceiverHost.getInstance(getConfiguration());
    // Some unit tests don't need a cluster, so no zookeeper at all
    if (!conf.getBoolean("hbase.testing.nocluster", false)) {
      // Open connection to zookeeper and set primary watcher
    //集群采用用，zk
      zooKeeper = new ZooKeeperWatcher(conf, getProcessName() + ":" +
        rpcServices.isa.getPort(), this, canCreateBaseZNode());
//协处理器管理器    
  this.csm = (BaseCoordinatedStateManager) csm;
      this.csm.initialize(this);
      this.csm.start();
//table 锁
      tableLockManager = TableLockManager.createTableLockManager(
        conf, zooKeeper, serverName);
//集群需要最终master的地址，
      masterAddressTracker = new MasterAddressTracker(getZooKeeper(), this);
      masterAddressTracker.start();
//集群需要追踪集群状态。
      clusterStatusTracker = new ClusterStatusTracker(zooKeeper, this);
      clusterStatusTracker.start();
    }
//rpc服务端启动，接收client的请求
    rpcServices.start();
//启动web页面
    putUpWebUI();
//WAL
    this.walRoller = new LogRoller(this, this);
//一些杂事处理。
    this.choreService = new ChoreService(getServerName().toString(), true);
    this.flushThroughputController = FlushThroughputControllerFactory.create(this, conf);
//这个没看懂，
    if (!SystemUtils.IS_OS_WINDOWS) {
      Signal.handle(new Signal("HUP"), new SignalHandler() {
        @Override
        public void handle(Signal signal) {
          getConfiguration().reloadConfiguration();
          configurationManager.notifyAllObservers(getConfiguration());
        }
      });
    }
//定时删除不用的压缩文件。
    this.compactedFileDischarger =
        new CompactedHFilesDischarger(cleanerInterval, (Stoppable)this, (RegionServerServices)this);
    choreService.scheduleChore(compactedFileDischarger);
  }

HMaster 一些特殊的启动

   
 public HMaster(final Configuration conf, CoordinatedStateManager csm)
      throws IOException, KeeperException, InterruptedException {
    super(conf, csm);
 
   
//集群状态发表。 比如当regionService 死了，要立即告知client ，不要用client等待socket回应了。    
clusterStatusPublisherChore = new ClusterStatusPublisher(this, conf, publisherClass);
     getChoreService().scheduleChore(clusterStatusPublisherChore);
     
    // Some unit tests don't need a cluster, so no zookeeper at all
    if (!conf.getBoolean("hbase.testing.nocluster", false)) {
//注入Master到zk，
      activeMasterManager = new ActiveMasterManager(zooKeeper, this.serverName, this);
      int infoPort = putUpJettyServer();
//启动Master 这里主要是，在zk里建立一个节点，然后将master的主机信息写入zk
      startActiveMasterManager(infoPort);
    } else {
      activeMasterManager = null;
    }
  }

这个startActiveMasterManager（InfoPort）很有研究价值。最终会调用下面的方法。在 ActiveMasterManager里




 //主要是当这个master启动起来后，去尝试王zk写入节点信息。如果写成功了，这这个master是有效的master节点，然后从备份中删除自己，如果写入不成功，这说明有有效的主节点了。然后去检查一下有效的节点是否和当前情况一样。如果一样的，说明已经启动过 ，删除zk信息，然后删除磁盘信息。

   

boolean blockUntilBecomingActiveMaster(
      int checkInterval, MonitoredTask startupStatus) {
    String backupZNode = ZKUtil.joinZNode(
      this.watcher.backupMasterAddressesZNode, this.sn.toString());
    while (!(master.isAborted() || master.isStopped())) {
      startupStatus.setStatus("Trying to register in ZK as active master");
      try {
        if (MasterAddressTracker.setMasterAddress(this.watcher,
            this.watcher.getMasterAddressZNode(), this.sn, infoPort)) {
          // If we were a backup master before, delete our ZNode from the backup
          // master directory since we are the active now)
          if (ZKUtil.checkExists(this.watcher, backupZNode) != -1) {
            LOG.info("Deleting ZNode for " + backupZNode + " from backup master directory");
            ZKUtil.deleteNodeFailSilent(this.watcher, backupZNode);
          }
          // Save the znode in a file, this will allow to check if we crash in the launch scripts
          ZNodeClearer.writeMyEphemeralNodeOnDisk(this.sn.toString());
          // We are the master, return
          startupStatus.setStatus("Successfully registered as active master.");
          this.clusterHasActiveMaster.set(true);
          LOG.info("Registered Active Master=" + this.sn);
          return true;
        }
        // There is another active master running elsewhere or this is a restart
        // and the master ephemeral node has not expired yet.
        this.clusterHasActiveMaster.set(true);
        String msg;
        byte[] bytes =
          ZKUtil.getDataAndWatch(this.watcher, this.watcher.getMasterAddressZNode());
        if (bytes == null) {
          msg = ("A master was detected, but went down before its address " +
            "could be read.  Attempting to become the next active master");
        } else {
          ServerName currentMaster;
          try {
            currentMaster = ServerName.parseFrom(bytes);
          } catch (DeserializationException e) {
            LOG.warn("Failed parse", e);
            // Hopefully next time around we won't fail the parse.  Dangerous.
            continue;
          }
          if (ServerName.isSameHostnameAndPort(currentMaster, this.sn)) {
            msg = ("Current master has this master's address, " +
              currentMaster + "; master was restarted? Deleting node.");
            // Hurry along the expiration of the znode.
            ZKUtil.deleteNode(this.watcher, this.watcher.getMasterAddressZNode());
            // We may have failed to delete the znode at the previous step, but
            //  we delete the file anyway: a second attempt to delete the znode is likely to fail again.
            ZNodeClearer.deleteMyEphemeralNodeOnDisk();
          } else {
            msg = "Another master is the active master, " + currentMaster +
              "; waiting to become the next active master";
          }
        }
        LOG.info(msg);
        startupStatus.setStatus(msg);
      } catch (KeeperException ke) {
        master.abort("Received an unexpected KeeperException, aborting", ke);
        return false;
      }
      synchronized (this.clusterHasActiveMaster) {
        while (clusterHasActiveMaster.get() && !master.isStopped()) {
          try {
            clusterHasActiveMaster.wait(checkInterval);
          } catch (InterruptedException e) {
            // We expect to be interrupted when a master dies,
            //  will fall out if so
            LOG.debug("Interrupted waiting for master to die", e);
          }
        }
        if (clusterShutDown.get()) {
          this.master.stop(
            "Cluster went down before this master became active");
        }
      }
    }
    return false;
  }

然后是启动的Master 不是backup的。需要调用这个方法finishActiveMasterInitialization

这个方法最主要的功能，

等待regionService 向HMaster 报告，如果没有 这会去zk 问一下原因。

   
private void finishActiveMasterInitialization(MonitoredTask status)
      throws IOException, InterruptedException, KeeperException, CoordinatedStateException {
    /*
     * We are active master now... go initialize components we need to run.
     * Note, there may be dross in zk from previous runs; it'll get addressed
     * below after we determine if cluster startup or failover.
     */
    status.setStatus("Initializing Master file system");
    this.masterActiveTime = System.currentTimeMillis();
    // TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
    this.fileSystemManager = new MasterFileSystem(this, 

//装载表信息到内存
    // enable table descriptors cache
    this.tableDescriptors.setCacheOn();
    // set the META's descriptor to the correct replication
    this.tableDescriptors.get(TableName.META_TABLE_NAME).setRegionReplication(
        conf.getInt(HConstants.META_REPLICAS_NUM, HConstants.DEFAULT_META_REPLICA_NUM));
    // warm-up HTDs cache on master initialization
    if (preLoadTableDescriptors) {
      status.setStatus("Pre-loading table descriptors");
      this.tableDescriptors.getAll();
    }
//写入Cluster ID 到zk
    // publish cluster ID
    status.setStatus("Publishing Cluster ID in ZooKeeper");
    ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId());
    this.serverManager = createServerManager(this, this);
    setupClusterConnection();
    // Invalidate all write locks held previously
    this.tableLockManager.reapWriteLocks();
    status.setStatus("Initializing ZK system trackers");
    initializeZKBasedSystemTrackers();
    // initialize master side coprocessors before we start handling requests
    status.setStatus("Initializing master coprocessors");
    this.cpHost = new MasterCoprocessorHost(this, this.conf);
    // start up all service threads.
    status.setStatus("Initializing master service threads");
    startServiceThreads();
    // Wake up this server to check in
    sleeper.skipSleepCycle();
//等待regionService注册
    // Wait for region servers to report in
    this.serverManager.waitForRegionServers(status);
    // Check zk for region servers that are up but didn't register
    for (ServerName sn: this.regionServerTracker.getOnlineServers()) {
      // The isServerOnline check is opportunistic, correctness is handled inside
      if (!this.serverManager.isServerOnline(sn)
          && serverManager.checkAndRecordNewServer(sn, ServerLoad.EMPTY_SERVERLOAD)) {
        LOG.info("Registered server found up in zk but who has not yet reported in: " + sn);
      }
    }
    // get a list for previously failed RS which need log splitting work
    // we recover hbase:meta region servers inside master initialization and
    // handle other failed servers in SSH in order to start up master node ASAP
    Set<ServerName> previouslyFailedServers =
      this.fileSystemManager.getFailedServersFromLogFolders();
    // log splitting for hbase:meta server
    ServerName oldMetaServerLocation = metaTableLocator.getMetaRegionLocation(this.getZooKeeper());
    if (oldMetaServerLocation != null && previouslyFailedServers.contains(oldMetaServerLocation)) {
      splitMetaLogBeforeAssignment(oldMetaServerLocation);
      // Note: we can't remove oldMetaServerLocation from previousFailedServers list because it
      // may also host user regions
    }
    Set<ServerName> previouslyFailedMetaRSs = getPreviouselyFailedMetaServersFromZK();
    // need to use union of previouslyFailedMetaRSs recorded in ZK and previouslyFailedServers
    // instead of previouslyFailedMetaRSs alone to address the following two situations:
    // 1) the chained failure situation(recovery failed multiple times in a row).
    // 2) master get killed right before it could delete the recovering hbase:meta from ZK while the
    // same server still has non-meta wals to be replayed so that
    // removeStaleRecoveringRegionsFromZK can't delete the stale hbase:meta region
    // Passing more servers into splitMetaLog is all right. If a server doesn't have hbase:meta wal,
    // there is no op for the server.
    previouslyFailedMetaRSs.addAll(previouslyFailedServers);
    this.initializationBeforeMetaAssignment = true;
    // Wait for regionserver to finish initialization.
    if (BaseLoadBalancer.tablesOnMaster(conf)) {
      waitForServerOnline();
    }
//设置负载均衡
    //initialize load balancer
    this.balancer.setClusterStatus(getClusterStatus());
    this.balancer.setMasterServices(this);
    this.balancer.initialize();
    // Check if master is shutting down because of some issue
    // in initializing the regionserver or the balancer.
    if (isStopped()) return;
    // Make sure meta assigned before proceeding.
    status.setStatus("Assigning Meta Region");
    assignMeta(status, previouslyFailedMetaRSs, HRegionInfo.DEFAULT_REPLICA_ID);
    // check if master is shutting down because above assignMeta could return even hbase:meta isn't
    // assigned when master is shutting down
    if (isStopped()) return;
    status.setStatus("Submitting log splitting work for previously failed region servers");
    // Master has recovered hbase:meta region server and we put
    // other failed region servers in a queue to be handled later by SSH
    for (ServerName tmpServer : previouslyFailedServers) {
      this.serverManager.processDeadServer(tmpServer, true);
    }
    // Update meta with new PB serialization if required. i.e migrate all HRI to PB serialization
    // in meta. This must happen before we assign all user regions or else the assignment will fail.
    if (this.conf.getBoolean("hbase.MetaMigrationConvertingToPB", true)) {
      MetaMigrationConvertingToPB.updateMetaIfNecessary(this);
    }
    // Fix up assignment manager status
    status.setStatus("Starting assignment manager");
    this.assignmentManager.joinCluster();
    // set cluster status again after user regions are assigned
    this.balancer.setClusterStatus(getClusterStatus());
    // Start balancer and meta catalog janitor after meta and regions have been assigned.
    status.setStatus("Starting balancer and catalog janitor");
    this.clusterStatusChore = new ClusterStatusChore(this, balancer);
    getChoreService().scheduleChore(clusterStatusChore);
    this.balancerChore = new BalancerChore(this);
    getChoreService().scheduleChore(balancerChore);
    this.normalizerChore = new RegionNormalizerChore(this);
    getChoreService().scheduleChore(normalizerChore);
    this.catalogJanitorChore = new CatalogJanitor(this, this);
    getChoreService().scheduleChore(catalogJanitorChore);
    // Do Metrics periodically
    periodicDoMetricsChore = new PeriodicDoMetrics(msgInterval, this);
    getChoreService().scheduleChore(periodicDoMetricsChore);
    status.setStatus("Starting namespace manager");
    initNamespace();
    if (this.cpHost != null) {
      try {
        this.cpHost.preMasterInitialization();
      } catch (IOException e) {
        LOG.error("Coprocessor preMasterInitialization() hook failed", e);
      }
    }
    status.markComplete("Initialization successful");
    LOG.info("Master has completed initialization");
    configurationManager.registerObserver(this.balancer);
    // Set master as 'initialized'.
    setInitialized(true);
    status.setStatus("Starting quota manager");
    initQuotaManager();
    // assign the meta replicas
    Set<ServerName> EMPTY_SET = new HashSet<ServerName>();
    int numReplicas = conf.getInt(HConstants.META_REPLICAS_NUM,
           HConstants.DEFAULT_META_REPLICA_NUM);
    for (int i = 1; i < numReplicas; i++) {
      assignMeta(status, EMPTY_SET, i);
    }
    unassignExcessMetaReplica(zooKeeper, numReplicas);
    // clear the dead servers with same host name and port of online server because we are not
    // removing dead server with same hostname and port of rs which is trying to check in before
    // master initialization. See HBASE-5916.
    this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();
    // Check and set the znode ACLs if needed in case we are overtaking a non-secure configuration
    status.setStatus("Checking ZNode ACLs");
    zooKeeper.checkAndSetZNodeAcls();
    status.setStatus("Calling postStartMaster coprocessors");
    if (this.cpHost != null) {
      // don't let cp initialization errors kill the master
      try {
        this.cpHost.postStartMaster();
      } catch (IOException ioe) {
        LOG.error("Coprocessor postStartMaster() hook failed", ioe);
      }
    }
    zombieDetector.interrupt();
  }

最后调用run方法。

里面主要是启动一下健康检查等类，然后等待停止。

regionServer 的run里面还有很多线程启动，最终要的是 MemStoreFlusher， CompactSplitThread， registerConfigurationObservers等。

memstore线程将在下个章节 flush 和 spit过程中使用到。

    
private void initializeThreads() throws IOException {
    // Cache flushing thread.
    this.cacheFlusher = new MemStoreFlusher(conf, this);
    // Compaction thread
    this.compactSplitThread = new CompactSplitThread(this);
    // Background thread to check for compactions; needed if region has not gotten updates
    // in a while. It will take care of not checking too frequently on store-by-store basis.
    this.compactionChecker = new CompactionChecker(this, this.threadWakeFrequency, this);
    this.periodicFlusher = new PeriodicMemstoreFlusher(this.threadWakeFrequency, this);
    this.leases = new Leases(this.threadWakeFrequency);
    // Create the thread to clean the moved regions list
    movedRegionsCleaner = MovedRegionsCleaner.create(this);
    if (this.nonceManager != null) {
      // Create the scheduled chore that cleans up nonces.
      nonceManagerChore = this.nonceManager.createCleanupScheduledChore(this);
    }
    // Setup the Quota Manager
    rsQuotaManager = new RegionServerQuotaManager(this);
    // Setup RPC client for master communication
    rpcClient = RpcClientFactory.createClient(conf, clusterId, new InetSocketAddress(
        rpcServices.isa.getAddress(), 0), clusterConnection.getConnectionMetrics());

    if (storefileRefreshPeriod > 0) {
      this.storefileRefresher = new StorefileRefresherChore(storefileRefreshPeriod,
          onlyMetaRefresh, this, this);
    }
    registerConfigurationObservers();
  }

到此结束。

上一章节：hbase 原代码分析 （11） WAL 写日志过程

http://blog.youkuaiyun.com/chenfenggang/article/details/75142075

regionServer 的run里面还有很多线程启动，最终要的是 MemStoreFlusher， CompactSplitThread， registerConfigurationObservers等。

    
private void initializeThreads() throws IOException {
    // Cache flushing thread.
    this.cacheFlusher = new MemStoreFlusher(conf, this);
    // Compaction thread
    this.compactSplitThread = new CompactSplitThread(this);
    // Background thread to check for compactions; needed if region has not gotten updates
    // in a while. It will take care of not checking too frequently on store-by-store basis.
    this.compactionChecker = new CompactionChecker(this, this.threadWakeFrequency, this);
    this.periodicFlusher = new PeriodicMemstoreFlusher(this.threadWakeFrequency, this);
    this.leases = new Leases(this.threadWakeFrequency);
    // Create the thread to clean the moved regions list
    movedRegionsCleaner = MovedRegionsCleaner.create(this);
    if (this.nonceManager != null) {
      // Create the scheduled chore that cleans up nonces.
      nonceManagerChore = this.nonceManager.createCleanupScheduledChore(this);
    }
    // Setup the Quota Manager
    rsQuotaManager = new RegionServerQuotaManager(this);
    // Setup RPC client for master communication
    rpcClient = RpcClientFactory.createClient(conf, clusterId, new InetSocketAddress(
        rpcServices.isa.getAddress(), 0), clusterConnection.getConnectionMetrics());
    boolean onlyMetaRefresh = false;
    int storefileRefreshPeriod = conf.getInt(
        StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD
      , StorefileRefresherChore.DEFAULT_REGIONSERVER_STOREFILE_REFRESH_PERIOD);
    if (storefileRefreshPeriod == 0) {
      storefileRefreshPeriod = conf.getInt(
          StorefileRefresherChore.REGIONSERVER_META_STOREFILE_REFRESH_PERIOD,
          StorefileRefresherChore.DEFAULT_REGIONSERVER_STOREFILE_REFRESH_PERIOD);
      onlyMetaRefresh = true;
    }
    if (storefileRefreshPeriod > 0) {
      this.storefileRefresher = new StorefileRefresherChore(storefileRefreshPeriod,
          onlyMetaRefresh, this, this);
    }
    registerConfigurationObservers();
  }