Introduction and reference:
http://hadoop.apache.org/docs/r2.7.1/hadoop-yarn/hadoop-yarn-site/FairScheduler.html
http://hadoop.apache.org/docs/r2.7.1/hadoop-yarn/hadoop-yarn-site/FairScheduler.html
Steps:
- Add following in yarn-site.xml:
<property>
<name>yarn.resourcemanager.scheduler.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.https.address.rm2</name>
<value>master02.dc.tj:8089</value>
</property>
<property>
<name>yarn.scheduler.fair.allocation.file</name>
<value>${HADOOP_HOME}/etc/hadoop/fair-scheduler.xml</value>
</property>
<property>
<name>yarn.scheduler.fair.user-as-default-queue</name>
<value>true</value>
</property>
<property>
<name>yarn.scheduler.fair.preemption</name>
<value>true</value>
</property> - Generate file ${HADOOP_HOME}/etc/hadoop/fair-scheduler.xml:
<?xml version="1.0"?>
<allocations>
<!--Currently for jobs submitted with user hadoop(hadoop admin) -->
<queue name="hadoop">
<aclSubmitApps>hadoop</aclSubmitApps>
<aclAdministerApps>hadoop</aclAdministerApps>
<minResources>5000 mb,0vcores</minResources>
<maxResources>90000 mb,0vcores</maxResources>
<maxRunningApps>100</maxRunningApps>
<maxAMShare>0.5</maxAMShare>
<weight>10.0</weight>
</queue>
<!--Currently for jobs submitted with user chenfangfang -->
<queue name="chenfangfang">
<aclSubmitApps>chenfangfang</aclSubmitApps>
<aclAdministerApps>hadoop,chenfangfang</aclAdministerApps>
<minResources>5000 mb,0vcores</minResources>
<maxResources>45000 mb,0vcores</maxResources>
<maxRunningApps>30</maxRunningApps>
<maxAMShare>0.5</maxAMShare>
<weight>1.0</weight>
</queue>
<queueMaxAMShareDefault>0.5</queueMaxAMShareDefault>
<!-- All other users besides hadoop and chenfangfang, will be included in clients queue -->
<queue name="clients" type="parent">
<aclSubmitApps>*</aclSubmitApps>
<aclAdministerApps>hadoop</aclAdministerApps>
<minResources>5000 mb,0vcores</minResources>
<maxResources>45000 mb,0vcores</maxResources>
<maxRunningApps>20</maxRunningApps>
<maxAMShare>0.5</maxAMShare>
<weight>1.0</weight>
</queue>
<queuePlacementPolicy>
<!-- Firstly, use spacified queue name as submitted queue-->
<rule name="specified" create="false" />
<!-- Then, use submitter user name as submitted queue-->
<rule name="user" create="false" />
<!-- Other jobs use clients queue as submitted queue-->
<rule name="default" queue="clients" />
</queuePlacementPolicy>
</allocations>