hadoop部署

草吃兔

已于 2023-07-31 15:34:15 修改

阅读量49

点赞数

分类专栏： hadoop 文章标签： hadoop 大数据分布式

于 2019-04-08 16:26:31 首次发布

本文链接：https://blog.youkuaiyun.com/waplys/article/details/89094587

版权

hadoop 专栏收录该内容

4 篇文章

订阅专栏

文章目录

安装
配置
启动服务

安装

官网地址：https://hadoop.apache.org/releases.html
下载：举例

To verify Hadoop releases using GPG:

    Download the release hadoop-X.Y.Z-src.tar.gz from a mirror site.
    Download the signature file hadoop-X.Y.Z-src.tar.gz.asc from Apache.
    Download the Hadoop KEYS file.
    gpg –import KEYS
    gpg –verify hadoop-X.Y.Z-src.tar.gz.asc

To perform a quick check using SHA-256:

    Download the release hadoop-X.Y.Z-src.tar.gz from a mirror site.
    Download the checksum hadoop-X.Y.Z-src.tar.gz.mds from Apache.
    shasum -a 256 hadoop-X.Y.Z-src.tar.gz

All previous releases of Hadoop are available from the Apache release archive site.

Many third parties distribute products that include Apache Hadoop and related tools. Some of these are listed on the Distributions wiki page.
License

The software licensed under Apache License 2.0

下载后解压即可，master配置完毕，slave节点直接scp同步即可。

【启动】
1、清空tmp目录：
	# rm -rf ${HADOOP_HOME}/tmp/*
2、格式化HDFS
	# hdfs namenode -format
3、启动Hadoop
	horn：# ${HADOOP_HOME}/sbin/start-dfs.sh 
	【检查：jps （检查NameNode、DataNode、SecondaryNamenode进程）】
	yarn：# ${HADOOP_HOME}/sbin/start-all.sh
	【检查：jps （检查NameNode、DataNode、SecondaryNamenode、ResourceManager进程）】
4、故障分析
	查询logs下日志酌情处理

配置

环境变量

[root@config]# vi ~/.bashrc
#jdk#
export JAVA_HOME=/home/java
export JRE_HOME=$JAVA_HOME/jre
export CLASS_PATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export PATH=$JAVA_HOME/bin:$PATH

#hadoop#
export HADOOP_HOME=/home/hadoop
export HADOOP_HDFS_HOME=${HADOOP_HOME}
export HADOOP_COMMON_HOME=${HADOOP_HOME}
export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop
export HADOOP_MAPRED_HOME=${HADOOP_HOME}
export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH

core-site.xml

# vim /home/hadoop/etc/hadoop/core-site.xml
【默认配置】
<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://yitaai-1:9000</value>  <!--限制IP，可配置0.0.0.0去除（类似Mysql-bindip）
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/home/hadoop/tmp</value>
    </property>
<property>
        <name>hadoop.security.authorization</name>
        <value>false</value>
</property>
</configuration>
【优化配置】
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<configuration>
<property>
<name>hadoop.security.auth_to_local</name>
<value/>
</property>
<property>
<name>ipc.client.kill.max</name>
<value>50</value>
</property>
<property>
<name>io.compression.codec.lzo.class</name>
<value>com.hadoop.compression.lzo.LzoCodec</value>
</property>
<property>
<name>hadoop.http.staticuser.user</name>
<value>hdfs</value>
</property>
<property>
<name>fs.trash.checkpoint.interval</name>
<value>1440</value>
</property>
<property>
<name>hadoop.proxyuser.mr.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.mr.groups</name>
<value>*</value>
</property>
<property>
<name>fs.defaultFS</name>
<value>hdfs://jfsh4:9000</value>
</property>
<property>
<name>hadoop.proxyuser.flume.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.HTTP.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.HTTP.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hive.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.oozie.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.mapred.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.mapred.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.flume.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hue.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hive.groups</name>
<value>*</value>
</property>
<property>
<name>io.bytes.per.checksum</name>
<value>4096</value>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.search.filter.user</name>
<value>(&amp;(objectClass=user)(sAMAccountName={0}))</value>
</property>
<property>
<name>fs.trash.interval</name>
<value>20160</value>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.search.filter.group</name>
<value>(objectClass=group)</value>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.search.attr.member</name>
<value>member</value>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.search.attr.group.name</name>
<value>cn</value>
</property>
<property>
<name>hadoop.security.authentication</name>
<value>simple</value>
</property>
<property>
<name>hadoop.security.authorization</name>
<value>false</value>
</property>
<property>
<name>hadoop.rpc.protection</name>
<value>authentication</value>
</property>
<property>
<name>hadoop.ssl.enabled</name>
<value>false</value>
</property>
<property>
<name>hadoop.proxyuser.oozie.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hue.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.yarn.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.yarn.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.httpfs.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.httpfs.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.bind.user</name>
<value/>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.bind.password</name>
<value/>
</property>
<property>
<name>hadoop.security.group.mapping</name>
<value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.url</name>
<value/>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.base</name>
<value/>
</property>
<property>
<name>io.file.buffer.size</name>
<value>131072</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/data1/hdfs/tmp</value>
</property>
<property>
<name>hadoop.proxyuser.hdfs.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hdfs.groups</name>
<value>*</value>
</property>
<property>
<name>io.compression.codecs</name>
<value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.SnappyCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec</value>
</property>
<property>
<name>fs.permissions.umask-mode</name>
<value>027</value>
</property>
</configuration>

hadoop-env.sh

注意JAVA_HOME是否正确：
# The java implementation to use.
export JAVA_HOME=/home/java

hdfs-site.xml

【默认配置】
<configuration>
        <property>
                <name>dfs.namenode.secondary.http-address</name>
                <value>yitaai-1:50090</value>
        </property>
        <property>
                <name>dfs.replication</name>
                <value>2</value>
        </property>
        <property>
                <name>dfs.namenode.name.dir</name>
                <value>file:/data1/hadoop/tmp/dfs/name</value>  
        </property>
        <property>
                <name>dfs.datanode.data.dir</name>
                <value>file:/data1/hadoop/tmp/dfs/data</value>
        </property>
        <property>
                <name>dfs.namenode.handler.count</name>
                <value>3000</value>
        </property>
        <property>
                <name>dfs.datanode.handler.count</name>
                <value>3000</value>
        </property>
        <property>
                <name>dfs.namenode.datanode.registration.ip-hostname-check</name>
                <value>false</value>
        </property>
</configuration>
【优化配置】
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<configuration>
<property>
<name>dfs.namenode.checkpoint.edits.dir</name>
<value>${dfs.namenode.checkpoint.dir}</value>
</property>
<property>
<name>dfs.namenode.edits.dir</name>
<value>/data1/hdfs/dfs/name</value>
</property>
<property>
<name>dfs.namenode.acls.enabled</name>
<value>true</value>
</property>
<property>
<name>dfs.permissions.superusergroup</name>
<value>users</value>
</property>
<property>
<name>dfs.namenode.support.allow.format</name>
<value>true</value>
</property>
<property>
<name>dfs.replication.max</name>
<value>512</value>
</property>
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<property>
<name>hdfs.ranger.enable</name>
<value>false</value>
</property>
<property>
<name>dfs.datanode.balance.max.concurrent.moves</name>
<value>5</value>
</property>
<property>
<name>dfs.namenode.checkpoint.dir</name>
<value>/data1/hdfs/dfs/namesecondary</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>/data1/hdfs/dfs/name</value>
</property>
<property>
<name>dfs.datanode.hdfs-blocks-metadata.enabled</name>
<value>true</value>
</property>
<property>
<name>dfs.client.use.datanode.hostname</name>
<value>false</value>
</property>
<property>
<name>dfs.client.read.shortcircuit</name>
<value>true</value>
</property>
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/data2/hdfs/dfs/journal</value>
</property>
<property>
<name>dfs.namenode.servicerpc-address</name>
<value/>
</property>
<property>
<name>dfs.client.failover.proxy.provider</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<property>
<name>dfs.encrypt.data.transfer</name>
<value>false</value>
</property>
<property>
<name>dfs.encrypt.data.transfer.algorithm</name>
<value>3des</value>
</property>
<property>
<name>dfs.domain.socket.path</name>
<value>/var/run/hdfs-sockets/dn</value>
</property>
<property>
<name>dfs.block.local-path-access.user</name>
<value/>
</property>
<property>
<name>dfs.journalnode.rpc-address</name>
<value>8485</value>
</property>
<property>
<name>dfs.namenode.http-address</name>
<value>jfsh4:50070</value>
</property>
<property>
<name>dfs.namenode.rpc-address</name>
<value/>
</property>
<property>
<name>dfs.blocksize</name>
<value>134217728</value>
</property>
<property>
<name>dfs.image.transfer.timeout</name>
<value>60000</value>
</property>
<property>
<name>dfs.client.file-block-storage-locations.timeout.millis</name>
<value>10000</value>
</property>
<property>
<name>dfs.image.transfer.bandwidthPerSec</name>
<value>0</value>
</property>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>jfsh4:50090</value>
</property>
<property>
<name>regions.shift</name>
<value>false</value>
</property>
<property>
<name>all.slaves</name>
<value>jfsh4</value>
</property>
</configuration>

mapred-site.xml

【默认】
<configuration>
        <property>
                <name>mapreduce.framework.name</name>
                <value>yarn</value>
        </property>
        <property>
                <name>mapreduce.jobhistory.address</name>
                <value>yitaai-1:10020</value>
        </property>
        <property>
                <name>mapreduce.jobhistory.webapp.address</name>
                <value>yitaai-1:19888</value>
        </property>
</configuration>
【优化】
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<configuration>
<property>
<name>mapreduce.jobhistory.max-age-ms</name>
<value>604800000</value>
</property>
<property>
<name>mapreduce.map.cpu.vcores</name>
<value>1</value>
</property>
<property>
<name>mapreduce.reduce.cpu.vcores</name>
<value>1</value>
</property>
<property>
<name>yarn.app.mapreduce.am.resource.cpu-vcores</name>
<value>1</value>
</property>
<property>
<name>mapreduce.job.counters.limit</name>
<value>120</value>
</property>
<property>
<name>mapreduce.job.ubertask.maxreduces</name>
<value>1</value>
</property>
<property>
<name>mapreduce.job.ubertask.maxmaps</name>
<value>9</value>
</property>
<property>
<name>mapreduce.output.fileoutputformat.compress.type</name>
<value>RECORD</value>
</property>
<property>
<name>mapreduce.map.output.compress</name>
<value>true</value>
</property>
<property>
<name>mapreduce.map.output.compress.codec</name>
<value>com.hadoop.compression.lzo.LzoCodec</value>
</property>
<property>
<name>mapreduce.application.classpath</name>
<value>/opt/parcels/lib/hadoop-mapreduce/*,/opt/parcels/lib/hadoop-mapreduce/lib/*</value>
</property>
<property>
<name>mapreduce.admin.user.env</name>
<value/>
</property>
<property>
<name>yarn.app.mapreduce.am.staging-dir</name>
<value>/hadoop/hadoop-yarn/staging</value>
</property>
<property>
<name>mapreduce.job.ubertask.enable</name>
<value>false</value>
</property>
<property>
<name>mapreduce.reduce.shuffle.parallelcopies</name>
<value>100</value>
</property>
<property>
<name>mapreduce.output.fileoutputformat.compress.codec</name>
<value>org.apache.hadoop.io.compress.DefaultCodec</value>
</property>
<property>
<name>mapreduce.output.fileoutputformat.compress</name>
<value>false</value>
</property>
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>jfsh3:19888</value>
</property>
<property>
<name>mapreduce.jobhistory.address</name>
<value>jfsh3:10020</value>
</property>
<property>
<name>mapreduce.jobhistory.webapp.https.address</name>
<value>jfsh3:19890</value>
</property>
<property>
<name>mapreduce.jobhistory.admin.address</name>
<value>jfsh3:10033</value>
</property>
<property>
<name>mapreduce.job.ubertask.maxbytes</name>
<value/>
</property>
<property>
<name>mapreduce.jobhistory.hostname</name>
<value>jfsh3</value>
</property>
<property>
<name>mapreduce.application.thirdparty.classpath</name>
<value>/home/mr/lib/3rd</value>
</property>
<property>
<name>mapreduce.user.specified</name>
<value>true</value>
</property>
<property>
<name>mapreduce.client.submit.file.replication</name>
<value>10</value>
</property>
<property>
<name>mapreduce.map.java.opts</name>
<value>-Xmx1024m</value>
</property>
<property>
<name>mapreduce.reduce.java.opts</name>
<value>-Xmx1536m</value>
</property>
<property>
<name>mapreduce.task.io.sort.factor</name>
<value>500</value>
</property>
<property>
<name>mapreduce.map.sort.spill.percent</name>
<value>0.8</value>
</property>
<property>
<name>mapreduce.reduce.speculative</name>
<value>false</value>
</property>
<property>
<name>mapreduce.job.reduces</name>
<value>1</value>
</property>
<property>
<name>mapreduce.shuffle.max.connections</name>
<value>0</value>
</property>
<property>
<name>mapreduce.job.reduce.slowstart.completedmaps</name>
<value>0.8</value>
</property>
<property>
<name>mapreduce.job.split.metainfo.maxsize</name>
<value>10000000</value>
</property>
<property>
<name>yarn.app.mapreduce.am.command-opts</name>
<value>-Xmx1024m</value>
</property>
<property>
<name>yarn.app.mapreduce.am.admin.user.env</name>
<value/>
</property>
<property>
<name>mapreduce.cluster.temp.dir</name>
<value>/data1/yarn/logs/mapred/temp</value>
</property>
<property>
<name>mapreduce.am.max-attempts</name>
<value>2</value>
</property>
<property>
<name>mapreduce.jobtracker.jobhistory.location</name>
<value/>
</property>
<property>
<name>mapreduce.map.speculative</name>
<value>false</value>
</property>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
</configuration>

yarn-site.xml

【默认】
<configuration>
        <property>
                <name>yarn.resourcemanager.hostname</name>
                <value>yitaai-1</value>
        </property>
        <property>
                <name>yarn.nodemanager.aux-services</name>
                <value>mapreduce_shuffle</value>
        </property>
</configuration>
【优化】
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<configuration>
<property>
<name>yarn.resourcemanager.container.liveness-monitor.interval-ms</name>
<value>600000</value>
</property>
<property>
<name>yarn.nodemanager.localizer.cache.target-size-mb</name>
<value>10240</value>
</property>
<property>
<name>yarn.nodemanager.log.retain-seconds</name>
<value>10800</value>
</property>
<property>
<name>yarn.resourcemanager.amliveliness-monitor.interval-ms</name>
<value>1000</value>
</property>
<property>
<name>yarn.am.liveness-monitor.expiry-interval-ms</name>
<value>60000</value>
</property>
<property>
<name>mapreduce.jobhistory.cleaner.interval-ms</name>
<value>86400000</value>
</property>
<property>
<name>yarn.nodemanager.localizer.cache.cleanup.interval-ms</name>
<value>600000</value>
</property>
<property>
<name>yarn.resourcemanager.nodemanagers.heartbeat-interval-ms</name>
<value>500</value>
</property>
<property>
<name>yarn.client.failover-sleep-max-ms</name>
<value>2000</value>
</property>
<property>
<name>mapreduce.task.io.sort.mb</name>
<value>256</value>
</property>
<property>
<name>yarn.client.failover-sleep-base-ms</name>
<value>100</value>
</property>
<property>
<name>yarn.log-aggregation.retain-seconds</name>
<value>691200</value>
</property>
<property>
<name>mapreduce.task.timeout</name>
<value>600000</value>
</property>
<property>
<name>yarn.resourcemanager.rm.container-allocation.expiry-interval-ms</name>
<value>60000</value>
</property>
<property>
<name>yarn.resourcemanager.fs.state-store.uri</name>
<value>/hadoop/rmstore</value>
</property>
<property>
<name>yarn.resourcemanager.fs.state-store.retry-policy-spec</name>
<value>2000,150</value>
</property>
<property>
<name>yarn.resourcemanager.connect.retry-interval.ms</name>
<value>2000</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.https.address</name>
<value>8090</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address</name>
<value>23141</value>
</property>
<property>
<name>yarn.resourcemanager.ha.admin.address</name>
<value>23142</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>23130</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>23125</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>8088</value>
</property>
<property>
<name>yarn.resourcemanager.address</name>
<value>23140</value>
</property>
<property>
<name>yarn.nodemanager.address</name>
<value>null:8041</value>
</property>
<property>
<name>yarn.nodemanager.webapp.address</name>
<value>null:8042</value>
</property>
<property>
<name>yarn.nodemanager.localizer.address</name>
<value>null:8040</value>
</property>
<property>
<name>mapreduce.map.memory.mb</name>
<value>1536</value>
</property>
<property>
<name>mapreduce.reduce.memory.mb</name>
<value>2048</value>
</property>
<property>
<name>yarn.nodemanager.webapp.https.address</name>
<value>null:8044</value>
</property>
<property>
<name>yarn.app.mapreduce.am.resource.mb</name>
<value>1536</value>
</property>
<property>
<name>yarn.scheduler.fair.locality-delay-rack-ms</name>
<value>4000</value>
</property>
<property>
<name>yarn.scheduler.fair.locality-delay-node-ms</name>
<value>2000</value>
</property>
<property>
<name>yarn.scheduler.maximum-allocation-mb</name>
<value>8192</value>
</property>
<property>
<name>yarn.resourcemanager.nm.liveness-monitor.interval-ms</name>
<value>1000</value>
</property>
<property>
<name>yarn.nm.liveness-monitor.expiry-interval-ms</name>
<value>60000</value>
</property>
<property>
<name>yarn.scheduler.minimum-allocation-mb</name>
<value>1024</value>
</property>
<property>
<name>yarn.scheduler.increment-allocation-mb</name>
<value>512</value>
</property>
<property>
<name>yarn.resourcemanager.hostname</name>
<value>jfsh3,jfsh4</value>
</property>
<property>
<name>yarn.resourcemanager.checkclients.enabled</name>
<value>false</value>
</property>
<property>
<name>yarn.rm1.hostname</name>
<value>jfsh3</value>
</property>
<property>
<name>yarn.rm2.hostname</name>
<value>jfsh4</value>
</property>
<property>
<name>yarn.nodemanager.health-checker.script.timeout-ms</name>
<value>1200000</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.application.submit.checker.class</name>
<value>com.zte.yarn.server.resourcemanager.scheduler.fair.AppSubmitLimitChecker</value>
</property>
<property>
<name>yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb</name>
<value>0</value>
</property>
<property>
<name>yarn.scheduler.increment-allocation-vcores</name>
<value>1</value>
</property>
<property>
<name>yarn.nodemanager.health-checker.interval-ms</name>
<value>600000</value>
</property>
<property>
<name>yarn.nodemanager.disk-health-checker.min-healthy-disks</name>
<value>0.25</value>
</property>
<property>
<name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
<value>90</value>
</property>
<property>
<name>yarn.nodemanager.disk-health-checker.enable</name>
<value>true</value>
</property>
<property>
<name>yarn.nodemanager.disk-health-checker.interval-ms</name>
<value>120000</value>
</property>
<property>
<name>yarn.nodemanager.linux-container-executor.cgroups.mount-path</name>
<value>/home/dapmanager-agent/cgroup</value>
</property>
<property>
<name>yarn.nodemanager.linux-container-executor.cgroups.mount</name>
<value>true</value>
</property>
<property>
<name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name>
<value>/yarn-NodeManager/hadoop-yarn</value>
</property>
<property>
<name>yarn.nodemanager.linux-container-executor.resources-handler.class</name>
<value>org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler</value>
</property>
<property>
<name>yarn.scheduler.minimum-allocation-vcores</name>
<value>1</value>
</property>
<property>
<name>yarn.scheduler.fair.allow-undeclared-pools</name>
<value>true</value>
</property>
<property>
<name>yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user</name>
<value>nobody</value>
</property>
<property>
<name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name>
<value>-1</value>
</property>
<property>
<name>yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users</name>
<value>false</value>
</property>
<property>
<name>yarn.scheduler.maximum-allocation-vcores</name>
<value>32</value>
</property>
<property>
<name>yarn.scheduler.fair.continuous-scheduling-enabled</name>
<value>false</value>
</property>
<property>
<name>yarn.resourcemanager.connect.max-wait.ms</name>
<value>600000</value>
</property>
<property>
<name>yarn.resourcemanager.store.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore</value>
</property>
<property>
<name>yarn.nodemanager.vmem-check-enabled</name>
<value>false</value>
</property>
<property>
<name>yarn.scheduler.fair.sizebasedweight</name>
<value>false</value>
</property>
<property>
<name>yarn.scheduler.fair.preemptionInterval</name>
<value>5000</value>
</property>
<property>
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</property>
<property>
<name>yarn.scheduler.fair.assignmultiple</name>
<value>false</value>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>jfsh3:2181,jfsh4:2181,jfsh5:2181</value>
</property>
<property>
<name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle,spark_shuffle,llama_nm_plugin</value>
</property>
<property>
<name>yarn.resourcemanager.zk-address</name>
<value>jfsh3:2181,jfsh4:2181,jfsh5:2181</value>
</property>
<property>
<name>yarn.nodemanager.container-executor.class</name>
<value>org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor</value>
</property>
<property>
<name>yarn.scheduler.fair.preemption.cluster-utilization-threshold</name>
<value>0</value>
</property>
<property>
<name>yarn.scheduler.fair.waitTimeBeforeKill</name>
<value>15000</value>
</property>
<property>
<name>yarn.scheduler.fair.preemption</name>
<value>false</value>
</property>
<property>
<name>yarn.scheduler.fair.user-as-default-queue</name>
<value>true</value>
</property>
<property>
<name>yarn.resourcemanager.proxy-user-privileges.enabled</name>
<value>false</value>
</property>
<property>
<name>yarn.resourcemanager.max-completed-applications</name>
<value>600</value>
</property>
<property>
<name>yarn.scheduler.fair.allocation.file</name>
<value>fair-scheduler.xml</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.client.thread-count</name>
<value>50</value>
</property>
<property>
<name>yarn.resourcemanager.am.max-attempts</name>
<value>2</value>
</property>
<property>
<name>yarn.security.authorization.enabled</name>
<value>false</value>
</property>
<property>
<name>yarn.am.blacklisting.disable-failure-threshold</name>
<value>0.8f</value>
</property>
<property>
<name>yarn.resourcemanager.zk-auth</name>
<value>digest:yarnacl:acl1234</value>
</property>
<property>
<name>yarn.resourcemanager.zk-timeout-ms</name>
<value>20000</value>
</property>
<property>
<name>yarn.log.server.url</name>
<value>jfsh3:19888</value>
</property>
<property>
<name>yarn.nodemanager.default-container-executor.log-dirs.permissions</name>
<value>710</value>
</property>
<property>
<name>yarn.resourcemanager.client.thread-count</name>
<value>50</value>
</property>
<property>
<name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
<value>100</value>
</property>
<property>
<name>yarn.resourcemanager.recovery.enabled</name>
<value>true</value>
</property>
<property>
<name>yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage</name>
<value>false</value>
</property>
<property>
<name>yarn.resourcemanager.admin.client.thread-count</name>
<value>1</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.client.thread-count</name>
<value>50</value>
</property>
<property>
<name>yarn.am.blacklisting.enabled</name>
<value>true</value>
</property>
<property>
<name>yarn.nodemanager.localizer.fetch.thread-count</name>
<value>4</value>
</property>
<property>
<name>yarn.nodemanager.localizer.client.thread-count</name>
<value>5</value>
</property>
<property>
<name>yarn.nodemanager.remote-app-log-dir-suffix</name>
<value>logs</value>
</property>
<property>
<name>yarn.nodemanager.remote-app-log-dir</name>
<value>/hadoop/agg-logs</value>
</property>
<property>
<name>yarn.nodemanager.delete.thread-count</name>
<value>4</value>
</property>
<property>
<name>yarn.nodemanager.container-manager.thread-count</name>
<value>20</value>
</property>
<property>
<name>yarn.resourcemanager.zk-acl</name>
<value>digest:yarnacl:ZT3FsZv+cEG6e84vtzIlcEn3MxQ=:cdrwa</value>
</property>
<property>
<name>yarn.scheduler.fair.max.assign</name>
<value>-1</value>
</property>
<property>
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2</value>
</property>
<property>
<name>yarn.log-aggregation-enable</name>
<value>true</value>
</property>
<property>
<name>yarn.resourcemanager.cluster-id</name>
<value>yarn-cluster</value>
</property>
<property>
<name>yarn.acl.enable</name>
<value>true</value>
</property>
<property>
<name>yarn.admin.acl</name>
<value>mr,hdfs</value>
</property>
<property>
<name>yarn.nodemanager.recovery.dir</name>
<value>/data1/yarn/yarn-nm-recovery</value>
</property>
<property>
<name>yarn.nodemanager.admin-env</name>
<value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value>
</property>
<property>
<name>yarn.nodemanager.local-dirs</name>
<value>/data1/yarn/local</value>
</property>
<property>
<name>yarn.nodemanager.log-dirs</name>
<value>/data1/yarn/logs/userlogs</value>
</property>
<property>
<name>yarn.nodemanager.env-whitelist</name>
<value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,HADOOP_YARN_HOME</value>
</property>
<property>
<name>yarn.nodemanager.health-checker.script.path</name>
<value/>
</property>
<property>
<name>yarn.nodemanager.health-checker.script.opts</name>
<value/>
</property>
<property>
<name>yarn.nodemanager.delete.debug-delay-sec</name>
<value>0</value>
</property>
<property>
<name>yarn.scheduler.fair.locality.threshold.node</name>
<value/>
</property>
<property>
<name>yarn.scheduler.fair.locality.threshold.rack</name>
<value/>
</property>
<property>
<name>yarn.resourcemanager.address.rm1</name>
<value>${yarn.rm1.hostname}:23140</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address.rm1</name>
<value>${yarn.rm1.hostname}:23130</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address.rm1</name>
<value>${yarn.rm1.hostname}:8088</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address.rm1</name>
<value>${yarn.rm1.hostname}:23125</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address.rm1</name>
<value>${yarn.rm1.hostname}:23141</value>
</property>
<property>
<name>yarn.resourcemanager.ha.admin.address.rm1</name>
<value>${yarn.rm1.hostname}:23142</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.https.address.rm1</name>
<value>${yarn.rm1.hostname}:8090</value>
</property>
<property>
<name>yarn.resourcemanager.address.rm2</name>
<value>${yarn.rm2.hostname}:23140</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address.rm2</name>
<value>${yarn.rm2.hostname}:23130</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address.rm2</name>
<value>${yarn.rm2.hostname}:8088</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address.rm2</name>
<value>${yarn.rm2.hostname}:23125</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address.rm2</name>
<value>${yarn.rm2.hostname}:23141</value>
</property>
<property>
<name>yarn.resourcemanager.ha.admin.address.rm2</name>
<value>${yarn.rm2.hostname}:23142</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.https.address.rm2</name>
<value>${yarn.rm2.hostname}:8090</value>
</property>
<property>
<name>yarn.application.classpath</name>
<value>$HADOOP_CONF_DIR,/opt/parcels/lib/hadoop/*,/opt/parcels/lib/hadoop/lib/*,/opt/parcels/lib/hadoop-hdfs/*,/opt/parcels/lib/hadoop-hdfs/lib/*,/opt/parcels/lib/hadoop-yarn/*,/opt/parcels/lib/hadoop-yarn/lib/*</value>
</property>
</configuration>

slaves

# slave节点列表
slave1
slave2

启动服务

# /home/hadoop/sbin
按需启动进程，如果需要yarn，直接start-all即可。