mapred-site.xml

最新推荐文章于 2024-10-31 10:28:30 发布

等上一层楼

最新推荐文章于 2024-10-31 10:28:30 发布

阅读量757

点赞数 1

CC 4.0 BY-SA版权

分类专栏： hadoop

本文链接：https://blog.youkuaiyun.com/dengshangyicenglou/article/details/82108016

hadoop 专栏收录该内容

3 篇文章

订阅专栏

此博客主要展示了MapReduce的相关配置信息，包含mapreduce.framework.name、mapreduce.cluster.administrators等众多属性的设置，如设置mapreduce.framework.name为yarn，还对内存、日志级别、压缩等方面进行了参数配置。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->

<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>

<property>
<name>mapreduce.cluster.administrators</name>
<value>hadoop</value>
</property>

<property>
<name>mapreduce.reduce.input.buffer.percent</name>
<value>0.0</value>
</property>

<property>
<name>mapreduce.reduce.shuffle.merge.percent</name>
<value>0.66</value>
</property>

<property>
<name>mapreduce.reduce.shuffle.input.buffer.percent</name>
<value>0.7</value>
</property>

<property>
<name>mapreduce.jobhistory.address</name>
<value>hadoop011:10020</value>
</property>

<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>hadoop011:19888</value>
</property>

<property>
<name>yarn.app.mapreduce.am.staging-dir</name>
<value>/opt/beh/data/yarn/staging</value>
</property>

<property>
<name>mapreduce.task.io.sort.mb</name>
<value>512</value>
</property>

<property>
<name>mapreduce.task.io.sort.factor</name>
<value>50</value>
</property>

<property>
<name>mapreduce.map.sort.spill.percent</name>
<value>0.7</value>
</property>

<property>
<name>mapred.job.reuse.jvm.num.tasks</name>
<value>-1</value>
</property>

<property>
<name>mapreduce.reduce.shuffle.parallelcopies</name>
<value>20</value>
</property>

<property>
<name>mapred.output.compress</name>
<value>true</value>
</property>

<property>
<name>mapred.output.compression.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
</property>

<property>
<name>mapreduce.map.output.compress</name>
<value>true</value>
</property>

<property>
<name>mapred.child.java.opts</name>
<value>-Xmx4096m</value>
</property>

<property>
<name>mapreduce.map.java.opts</name>
<value>-Xmx4096m</value>
</property>

<property>
<name>mapreduce.reduce.java.opts</name>
<value>-Xmx4096m</value>
</property>

<!--
<property>
<name>yarn.app.mapreduce.am.admin-command-opts</name>
<value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN</value>
</property>

<property>
<name>mapreduce.admin.map.child.java.opts</name>
<value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN</value>
</property>

<property>
<name>mapreduce.admin.reduce.child.java.opts</name>
<value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN</value>
</property>

<property>
<name>mapreduce.admin.user.env</name>
<value>LD_LIBRARY_PATH=/opt/beh/core/hadoop/lib/native:/opt/beh/core/hadoop/lib/native/`$JAVA_HOME/bin/java -d32 -version &> /dev/null;if [ $? -eq 0 ]; then echo Linux-i386-32; else echo Linux-amd64-64;fi`</value>
</property>
-->

<property>
<name>mapreduce.map.log.level</name>
<value>INFO</value>
</property>

<property>
<name>mapreduce.reduce.log.level</name>
<value>INFO</value>
</property>

<property>
<name>yarn.app.mapreduce.am.log.level</name>
<value>INFO</value>
</property>

<property>
<name>mapreduce.map.memory.mb</name>
<value>4096</value>
</property>

<property>
<name>yarn.app.mapreduce.am.resource.mb</name>
<value>4096</value>
<description>The amount of memory the MR AppMaster needs.</description>
</property>

<property>
<name>mapreduce.reduce.memory.mb</name>
<value>4096</value>
</property>

<property>
<name>mapred.output.compression.type</name>
<value>BLOCK</value>
</property>

<property>
<name>mapreduce.cluster.local.dir</name>
<value>/opt/beh/data/mapred/local</value>
<description>The local directory where MapReduce stores intermediate data files. May be a comma-separated list of directories on different devices in order to spread disk i/o. Directories that do not exist are ignored. </description>
</property>

<property>
<name>mapreduce.map.speculative</name>
<value>false</value>
</property>

<property>
<name>mapreduce.reduce.speculative</name>
<value>false</value>
</property>

<property>
<name>mapreduce.job.reduce.slowstart.completedmaps</name>
<value>1.0</value>
</property>

<property>
<name>mapreduce.job.counters.limit</name>
<value>2000</value>
</property>

</configuration>