自己架设hadoop集群的配置

最新推荐文章于 2024-03-18 17:53:34 发布

原创最新推荐文章于 2024-03-18 17:53:34 发布 · 119 阅读

0 ·

CC 4.0 BY-SA版权

本文详细介绍了Hadoop集群的核心配置文件，包括core-site.xml、hdfs-site.xml、mapred-site.xml和yarn-site.xml等，并提供了具体的参数设置及说明。

自己的配置：

[@tc_12_113 hadoop]# cat core-site.xml
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
? Licensed under the Apache License, Version 2.0 (the "License");
? you may not use this file except in compliance with the License.
? You may obtain a copy of the License at

??? http://www.apache.org/licenses/LICENSE-2.0

? Unless required by applicable law or agreed to in writing, software
? distributed under the License is distributed on an "AS IS" BASIS,
? WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
? See the License for the specific language governing permissions and
? limitations under the License. See accompanying LICENSE file.
-->

?? <property>

???? <name>fs.defaultFS</name>

???? <value>hdfs://hadoopmaster:9000</value>

?? </property>

?? <property>

???? <name>io.file.buffer.size</name>

???? <value>131072</value>

?? </property>

? <property>
??? <name>Hadoop.tmp.dir</name>
??? <value>/tmp/hadoop-root</value>
? </property>

?????? <property>

?????????????? <name>hadoop.proxyuser.hduser.hosts</name>

?????????????? <value>*</value>

?????? </property>

?????? <property>

?????????????? <name>hadoop.proxyuser.hduser.groups</name>

?????????????? <value>*</value>

?????? </property>

</configuration>

[@tc_12_113 hadoop]# cat hdfs-site.xml
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
? Licensed under the Apache License, Version 2.0 (the "License");
? you may not use this file except in compliance with the License.
? You may obtain a copy of the License at

??? http://www.apache.org/licenses/LICENSE-2.0

?????? <property>

??????????????? <name>dfs.namenode.secondary.http-address</name>

?????????????? <value>hadoopmaster:9001</value>

??????? </property>

???????? <property>

????????????????? <name>dfs.namenode.name.dir</name>

???????????????? <value>file:/tmp/hadoop/dfs/name</value>

??????????? </property>

?????????? <property>

??????????????????? <name>dfs.datanode.data.dir</name>

??????????????????? <value>file:/tmp/hadoop/dfs/data</value>

??????????? </property>

??????????? <property>

???????????????????? <name>dfs.replication</name>

???????????????????? <value>2</value>

???????????? </property>

???????????? <property>

???????????????????? <name>dfs.webhdfs.enabled</name>

???????????????????? <value>true</value>

???????? </property>

<name>dfs.permission</name>

<value>false</value>

</property>

</configuration>

[@tc_12_113 hadoop]# cat mapred-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
? Licensed under the Apache License, Version 2.0 (the "License");
? you may not use this file except in compliance with the License.
? You may obtain a copy of the License at

??? http://www.apache.org/licenses/LICENSE-2.0

??????????????? <property>

??????????????????????????????? <name>mapreduce.framework.name</name>

??????????????????????????????? <value>yarn</value>

??????????????? </property>

??????????????? <property>

??????????????????????????????? <name>mapreduce.jobhistory.address</name>

??????????????????????????????? <value>hadoopmaster:10020</value>

??????????????? </property>

??????????????? <property>

?????????????? <name>mapreduce.jobhistory.webapp.address</name>

?????????????? <value>hadoopmaster:19888</value>

?????? </property>

</configuration>

[@tc_12_113 hadoop]# cat yarn-site.xml
<?xml version="1.0"?>
<!--
? Licensed under the Apache License, Version 2.0 (the "License");
? you may not use this file except in compliance with the License.
? You may obtain a copy of the License at

??? http://www.apache.org/licenses/LICENSE-2.0

? <property>

?????????????? <name>yarn.nodemanager.aux-services</name>

?????????????? <value>mapreduce_shuffle</value>

??????? </property>

???????????????? <property>

?????????????? <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>

?????????????? <value>org.apache.hadoop.mapred.ShuffleHandler</value>

??????? </property>

??????? <property>

?????????????? <name>yarn.resourcemanager.address</name>