配置hadoop-优快云博客

本文链接：https://blog.youkuaiyun.com/qq_45382931/article/details/120323719

本文档详细介绍了如何使用xftp上传并解压Hadoop软件包，然后通过配置环境变量、创建目录以及编辑配置文件（如hadoop-env.sh、core-site.xml等）来设置Hadoop集群。配置包括设置HADOOP_HOME、JAVA_HOME，定义HDFS存储路径，配置YARN和MapReduce的相关参数。所有配置完成后，集群将能够在本地主机上运行。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

1. 上传

用xftp

2. 解压

tar -zvxf xxx.tar.gz

3. 配置

vi /etc/profile.d/hadoop-eco.sh
加入

HADOOP_HOME=/opt/hadoop
PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH

source /etc/profile.d/hadoop-eco.sh

4. 新建文件夹

mkdir hadoop-record
cd hadoop-record/
mkdir name secondary data tmp

修改yarn-env.sh
export JAVA_HOME=/opt/jdk
修改hadoop-env.sh
export JAVA_HOME=/opt/jdk
core-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
	<!-- NameNode节点的URI-->
	<property>
		<name>fs.defaultFS</name>
		<value>hdfs://localhost:9000</value>
	</property>
	
	<!-- 指定Hadoop运行产生文件的存储路径-->
	<property>
		<name>hadoop.tmp.dir</name>
		<value>file:///opt/hadoop-record/tmp</value>
	</property>
</configuration>

hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
	<property>
	<!-- 数据副本数量-->
		<name>dfs.replication</name>
		<value>1</value>
	</property>
	<property>
	<!--namenode数据存放地址-->
		<name>dfs.namenode.name.dir</name>
		<value>file:///opt/hadoop-record/name</value>
	</property>
	<property>
	<!--datanode数据存放地址-->
		<name>dfs.namenode.name.dir</name>
		<value>file:///opt/hadoop-record/data</value>
	</property>

</configuration>

mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
	<property>
	<!--mapreduce运行的平台，默认local -->
		<name>mapreduce.framework.name</name>
		<value>yarn</value>
	</property>

</configuration>

yarn-site.xml

<?xml version="1.0"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<configuration>

	<property>
	<!-- resourcemanager所在的机器 -->
		<name>yarn.resourcemanager.hostname</name>
		<value>localhost</value>
	</property>
		<property>
	<!-- 所需要的服务 -->
		<name>yarn.nodemanager.aux-services</name>
		<value>mapreduce_shuffle</value>
	</property>

</configuration>