一：hadoop3.1.3离线安装_hdfs离线安装教程-优快云博客

本文链接：https://blog.youkuaiyun.com/csdn_gyb/article/details/120719026

环境初始化

前提准备
192.168.100.29 hadoop01
192.168.100.28 hadoop02
192.168.100.27 hadoop03
192.168.100.26 hadoop04
修改ip和主机名
#修改ip
vi /etc/sysconfig/network-scripts/ifcfg-ens192
IPADDR=“192.168.100.151”
#重启网卡服务
systemctl restart network
#修改主机名(新增的服务器)
hostnamectl set-hostname server19
#安装常用的基础工具
yum -y install sudo
yum install -y vim
#yum -y install lrzsz
yum -y install wget
yum -y install net-tools
#主节点编辑host文件
vim /etc/hosts
192.168.100.29 hadoop01 hadoop01
192.168.100.28 hadoop02 hadoop02
192.168.100.27 hadoop03 hadoop03
192.168.100.26 hadoop04 hadoop04
#配置免密
#集群服务器
cd /root/
ssh-keygen -t rsa
#然后三个回车
#复制主节点公钥到扩展的节点机器上(主节点上执行)
#ssh-copy-id -i /root/.ssh/id_rsa.pub root@ip
#主节点执行
ssh-copy-id -i /root/.ssh/id_rsa.pub root@hadoop01
ssh-copy-id -i /root/.ssh/id_rsa.pub root@hadoop02
ssh-copy-id -i /root/.ssh/id_rsa.pub root@hadoop03
#主节点上传文件分发脚本
资源如下
https://download.youkuaiyun.com/download/csdn_gyb/31849299
#主节点执行同步
sh /home/scp.sh /etc/hosts
#上传安装包到/tmp路径下
#关闭防火墙并查看状态(集群服务器)
systemctl stop firewalld
systemctl status firewalld

#安装ntp服务(所有服务器)为了集群时间同步
yum -y install ntp
service ntpd start
systemctl enable ntpd.service
#查看ntp服务状态
service ntpd status
#设置时间同步(从主节点同步)

安装mysql和jdk

#卸载清除主节点mysql
#先检查是否安装有mysql
rpm -qa|grep -i mysql
#停止mysql服务
systemctl stop mysqld
#删除软件命令
rpm -e –nodeps 包名
#如果上面命令提示依赖包错误，则使用以下命令尝试：
rpm -ev 包名 --nodeps
#查找卸载残留的mysql的目录
find / -name mysql #然后使用 rm -rf 将查询到的目录全部删除
#删除老版本mysql的配置文件
rm -rf /etc/my.cnf
rm -rf /etc/my.cnf.d
#mysql安装(见子节点)暂时先不安装
#安装jdk
#解压jdk到指定目录
tar -zxvf /tmp/jdk-8u181-linux-x64.tar.gz -C /usr/share/
对应路径如下(配置环境变量要用到该路径)
/usr/share/jdk1.8.0_181
#然后配置jdk环境变量(见子节点)
#然后把jdk目录复制到其他节点
sh /home/scp.sh /usr/share/jdk1.8.0_181
sh /home/scp.sh /etc/profile
安装hadoop及其配置

安装配置hadoop

#解压hadoop到指定路径
tar -zxvf /tmp/hadoop-3.1.3.tar.gz -C /home/
#到hadoop目录下建立目录
cd /home/hadoop-3.1.3
mkdir tmp
mkdir hdfs
mkdir hdfs/data
mkdir hdfs/name
#修改对应的配置文件(见子节点)
/home/hadoop-3.1.3/etc/hadoop/core-site.xml
/home/hadoop-3.1.3/etc/hadoop/hdfs-site.xml
/home/hadoop-3.1.3/etc/hadoop/yarn-site.xml
/home/hadoop-3.1.3/etc/hadoop/mapred-site.xml
/home/hadoop-3.1.3/etc/hadoop/yarn-env.sh
#配置workers
vim /home/hadoop-3.1.3/etc/hadoop/workers
#加入集群主机名称
hadoop01
hadoop02
hadoop03
hadoop04
#该文件夹分发到集群
sh scp.sh /home/hadoop-3.1.3/
#配置hadoop的环境变量
vi /etc/profile
文件末尾添加
export HADOOP_HOME=/home/hadoop-3.1.3
export PATH= $P A T H :$ HADOOP_HOME/bin:$HADOOP_HOME/sbin
source /etc/profile
#修改hadoop-3.1.3/sbin/目录下的start-dfs.sh、start-yarn.sh文件，在文件头添加如下用户设置(对应的stop-dfs.sh、stop-yarn.sh也要修改，此处为hadoop3的特殊配置处，不修改，启动失败)。
start-dfs.sh
HDFS_DATANODE_USER=root
HDFS_DATANODE_SECURE_USER=hdfs
HDFS_NAMENODE_USER=root
HDFS_SECONDARYNAMENODE_USER=root
start-yarn.sh

YARN_RESOURCEMANAGER_USER=root
HADOOP_SECURE_DN_USER=yarn
YARN_NODEMANAGER_USER=root
对应配置如下
core-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://hadoop04:9000</value>
    </property>
    <property>
        <name>io.file.buffer.size</name>
        <value>131072</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/home/hadoop-3.1.3/tmp</value>
    </property>
    <property>
        <name>hadoop.proxyuser.root.hosts</name>
        <value>*</value>
    </property>
    <property>
        <name>hadoop.proxyuser.root.groups</name>
        <value>*</value>
    </property>
    <property>
        <name>hadoop.http.staticuser.user</name>
        <value>root</value>
    </property>
</configuration>

hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
   <property>
        <name>dfs.namenode.name.dir</name>
        <value>/home/hadoop-3.1.3/hdfs/name</value>
    </property>
	
   <property>
        <name>dfs.datanode.data.dir</name>
        <value>/home/hadoop-3.1.3/hdfs/data</value>
    </property>
	
    <property>
        <name>dfs.replication</name>
        <value>3</value>
    </property>
	
    <property>
        <name>dfs.webhdfs.enabled</name>
        <value>true</value>
    </property>

    <property>
        <name>dfs.namenode.http-address</name>
        <value>hadoop04:9870</value>
    </property>

    <property>
        <name>dfs.namenode.secondary.http-address</name>
        <value>hadoop04:9001</value>
    </property>
</configuration>

yarn-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
         <name>yarn.resourcemanager.webapp.address</name>
         <value>hadoop04:8088</value>
    </property>
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>hadoop04</value>
    </property>
<!-- 开启yarn.webapp.ui2 -->
  <property>
    <description>To enable RM web ui2 application.</description>
    <name>yarn.webapp.ui2.enable</name>
    <value>true</value>
  </property>
    <property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
    </property>
    <property>
        <name>yarn.scheduler.minimum-allocation-mb</name>
        <value>512</value>
    </property>
    <property>
        <name>yarn.scheduler.maximum-allocation-mb</name>
        <value>4096</value>
    </property>
    <property>
        <name>yarn.nodemanager.resource.memory-mb</name>
        <value>4096</value>
    </property>
    <property>
        <name>yarn.nodemanager.pmem-check-enabled</name>
        <value>false</value>
    </property>
    <property>
        <name>yarn.nodemanager.vmem-check-enabled</name>
        <value>false</value>
    </property>
</configuration>

mapred-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
</configuration>

yarn-env.sh

export JAVA_HOME=/usr/share/jdk1.8.0_181

hadoop-env.sh

export JAVA_HOME=/usr/share/jdk1.8.0_181
export HADOOP_HOME=/home/hadoop-3.1.3

start-dfs.sh，stop-dfs.sh&start-yarn.sh，stop-yarn.sh

在/hadoop/sbin路径下：
将start-dfs.sh，stop-dfs.sh两个文件顶部添加以下参数

#!/usr/bin/env bash
HDFS_DATANODE_USER=root
HADOOP_SECURE_DN_USER=hdfs
HDFS_NAMENODE_USER=root
HDFS_SECONDARYNAMENODE_USER=root

还有，start-yarn.sh，stop-yarn.sh顶部也需添加以下：
#!/usr/bin/env bash
YARN_RESOURCEMANAGER_USER=root
HADOOP_SECURE_DN_USER=yarn
YARN_NODEMANAGER_USER=root