二、HADOOP入门(部署快速总结)

HADOOP入门部署总结:

--IP设置的三个配置文件
vim /etc/sysconfig/networt-scripts/ifcfg-ens33  --IP地址
vim /etc/hostname  --主机名称
vim /etc/hosts  --映射关系

BOOTPROTO="dhcp"    --动态IP地址
BOOTPROTO="static"  --静态IP地址
IPADDR=192.168.10.105  --IP地址
GATEWAY=192.168.10.2   --网关
DNS1=192.168.10.2      --域名解释器

--验证IP
ifconfig
ping www.baidu.com
hostname

--权限配置
vim /etc/sudoers

--下载软件包
yum install -y epel-release

--关闭防火墙
systemctl stop firewalld
systemctl disable firewalld.service

--分发脚本
scp -r --全量
rsync  --更新
xsync  --分发

--配置JDK、HADOOP
环境变量 my_env.sh 
cd /home/atguigu/bin
source /etc/profile

#JAVA_HOME
export JAVA_HOME=/opt/module/jdk1.8.0_212
export PATH=$PATH:$JAVA_HOME/bin

#HADOOP_HOME                                   
export HADOOP_HOME=/opt/module/hadoop-3.1.3
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin

--配置公钥/私钥
cd /home/atguigu/.ssh/
ssh-keygen -t rsa
--发送公钥
ssh-copy-id hadoop103

--配置集群
+ - - - + - - - - - - - - + - - - - - - - - + - - - - - - - - +
         hadoop102          hadoop103         hadoop104
+ - - - + - - - - - - - - + - - - - - - - - + - - - - - - - - +
  HDFS  :NameNode                             SecondaryNameNode
         DataNode           DataNode          DataNode
+ - - - + - - - - - - - - + - - - - - - - - + - - - - - - - - +
  YARN  :                   ResourceManager   
         NodeManager        NodeManager       NodeManager
+ - - - + - - - - - - - - + - - - - - - - - + - - - - - - - - +
+ - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - +
要获取的默认文件      文件存放在 Hadoop 的 jar 包中的位置
+ - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - +
[core-default.xml]    hadoop-common-3.1.3.jar/core-default.xml
[hdfs-default.xml]    hadoop-hdfs-3.1.3.jar/hdfs-default.xml
[yarn-default.xml]    hadoop-yarn-common-3.1.3.jar/yarn-default.xml
[mapred-default.xml]  hadoop-mapreduce-client-core-3.1.3.jar/mapred-default.xml
+ - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - +

--[core-site.xml]  
<configuration>
<!-- 指定 NameNode 的地址 -->
 <property>
 <name>fs.defaultFS</name>
 <value>hdfs://hadoop102:8020</value>
 </property>
 
 <!-- 指定 hadoop 数据的存储目录 -->
 <property>
 <name>hadoop.tmp.dir</name>
 <value>/opt/module/hadoop-3.1.3/data</value>
 </property>

 <!-- 配置 HDFS 网页登录使用的静态用户为 atguigu -->
 <property>
 <name>hadoop.http.staticuser.user</name>
 <value>atguigu</value>
 </property>

<configuration>

--[hdfs-site.xml]
<configuration>
<!-- nn web 端访问地址-->
<property>
 <name>dfs.namenode.http-address</name>
 <value>hadoop102:9870</value>
 </property>
 
<!-- 2nn web 端访问地址-->
 <property>
 <name>dfs.namenode.secondary.http-address</name>
 <value>hadoop104:9868</value>
 </property>
 
</configuration>

--[yarn-site.xml]
</configuration>
 <!-- 指定 MR 走 shuffle -->
 <property>
 <name>yarn.nodemanager.aux-services</name>
 <value>mapreduce_shuffle</value>
 </property>
 
 <!-- 指定 ResourceManager 的地址-->
 <property>
 <name>yarn.resourcemanager.hostname</name>
 <value>hadoop103</value>
 </property>
 
 <!-- 环境变量的继承 -->
 <property>
 <name>yarn.nodemanager.env-whitelist</name>
<value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CO
NF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAP
RED_HOME</value>
 </property>
 
<!-- 开启日志聚集功能 -->
<property>
 <name>yarn.log-aggregation-enable</name>
 <value>true</value>
</property>

<!-- 设置日志聚集服务器地址 -->
<property> 
 <name>yarn.log.server.url</name> 
 <value>http://hadoop102:19888/jobhistory/logs</value>
</property>

<!-- 设置日志保留时间为 7 天 -->
<property>
 <name>yarn.log-aggregation.retain-seconds</name>
 <value>604800</value>
</property>

</configuration>

--[mapred-site.xml]
<configuration>

<!-- 指定 MapReduce 程序运行在 Yarn 上 -->
 <property>
 <name>mapreduce.framework.name</name>
 <value>yarn</value>
 </property>
 
 <property>
    <name>yarn.app.mapreduce.am.env</name>
    <value>HADOOP_MAPRED_HOME=/opt/module/hadoop-3.1.3</value>
</property>
<property>
    <name>mapreduce.map.env</name>
    <value>HADOOP_MAPRED_HOME=/opt/module/hadoop-3.1.3</value>
</property>
<property>
    <name>mapreduce.reduce.env</name>
    <value>HADOOP_MAPRED_HOME=/opt/module/hadoop-3.1.3</value>
</property>

<!-- 历史服务器端地址 -->
<property>
 <name>mapreduce.jobhistory.address</name>
 <value>hadoop102:10020</value>
</property>

<!-- 历史服务器 web 端地址 -->
<property>
 <name>mapreduce.jobhistory.webapp.address</name>
 <value>hadoop102:19888</value>
</property>

</configuration>


--配置workers
hadoop102
hadoop103
hadoop104

--首次启动需要进行初始化
cd /opt/module/hadoop-3.1.3  
hdfs namenode -format        --多了data logs

--启动集群
sbin/start-dfs.sh
sbin/stop-dfs.sh

--启动ResourceManager
sbin/start-yarn.sh
sbin/stop-yarn.sh

http://hadoop105:9870/dfshealth.html#tab-overview
http://hadoop106:8088/cluster

hadoop fs -mkdir /wcinput  --创建HDFS文件夹
hadoop fs -put wcinput/word.txt /wcinput  --把本地文件上传到HDFS
hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-3.1.3.jar wordcount /wcinput /wcoutput --计算

--启动历史服务器
bin/mapred --daemon start historyserver
bin/mapred --daemon stop historyserver

--计算/为了查看历史服务
hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-3.1.3.jar wordcount /input /output

--单点启动
hdfs --daemon start namenode
hdfs --daemon start datanode
hdfs --daemon start secondarynamenode
yarn --daemon start resourcemanager
yarn --daemon start nodemanager

start-dfs.sh
start-yarn.sh