hadoop HA

修改配置文件core-site.xml、hdfs-site.xml、mapred-site.xml、yarn-site.xml、 hadoop-env.sh & yarn-env.sh、workers

1、core-site.xml
<configuration>
<property>
<name>hadoop.tmp.dir</name>
<value>file:/usr/hadoop/hadoop-3.3.0/hdfs/tmp</value>
<description>A base for other temporary directories.</description>
</property>
<property>
<name>io.file.buffer.size</name>
<value>131072</value>
</property>
<property>
<name>fs.defaultFS</name>
<value>hdfs://bigdatacluster-ha</value>
</property>
<!-- 指定zookeeper地址 -->
<property>
<name>ha.zookeeper.quorum</name>
<value>hd1:2181,hd2:2181,hd3:2181</value>
</property>
<property>
<name>ha.zookeeper.session-timeout.ms</name>
<value>300000</value>
</property>

<property>
<name>hadoop.proxyuser.root.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.root.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.devops.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.devops.groups</name>
<value>*</value>
</property>
<property>
<name>ipc.client.connect.timeout</name>
<value>90000</value>
</property>
</configuration>

2、hdfs-site.xml
<configuration>
<!--指定hdfs的nameservice为bigdatacluster-ha，需要和core-site.xml中的保持一致 -->
<property>
<name>dfs.nameservices</name>
<value>bigdatacluster-ha</value>
</property>

<!-- bigdatacluster-ha下面有两个NameNode，分别是namenode1，namenode2 -->
<property>
<name>dfs.ha.namenodes.bigdatacluster-ha</name>
<value>namenode1,namenode2</value>
</property>
<!-- namenode1的RPC通信地址,这里端口要和core-site.xml中fs.defaultFS保持一致 -->
<property>
<name>dfs.namenode.rpc-address.bigdatacluster-ha.namenode1</name>
<value>hd1:9010</value>
</property>
<!-- namenode1的http通信地址 -->
<property>
<name>dfs.namenode.http-address.bigdatacluster-ha.namenode1</name>
<value>hd1:50070</value>
</property>
<!-- namenode2的RPC通信地址,这里端口要和core-site.xml中fs.defaultFS保持一致 -->
<property>
<name>dfs.namenode.rpc-address.bigdatacluster-ha.namenode2</name>
<value>hd2:9010</value>
</property>
<!-- namenode2的http通信地址 -->
<property>
<name>dfs.namenode.http-address.bigdatacluster-ha.namenode2</name>
<value>hd2:50070</value>
</property>
<!-- 指定NameNode的元数据在JournalNode上的存放位置 -->
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://hd1:8485;hd2:8485;hd3:8485/bigdatacluster-ha</value>
</property>
<!-- 配置失败自动切换实现方式 -->
<property>
<name>dfs.client.failover.proxy.provider.bigdatacluster-ha</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<!-- 配置隔离机制,主要用户远程管理监听其他机器相关服务 -->
<property>
<name>dfs.ha.fencing.methods</name>
<value>
sshfence
shell(/bin/true)
</value>
</property>
<!-- 使用隔离机制时需要ssh免密码登陆 -->
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/root/.ssh/id_rsa</value>
</property>
<!-- 指定NameNode的元数据在JournalNode上的存放位置 -->
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/opt/hadoop/journal</value>
</property>
<!--指定支持高可用自动切换机制-->
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<!--指定namenode名称空间的存储地址-->
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/usr/hadoop/hadoop-3.3.0/hdfs/name</value>
</property>
<!--指定datanode数据存储地址-->
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/usr/hadoop/hadoop-3.3.0/hdfs/data</value>
</property>
<!--指定数据冗余份数-->
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<!--指定可以通过web访问hdfs目录-->
<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>hd1:2181,hd2:2181,hd3:2181</value>
</property>
<property>
<name>dfs.namenode.handler.count</name>
<value>600</value>
<description>The number of server threads for the namenode.</description>
</property>
<property>
<name>dfs.datanode.handler.count</name>
<value>600</value>
<description>The number of server threads for the datanode.</description>
</property>
<property>
<name>dfs.client.socket-timeout</name>
<value>600000</value>
</property>
<property>
<!--这里设置Hadoop允许打开最大文件数，默认4096，不设置的话会提示xcievers exceeded错误-->
<name>dfs.datanode.max.transfer.threads</name>
<value>409600</value>
</property>

<!--true 要开启权限检查-->
<property>
<name>dfs.permissions</name>
<value>true</value>
</property>
<property>
<name>dfs.qjournal.start-segment.timeout.ms</name>
<value>90000</value>
</property>
<property>
<name>dfs.qjournal.select-input-streams.timeout.ms</name>
<value>90000</value>
</property>
<property>
<name>dfs.qjournal.write-txns.timeout.ms</name>
<value>90000</value>
</property>
</configuration>

3、mapred-site.xml
<configuration>
<property>
<name>mapreduce.map.memory.mb</name>
<value>1024</value>
</property>
<property>
<name>mapreduce.map.java.opts</name>
<value>-Xmx300M</value>
</property>
<property>
<name>mapreduce.reduce.memory.mb</name>
<value>1536</value>
</property>
<property>
<name>mapreduce.reduce.java.opts</name>
<value>-Xmx450M</value>
</property>

<!-- 配置MapReduce运行于yarn中 -->
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<property>
<name>mapreduce.job.maps</name>
<value>12</value>
</property>
<property>
<name>mapreduce.job.reduces</name>
<value>12</value>
</property>
<property>
<name>mapreduce.jobhistory.address</name>
<value>hd2:10020</value>
</property>
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>hd2:19888</value>
</property>

<property>
<name>yarn.app.mapreduce.am.env</name>
<value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>
</property>
<property>
<name>mapreduce.map.env</name>
<value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>
</property>
<property>
<name>mapreduce.reduce.env</name>
<value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>
</property>
<!-- 下面暂时全部注释掉：指定Hadoop压缩格式，Apache官网下载的安装包不支持snappy，需要自己编译安装，如何编译安装参考：http://aperise.iteye.com/blog/2254487有讲解,不适用snappy的话可以不配置
<property>
<name>mapreduce.output.fileoutputformat.compress</name>
<value>true</value>
<description>Should the job outputs be compressed?
</description>
</property>
<property>
<name>mapreduce.output.fileoutputformat.compress.type</name>
<value>RECORD</value>
<description>If the job outputs are to compressed as SequenceFiles, how should
they be compressed? Should be one of NONE, RECORD or BLOCK.
</description>
</property>
<property>
<name>mapreduce.output.fileoutputformat.compress.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description>If the job outputs are compressed, how should they be compressed?
</description>
</property>
<property>
<name>mapreduce.map.output.compress</name>
<value>true</value>
<description>Should the outputs of the maps be compressed before being
sent across the network. Uses SequenceFile compression.
</description>
</property>
<property>
<name>mapreduce.map.output.compress.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description>If the map outputs are compressed, how should they be
compressed?
</description>
</property>
-->
</configuration>

4、yarn-site.xml
<configuration>
<!--日志聚合功能yarn.log start-->
<property>
<name>yarn.log-aggregation-enable</name>
<value>true</value>
</property>
<!--在HDFS上聚合的日志最长保留多少秒。3天-->
<property>
<name>yarn.log-aggregation.retain-seconds</name>
<value>259200</value>
</property>
<!--日志聚合功能yarn.log end-->
<!--resourcemanager失联后重新链接的时间-->
<property>
<name>yarn.resourcemanager.connect.retry-interval.ms</name>
<value>2000</value>
</property>
<!--配置resourcemanager start-->
<property>
<name>yarn.resourcemanager.zk-address</name>
<value>hd1:2181,hd2:2181,hd3:2181</value>
</property>
<property>
<name>yarn.resourcemanager.cluster-id</name>
<value>besttonecluster-yarn</value>
</property>
<!--开启resourcemanager HA,默认为false-->
<property>
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</property>
<property>
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm1</name>
<value>hd1</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm2</name>
<value>hd2</value>
</property>
<!--配置rm1-->
<property>
<name>yarn.resourcemanager.webapp.address.rm1</name>
<value>hd1:18088</value>
</property>
<!--配置rm2-->
<property>
<name>yarn.resourcemanager.webapp.address.rm2</name>
<value>hd2:18088</value>
</property>
<!--开启故障自动切换-->
<property>
<name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<property>
<name>yarn.resourcemanager.ha.automatic-failover.embedded</name>
<value>true</value>
</property>
<property>
<name>yarn.resourcemanager.ha.automatic-failover.zk-base-path</name>
<value>/yarn-leader-election</value>
</property>
<!--开启自动恢复功能-->
<property>
<name>yarn.resourcemanager.recovery.enabled</name>
<value>true</value>
</property>
<property>
<name>yarn.resourcemanager.store.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
</property>
<!--配置resourcemanager end->
<!-配置nodemanager start-->
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<!--配置nodemanager end-->

<!-- <property>
<name>yarn.resourcemanager.address</name>
<value>hd1:18040</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>hd1:18030</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>hd1:18088</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>hd1:18025</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address</name>
<value>hd1:18141</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.auxservices.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property> -->
</configuration>

5、 hadoop-env.sh & yarn-env.sh
export JAVA_HOME=/usr/java/jdk1.8.0_201-amd64

6、workers
hd1
hd2
hd3

7、启动zk
#格式化 hdfs zkfc -formatZK
#验证：检查zookeeper上是否已经有Hadoop HA目录 $ZOOKEEPER_HOME/bin/zkCli.sh -server hd1:2181,hd2:2181,hd3:2181

8、启动namenode日志同步服务journalnode
所有ZooKeeper节点均启动
#当前节点 $HADOOP_HOME/sbin/hadoop-daemon.sh start journalnode #其他节点 ssh hd2 $HADOOP_HOME/sbin/hadoop-daemon.sh start journalnode ssh hd3 $HADOOP_HOME/sbin/hadoop-daemon.sh start journalnode

9、Hadoop集群格式化并启动
在主namenode节点格式化NAMENODE
hdfs namenode -format

在主namenode节点启动namenode服务
hadoop-daemon.sh start namenode

在备namenode节点同步元数据（此前一定要先启动主namenode) 并启动namenode 服务
#同步元数据 ssh hd2 $HADOOP_HOME/bin/hdfs namenode -bootstrapStandby

#备NameNode节点启动namenode服务 ssh hd2 $HADOOP_HOME/sbin/hadoop-daemon.sh start namenode

在所有namenode节点上启动DFSZKFailoverController
$HADOOP_HOME/sbin/hadoop-daemon.sh start zkfc ssh hd2 $HADOOP_HOME/sbin/hadoop-daemon.sh start zkfc

启动datanode服务
#注意hadoop-daemons.sh datanode是启动所有datanode，而hadoop-daemon.sh datanode是启动单个datanode
$HADOOP_HOME/sbin/hadoop-daemons.sh start datanode

启动yarn
#主Namenode节点hd1启动resourcemanager，hd3启动nodemanager hd1:$HADOOP_HOME/sbin/start-yarn.sh #备Namenode节点hdp-02启动resourcemanager hd1:ssh hd2 $HADOOP_HOME/sbin/yarn-daemon.sh start resourcemanager

10、查看namenode情况

主 http://hd1:50070/
备 http://hd2:50070/

 

11、在hd1使用jps查看进程，kill namenode进程，在备用节点查看，http://hd2:50070/ standby状态切换为active。