Hadoop高可用集群搭建

1. 配置免密登录

    1.ssh-keygen 生成秘钥(文件在目录/root/.ssh下)

     2.将公钥追加到cat authorized_keys >> authorized_keys

     3.将需要相互登录的机器的公钥发送给各设备,并追加  cat authorized_keys >> authorized_keys

2.集群规划 

   node1: JournalNode NameNode  ResourceManager QuorumPeerMain DataNode DFSZKFailoverController

   node2: JournalNode NameNode  ResourceManager QuorumPeerMain DataNode DFSZKFailoverController

   node3: QuorumPeerMain  JournalNode  DataNode

3.配置文件

  hdfs-site.xml

 

 

<configuration>
<!--指定hdfs的nameservice为bi，需要和core-site.xml中的保持一致 -->
<property>
<name>dfs.nameservices</name>
<value>xxy</value>
</property>
<!-- bi下面有两个NameNode，分别是nn1，nn2 -->
<property>
<name>dfs.ha.namenodes.xxy</name>
<value>nn1,nn2</value>
</property>
<!-- nn1的RPC通信地址 -->
<property>
<name>dfs.namenode.rpc-address.xxy.nn1</name>
<value>hadoop1:9000</value>
</property>
<!-- nn1的http通信地址 -->
<property>
<name>dfs.namenode.http-address.xxy.nn1</name>
<value>hadoop1:50070</value>
</property>
<!-- nn2的RPC通信地址 -->
<property>
<name>dfs.namenode.rpc-address.xxy.nn2</name>
<value>hadoop2:9000</value>
</property>
<!-- nn2的http通信地址 -->
<property>
<name>dfs.namenode.http-address.xxy.nn2</name>
<value>hadoop2:50070</value>
</property>
<!-- 指定NameNode的edits元数据在JournalNode上的存放位置 -->
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://hadoop1:8485;hadoop2:8485;hadoop3:8485/xxy</value>
</property>

<!-- 指定datanode存放位置 -->
<property>
<name>dfs.datanode.data.dir</name>
<value>file:///opt/apps/data/dfs/data</value>
</property>
<!-- 指定JournalNode在本地磁盘存放数据的位置 -->
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/home/hadoop/apps/journaldata</value>
</property>
<!-- 开启NameNode失败自动切换 -->
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<!-- 配置失败自动切换实现方式 -->
<property>
<name>dfs.client.failover.proxy.provider.xxy</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<!-- 配置隔离机制方法，多个机制用换行分割，即每个机制暂用一行-->
<property>
<name>dfs.ha.fencing.methods</name>
<value>
sshfence
shell(/bin/true)
</value>
</property>
<!-- 使用sshfence隔离机制时需要ssh免登陆 -->
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/home/hadoop/.ssh/id_rsa</value>
</property>
<!-- 配置sshfence隔离机制超时时间 -->
<property>
<name>dfs.ha.fencing.ssh.connect-timeout</name>
<value>30000</value>
</property>
</configuration>

core-site.xml

<configuration>
<!-- 指定hdfs的nameservice为ns1 -->
<property>
<name>fs.defaultFS</name>
<value>hdfs://xxy/</value>
</property>
<!-- 指定hadoop临时目录 -->
<property>
<name>hadoop.tmp.dir</name>
<value>/home/hadoop/apps/hdpdata/</value>
</property>

<!-- 指定zookeeper地址 -->
<property>
<name>ha.zookeeper.quorum</name>
<value>hadoop1:2181,hadoop2:2181,hadoop3:2181</value>
</property>

</configuration>

mapred-site.xml

<configuration>
<!-- 指定mr框架为yarn方式 -->
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
</configuration>

yarn-site.xml

<configuration>
<!-- 开启RM高可用 -->
<property>
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</property>
<!-- 指定RM的cluster id -->
<property>
<name>yarn.resourcemanager.cluster-id</name>
<value>yrc</value>
</property>
<!-- 指定RM的名字 -->
<property>
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2</value>
</property>
<!-- 分别指定RM的地址 -->
<property>
<name>yarn.resourcemanager.hostname.rm1</name>
<value>hadoop1</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm2</name>
<value>hadoop2</value>
</property>
<!-- 指定zk集群地址 -->
<property>
<name>yarn.resourcemanager.zk-address</name>
<value>hadoop1:2181,hadoop2:2181,hadoop3:2181</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
</configuration>

slaves

hadoop1
hadoop2
hadoop3

host文件

192.168.12.126 hadoop1
192.168.12.127 hadoop2
192.168.12.128 hadoop3

/etc/profile

 

export JAVA_HOME=/opt/apps/software/jdk1.8.0_201
export ZK_HOME=/opt/apps/software/zookeeper-3.4.6
export HADOOP_HOME=/opt/apps/software/hadoop-2.7.3
export PATH=$JAVA_HOME/bin:$ZK_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH
export TZ=Asia/Shanghai
export CLASSPATH=.:/opt/apps/software/jdk1.8.0_201/lib/dt.jar:/opt/apps/software/jdk1.8.0_201/lib/tools.jar

zk安装

zoo.cfg增加如下

#数据目录与日志目录

dataDir=/opt/apps/data/zk/data
dataLogDir=/opt/apps/data/zk/logs
server.1=hadoop1:2888:3888
server.2=hadoop2:2888:3888
server.3=hadoop3:2888:3888

在每台设备日志目录下增加myid并赋值,数值与zoo.cfg文件数字一样

echo 1 >> /opt/apps/data/zk/data/myid

端口开放(2888 3888 50070 8088 8545)

内网开发所有端口

firewall-cmd --permanent --add-rich-rule='rule family=ipv4 source address=192.168.12.0/24 accept'

firewall-cmd --reload

查看开发端口 firewall-cmd --list-ports

端口开放示例:firewall-cmd --zone=public --add-port=80/tcp --permanent

–zone #作用域

–add-port=80/tcp #添加端口，格式为：端口/通讯协议

–permanent #永久生效，没有此参数重启后失效

重启防火墙

firewall-cmd --reload #重启

firewall systemctl stop firewalld.service #停止

firewall systemctl disable firewalld.service #禁止firewall开机启动

问题汇总

jdk为32位但linux为64 报错  /lib/ld-linux.so.2: bad ELF interpreter: No such file or directory

解决办法:yum install glibc.i686

端口连接refuse:

已经开放端口,但端口未监听

noroute to 端口:

端口未开放

 

队列设置

yarn-site.xml

增加(指定调度规则)

<property>
<name>yarn.resourcemanager.scheduler.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
</property>

 

capacity-scheduler.xml

<configuration>

  <property>
   <name>yarn.scheduler.capacity.root.queues</name>
   <value>default,hive,spark</value>
   <description>The queues at the this level (root is the root queue).</description>
  </property>
  
    <property>
    <name>yarn.scheduler.capacity.root.default.capacity</name>
    <value>40</value>
    <description>Default queue target capacity.</description>
  </property>
  
      <property>
    <name>yarn.scheduler.capacity.root.hive.capacity</name>
    <value>30</value>
    <description>Default queue target capacity.</description>
  </property>
  
        <property>
    <name>yarn.scheduler.capacity.root.spark.capacity</name>
    <value>30</value>
    <description>Default queue target capacity.</description>
  </property>
</configuration>