Hadoop 搭建高可用完全分布式集群

部署规划

主机
用途
IP
ResourceManager01
192.168.137.11
NameNode01 DFSZKFailoverController
192.168.137.12
(backup resourcemanager)
ResourceManager02
192.168.137.13
(backup namenode)
NameNode02 DFSZKFailoverController
192.168.137.14
Dat a Node NodeManager QuorumPeerMain JournalNode
192.168.137.21
Dat a Node NodeManager QuorumPeerMain JournalNode
192.168.137.22
Dat a Node NodeManager QuorumPeerMain JournalNode
192.168.137.23

[hadoop@dn01 ~]$ tar -zxf /nfs_share/software/zookeeper-3.4.11.tar.gz -C ~
[hadoop@dn01 ~]$ vi .bashrc
export ZOOKEEPER_HOME=/home/hadoop/zookeeper-3.4.11
export PATH=$PATH:/home/hadoop/zookeeper-3.4.11/bin
[hadoop@dn01 ~]$ source .bashrc
[hadoop@dn01 ~]$ cd zookeeper-3.4.11/conf
[hadoop@dn01 conf]$ mv zoo_sample.cfg zoo.cfg
[hadoop@dn01 conf]$ vi zoo.cfg

dataLogDir=/home/hadoop/zookeeper-3.4.11/log
dataDir=/home/hadoop/zookeeper-3.4.11/data
server.1=192.168.137.21:2888:3888
server.2=192.168.137.22:2888:3888
server.3=192.168.137.23:2888:3888

[hadoop@dn01 conf]$ cd ..
[hadoop@dn01 zookeeper-3.4.11]$ mkdir data && mkdir log && cd data && echo "1">>myid
[hadoop@dn01 zookeeper-3.4.11]$ cd
[hadoop@dn01 ~]$ scp -r zookeeper-3.4.11 dn02.hadoop.com:/home/hadoop
[hadoop@dn01 ~]$ scp -r zookeeper-3.4.11 dn03.hadoop.com:/home/hadoop
[hadoop@dn01 ~]$ ssh hadoop@dn02.hadoop.com 'cd /home/hadoop/zookeeper-3.4.11/data && echo "2">myid'
[hadoop@dn01 ~]$ ssh hadoop@dn03.hadoop.com 'cd /home/hadoop/zookeeper-3.4.11/data && echo "3">myid'
[hadoop@dn01 ~]$ zkServer.sh start
[hadoop@dn02 ~]$ zkServer.sh start
[hadoop@dn03 ~]$ zkServer.sh start
[hadoop@dn01 ~]$ zkServer.sh status
[hadoop@dn02 ~]$ zkServer.sh status
[hadoop@dn03 ~]$ zkServer.sh status

[hadoop@dn01 ~]$ cd hadoop-2.9.0 && mkdir journal
[hadoop@dn02 ~]$ cd hadoop-2.9.0 && mkdir journal
[hadoop@dn03 ~]$ cd hadoop-2.9.0 && mkdir journal

[hadoop@nn01 ~]$ cd hadoop-2.9.0/etc/hadoop/
[hadoop@nn01 hadoop]$ vi core-site.xml

<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://ns1</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>file:/home/hadoop/hadoop-2.9.0/tmp</value>
    </property>
    <property>
        <name>io.file.buffer.size</name>
        <value>131072</value>
    </property>
    <property>
           <name>hadoop.proxyuser.hadoop.hosts</name>
           <value>*</value>
       </property>
    <property>
        <name>hadoop.proxyuser.hadoop.groups</name>
        <value>*</value>
    </property>
    <!--HDFS HA Configuration,HDFS联邦都不需要配置  -->
    <property>
        <name>ha.zookeeper.quorum</name>
        <value>dn01.hadoop.com:2181,dn02.hadoop.com:2181,dn03.hadoop.com:2181</value>
    </property>
    <property>
        <name>ha.zookeeper.session-timeout.ms</name>
        <value>1000</value>
    </property>
</configuration>

[hadoop@nn01 hadoop]$ vi hdfs-site.xml

<configuration>
    <property>
        <name>dfs.namenode.name.dir</name>
        <value>/home/hadoop/hadoop-2.9.0/dfs/name</value>
    </property>
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>/home/hadoop/hadoop-2.9.0/dfs/data</value>
    </property>
    <property>
        <name>dfs.blocksize</name>
        <value>64m</value>
    </property>
    <property>
        <name>dfs.replication</name>
        <value>3</value>
    </property>
    <property>
        <name>dfs.webhdfs.enabled</name>
        <value>true</value>
    </property>
    <property>
        <name>dfs.permissions</name>
         <value>false</value>
    </property>
    <property>
        <name>dfs.permissions.enabled</name>
         <value>false</value>
    </property>
    <!-- NameNode HA Configuation-->
    <property>
        <name>dfs.nameservices</name>
         <value>ns1</value> <!--可以指定多个命令空间来配置HDFS联邦-->
    </property>
    <!--ns1,并且无需在配置SecondaryNameNode了,standby的namenode同时扮演了这角色-->
    <property>
        <name>dfs.ha.namenodes.ns1</name>
         <value>nn1,nn2</value> <!--ns1中namenode的唯一标识号-->
    </property>
    <property>
        <name>dfs.namenode.rpc-address.ns1.nn1</name>
         <value> nn01.hadoop.com:8020</value>
    </property>    
    <property>
        <name>dfs.namenode.rpc-address.ns1.nn2</name>
         <value> nn02.hadoop.com:8020</value>
    </property>    
    <property>
        <name>dfs.namenode.servicerpc-address.ns1.nn1</name>
         <value> nn01.hadoop.com:53310</value>
    </property>    
    <property>
        <name>dfs.namenode.servicerpc-address.ns1.nn2</name>
         <value> nn02.hadoop.com:53310</value>
    </property>    
    <property>
        <name>dfs.namenode.http-address.ns1.nn1</name>
         <value> nn01.hadoop.com:50070</value>
    </property>    
    <property>
        <name>dfs.namenode.http-address.ns1.nn2</name>
         <value> nn02.hadoop.com:50070</value>
    </property>
    <property>
        <name>dfs.namenode.shared.edits.dir.ns1.nn1</name>
    </property>
    <property>
        <name>dfs.namenode.shared.edits.dir.ns1.nn2</name>
    </property>
    <property>
        <name>dfs.ha.automatic-failover.enabled.ns1</name>
        <value>true</value>
    </property>
    <property>
        <name>dfs.client.failover.proxy.provider.ns1</name>
        <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
    </property>
    <property>
        <name>dfs.ha.fencing.methods</name>
        <value>sshfence</value>
    </property>
    <property>
        <name>dfs.ha.fencing.ssh.private-key-files</name>
        <value>/home/hadoop/.ssh/id_rsa</value>
    </property>
    <property>
        <name>dfs.ha.fencing.ssh.connect-timeout</name>
        <value>30000</value>
    </property>
    <property>
        <name>dfs.journalnode.edits.dir</name>
        <value>/home/hadoop/hadoop-2.9.0/journal</value>
    </property>
    <property>
        <name>ha.failover-controller.cli-check.rpc-timeout.ms</name>
        <value>50000</value>
    </property>
    <property>
        <name>ipc.client.connect.timeout</name>
        <value>60000</value>
    </property>
    <property>
        <name>dfs.image.transfer.bandwidthPerSec</name>
        <value>4194304</value>
    </property>
</configuration>


[hadoop@nn01 hadoop]$ vi   mapred -site.xml

<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property>
        <name>mapreduce.jobhistory.address</name>
        <value>rm01.hadoop.com:10020</value>
    </property>
    <property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>rm01.hadoop.com:19888</value>
    </property>
</configuration>

[hadoop@nn01 hadoop]$ vi   yarn-site.xml

<configuration>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
       <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
       <value>org.apache.hadoop.mapred.ShuffleHandler</value>
    </property>
    <property>
        <name>yarn.resourcemanager.connect.retry-interval.ms</name>
        <value>2000</value>
    </property>
    <property>
        <name>yarn.resourcemanager.scheduler.class</name>
        <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
    </property>
    <property>
        <name>yarn.app.mapreduce.am.scheduler.connection.wait.interval-ms</name>
        <value>5000</value>
    </property>
    
  <!--ResourceManager Restart Configuration-->
  <property>
    <name>yarn.resourcemanager.recovery.enabled</name>
    <value>true</value>
  </property>
  <property>
    <name>yarn.resourcemanager.store.class</name>
    <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
  </property>
  <property>
    <name>yarn.resourcemanager.zk-address</name>
    <value>dn01.hadoop.com:2181,dn02.hadoop.com:2181,dn03.hadoop.com:2181</value>
  </property>
  <property>
    <name>yarn.resourcemanager.zk-state-store.parent-path</name>
    <value>/rmstore</value>
  </property>
  <property>
    <name>yarn.resourcemanager.zk-num-retries</name>
    <value>500</value>
  </property>
  <property>
    <name>yarn.resourcemanager.zk-retry-interval-ms</name>
    <value>2000</value>
  </property>
  <property>
    <name>yarn.resourcemanager.zk-timeout-ms</name>
    <value>10000</value>
  </property>
  <property>
    <name>yarn.resourcemanager.zk-acl</name>
    <value>world:anyone:rwcda</value>
  </property>
  <property>
    <name>yarn.resourcemanager.am.max-attempts</name>
    <value>2</value>
  </property>
  
  <!--ResourceManager HA Configuration-->
  <property>
    <name>yarn.resourcemanager.cluster-id</name>
    <value>yarn-cluster</value>
  </property>
  <property>
    <name>yarn.resourcemanager.ha.enabled</name>
    <value>true</value>
  </property>
  <property>
    <name>yarn.resourcemanager.ha.rm-ids</name>
    <value>rm1,rm2</value>
  </property>
  <property>
    <name>yarn.resourcemanager.hostname.rm1</name>
    <value>rm01.hadoop.com</value>
  </property>
  <property>
    <name>yarn.resourcemanager.hostname.rm2</name>
    <value>rm02.hadoop.com</value>
  </property>
  <property>
    <name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
    <value>true</value>
  </property>
  <property>
    <name>yarn.resourcemanager.ha.automatic-failover.embedded</name>
    <value>true</value>
  </property>
  
  <!-- rm1 Configuration-->
  <property>
    <name>yarn.resourcemanager.address.rm1</name>
    <value>rm01.hadoop.com:8032</value>
  </property>
  <property>
    <name>yarn.resourcemanager.scheduler.address.rm1</name>
    <value>rm01.hadoop.com:8030</value>
  </property>
  <property>
    <name>yarn.resourcemanager.admin.address.rm1</name>
    <value>rm01.hadoop.com:8033</value>
  </property>
  <property>
    <name>yarn.resourcemanager.resource-tracker.address.rm1</name>
    <value>rm01.hadoop.com:8031</value>
  </property>
  <property>
    <name>yarn.resourcemanager.webapp.address.rm1</name>
    <value>rm01.hadoop.com:8088</value>
  </property>
  <property>
    <name>yarn.resourcemanager.webapp.https.address.rm1</name>
    <value>rm01.hadoop.com:8090</value>
  </property>
  
  <!-- rm2 Configuration-->
  <property>
    <name>yarn.resourcemanager.address.rm2</name>
    <value>rm02.hadoop.com:8032</value>
  </property>
  <property>
    <name>yarn.resourcemanager.scheduler.address.rm2</name>
    <value>rm02.hadoop.com:8030</value>
  </property>
  <property>
    <name>yarn.resourcemanager.admin.address.rm2</name>
    <value>rm02.hadoop.com:8033</value>
  </property>
  <property>
    <name>yarn.resourcemanager.resource-tracker.address.rm2</name>
    <value>rm02.hadoop.com:8031</value>
  </property>
  <property>
    <name>yarn.resourcemanager.webapp.address.rm2</name>
    <value>rm02.hadoop.com:8088</value>
  </property>
  <property>
    <name>yarn.resourcemanager.webapp.https.address.rm2</name>
    <value>rm02.hadoop.com:8090</value>
  </property>
</configuration>

[hadoop@nn01 hadoop]$ vi   slaves


[hadoop@nn01 ~]$ hdfs zkfc -formatZK
启动journalnode节点用于namenode主备数据同步
[hadoop@dn01 ~]$ hadoop-daemon.sh start journalnode
[hadoop@dn02 ~]$ hadoop-daemon.sh start journalnode
[hadoop@dn03 ~]$ hadoop-daemon.sh start journalnode
启动主namenode
[hadoop@nn01 ~]$ hdfs namenode -format -clusterId c1
[hadoop@nn01 ~]$ hadoop-daemon.sh start namenode
启动备用namenode
[hadoop@nn02 ~]$ hdfs namenode -bootstrapStandby
[hadoop@nn02 ~]$ hadoop-daemon.sh start namenode
启动namenode故障转移程序
[hadoop@nn01 ~]$ hadoop-daemon.sh start zkfc
[hadoop@nn02 ~]$ hadoop-daemon.sh start zkfc
启动datanode
[hadoop@dn01 ~]$ hadoop-daemon.sh start datanode
[hadoop@dn02 ~]$ hadoop-daemon.sh start datanode
[hadoop@dn03 ~]$ hadoop-daemon.sh start datanode
启动主resoucemanager
[hadoop@rm01 ~]$ start-yarn.sh
启动备用resoucemanager
[hadoop@rm02 ~]$ yarn-daemon.sh start resourcemanager




HDFS HA 检验实验
[hadoop@nn01 ~]$ jps
2352 DFSZKFailoverController
2188 NameNode
3105 Jps
执行命令
[hadoop@nn01 ~]$ kill -9 2188
刷新页面,看到

说明切换成功。

ResourceManager HA 检验实验
[hadoop@rm01 ~]$ jps
1599 ResourceManager
1927 Jps
启动wordcount程序
kill掉主ResourceManager进程
[hadoop@rm01 ~]$ kill -9 1599
看控制台输出,可以看到备的ResourceManager被启用


说明切换成功。

<think>嗯,用户现在的问题是希望在Hadoop完全分布式集群搭建Kafka高可用集群。首先,需要理解用户的场景:用户可能正在搭建一个大数据处理平台,需要集Kafka实现实时数据流处理,同时要求高可用性。用户引用的资料里提到SSH免密登录、Hadoop集群配置、Zookeeper集群搭建等,说明用户已经有基本环境,但可能对Kafka集部分不太熟悉。用户的需求明确,但深层需求可能是确保Kafka在Hadoop集群上稳定运行,避免单点故障,并能够和现有组件如Hive、Spark等协同工作。用户提供的引用中有提到Zookeeper、Flink、HiveServer2等,推测用户可能计划构建包含流处理和批处理的综合系统,所以Kafka的高可用性至关重要。接下来需要确定用户的环境:从引用看,用户似乎已经部署了Hadoop集群(包括NameNode和ResourceManager的高可用),Zookeeper集群也已搭建(引用[2][3])。因此,步骤应该从验证Zookeeper状态开始,因为Kafka依赖Zookeeper进行协调和元数据管理。Kafka高可用集群的核心在于多Broker配置,每个Broker需要唯一的ID和相同的Zookeeper连接地址。配置文件中必须设置监听地址、日志存储路径、副本数等关键参数。分发配置文件到所有节点并启动Kafka时,需要注意启动顺序:先Zookeeper,后Kafka。用户可能遇到的常见问题包括端口冲突、磁盘权限不足、副本配置错误导致高可用失效。因此,准备在相关问题中补充这些运维知识点。最后,测试部分需要覆盖创建多分区Topic、生产者-消费者验证、模拟故障转移等场景,确保高可用机制生效。还需要考虑和现有Hadoop组件的兼容性,比如HDFS路径权限(如果Kafka日志存HDFS)、防火墙设置等。引用[4]提到Spark启动命令,用户可能有流计算需求,所以应该提醒后续集SparkStreaming或Flink的配置要点。</think> 以下是基于 Hadoop 完全分布式集群搭建 Kafka 高可用集群的详细步骤(需确保 Hadoop 集群、Zookeeper 集群及 SSH 免密登录已就绪): --- ### **一、前置条件** 1. **Hadoop 集群** - 确认 HDFS/YARN 高可用已启用(如 NameNode HA、ResourceManager HA)[^3] 2. **Zookeeper 集群** - 至少 3 节点(例如 `hadoop01,hadoop02,hadoop03:2181`)[^2][^5] - 验证状态:`zkServer.sh status` 3. **Java 环境** - 所有节点安装 JDK 8+,配置 `JAVA_HOME`(参考 `hadoop-env.sh` 设置)[^1][^3] 4. **SSH 免密登录** - 确保集群节点间可无密码互访[^1] --- ### **二、Kafka 集群搭建步骤** #### **1. 下载并分发 Kafka** ```bash # 在 hadoop01 操作 wget https://downloads.apache.org/kafka/3.6.1/kafka_2.13-3.6.1.tgz tar -zxvf kafka_2.13-3.6.1.tgz -C /usr/local/ scp -r /usr/local/kafka_2.13-3.6.1 hadoop02:/usr/local/ scp -r /usr/local/kafka_2.13-3.6.1 hadoop03:/usr/local/ ``` #### **2. 配置 Kafka 高可用** 编辑每台节点的 `server.properties`(路径:`$KAFKA_HOME/config/server.properties`): ```properties # 唯一 Broker ID(不同节点不同ID) broker.id=1 # hadoop01=1, hadoop02=2, hadoop03=3 # 监听地址(内网IP) listeners=PLAINTEXT://hadoop01:9092 # 日志存储目录(确保目录存在且可写) log.dirs=/data/kafka-logs # 建议使用独立磁盘 # Zookeeper 集群地址 zookeeper.connect=hadoop01:2181,hadoop02:2181,hadoop03:2181 # 高可用关键配置 default.replication.factor=3 # 副本数≥Broker数 min.insync.replicas=2 # 最小同步副本数 offsets.topic.replication.factor=3 transaction.state.log.replication.factor=3 ``` #### **3. 同步配置并设置环境变量** ```bash # 分发配置文件(以 hadoop01 为基准) scp /usr/local/kafka/config/server.properties hadoop02:/usr/local/kafka/config/ scp /usr/local/kafka/config/server.properties hadoop03:/usr/local/kafka/config/ # 所有节点添加环境变量 echo 'export KAFKA_HOME=/usr/local/kafka' >> /etc/profile echo 'export PATH=$PATH:$KAFKA_HOME/bin' >> /etc/profile source /etc/profile # [^4] ``` #### **4. 启动 Kafka 集群** ```bash # 所有节点依次启动(先启Zookeeper,再启Kafka) zkServer.sh start # 若Zookeeper未运行 kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties # 验证进程 jps | grep Kafka # 应看到 Kafka 进程 ``` #### **5. 测试高可用** ```bash # 创建 Topic(副本数=3) kafka-topics.sh --create \ --bootstrap-server hadoop01:9092 \ --replication-factor 3 \ --partitions 3 \ --topic test-ha # 查看 Topic 状态 kafka-topics.sh --describe \ --bootstrap-server hadoop01:9092 \ --topic test-ha # 模拟故障(关闭一个Broker后观察Leader切换) kafka-server-stop.sh # 在某个节点执行 ``` --- ### **三、关键配置说明** | **配置项** | **值示例** | **作用** | |------------|------------|----------| | `broker.id` | 1/2/3 | Broker 唯一标识 | | `zookeeper.connect` | `hadoop01:2181,...` | 集群协调服务地址 | | `default.replication.factor` | 3 | 数据默认副本数 | | `min.insync.replicas` | 2 | 最小同步副本数(保证数据安全) | | `log.dirs` | `/data/kafka-logs` | 高性能磁盘目录 | --- ### **四、常见问题** 1. **端口冲突** - 检查 `server.properties` 中的 `listeners` 端口是否被占用 2. **磁盘空间不足** - 监控 `/data/kafka-logs` 目录,设置日志清理策略: ```properties log.retention.hours=168 # 保留7天 log.retention.bytes=1073741824 # 1GB/分区 ``` 3. **副本不同步** - 确保 `min.insync.replicas < replication.factor` --- ### **相关配置参考** - **Zookeeper 集**:Zookeeper 管理 Kafka Broker 元数据与选举[^5] - **Hadoop 协作**:Kafka 数据可下沉至 HDFS(需配置 `connect-hdfs` 插件) - **安全加固**:建议启用 SASL/SSL 认证(生产环境必备)
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值