jobtracker更换内存方案

最新推荐文章于 2021-08-26 17:40:23 发布

原创最新推荐文章于 2021-08-26 17:40:23 发布 · 507 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#hadoop

hadoop 专栏收录该内容

47 篇文章

订阅专栏

本文详细介绍了如何将一台新的服务器（备份机）加入到现有的Hadoop集群中，并确保该节点能够正常工作。主要内容包括配置主机文件、SSH密钥授权、分发软件包、创建必要的目录结构、启动和停止相关服务等。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

使用10.180.92.198作为备份机
修改/etc/hosts.allow添加nn1.
把/root/.ssh/authorized_keys中添加nn1的root的id_rsa.pub

[root@hadoopNN1 ~]# cd /home/hadoop/setup
echo "10.180.92.198" > newslaves
./upgrade.sh distribute newslaves /etc/hosts /etc/
./upgrade.sh distribute newslaves /root/.ssh/id_rsa /root/.ssh/
./upgrade.sh distribute newslaves /root/.ssh/id_rsa.pub /root/.ssh/

./upgrade.sh distribute newslaves hadoop-1.1.2.tar.gz /tmp
./upgrade.sh distribute newslaves jdk1.7.0_51.tar.gz /tmp
./upgrade.sh distribute newslaves lzo-2.06.tar.gz /tmp
./upgrade.sh distribute newslaves init.sh /tmp
./upgrade.sh distribute newslaves letv.conf /tmp

./upgrade.sh common newslaves mkdir /letv/slot0
./upgrade.sh common newslaves mkdir /letv/slot1
./upgrade.sh common newslaves mkdir /letv/slot2
./upgrade.sh common newslaves mkdir /letv/slot3
./upgrade.sh common newslaves mkdir /letv/slot4
./upgrade.sh common newslaves mkdir /letv/slot5
./upgrade.sh common newslaves mkdir /letv/slot6
./upgrade.sh common newslaves mkdir /letv/slot7
./upgrade.sh common newslaves mkdir /letv/slot8
./upgrade.sh common newslaves mkdir /letv/slot9
./upgrade.sh common newslaves mkdir /letv/slota
./upgrade.sh common newslaves mkdir /letv/slotb

./upgrade.sh common newslaves sh /tmp/init.sh
./upgrade.sh common newslaves sh /tmp/setup.sh

su - hadoop
cd ~/setup
./upgrade.sh distribute newslaves /data/hadoop/data1/usr/local/setup /data/hadoop/data1/usr/local/
./upgrade.sh distribute newslaves /data/hadoop/data1/usr/local/setupKafka /data/hadoop/data1/usr/local/
./upgrade.sh distribute newslaves /data/hadoop/data1/usr/local/setupSpark /data/hadoop/data1/usr/local/
./upgrade.sh distribute newslaves /usr/local/hadoop/conf /usr/local/hadoop/

＃看http://10.200.91.164:60010/master-status ，确保当前结点成为master
一、停止相关服务
到hadoopNN1
停掉crontab
head -n 107 conf/slaves > conf/slaves1
tail -n 214 conf/slaves > conf/slavest
head -n 107 conf/slavest > conf/slaves2
tail -n 107 conf/slavest > conf/slaves3
sh bin/slave_cmd1_nocheck.sh "ls" >logs/cmd1.log 2>&1 &

sh bin/slave_cmd1_nocheck.sh "/usr/local/hadoop/bin/hadoop-daemon.sh stop tasktracker" >logs/cmd1.log 2>&1 &
sh bin/slave_cmd2_nocheck.sh "/usr/local/hadoop/bin/hadoop-daemon.sh stop tasktracker" >logs/cmd1.log 2>&1 &
sh bin/slave_cmd3_nocheck.sh "/usr/local/hadoop/bin/hadoop-daemon.sh stop tasktracker" >logs/cmd1.log 2>&1 &

hbase-daemon.sh stop master
hadoop-daemon.sh stop zkfc
hadoop-daemon.sh stop namenode
hadoop-daemon.sh stop jobtracker

二、启动过程中，在新的服务器执行以下命令
sh bin/slave_cmd1_nocheck.sh "mkdir -p /data/hadoop/data1/mapred/local/taskTracker;mkdir -p /data/hadoop/data2/mapred/local/taskTracker;mkdir -p /data/hadoop/data3/mapred/local/taskTracker;mkdir -p /data/hadoop/data4/mapred/local/taskTracker;mkdir -p /data/hadoop/data5/mapred/local/taskTracker;mkdir -p /data/hadoop/data6/mapred/local/taskTracker;mkdir -p /data/hadoop/data7/mapred/local/taskTracker;mkdir -p /data/hadoop/data8/mapred/local/taskTracker;mkdir -p /data/hadoop/data9/mapred/local/taskTracker;mkdir -p /data/hadoop/data10/mapred/local/taskTracker;mkdir -p /data/hadoop/data11/mapred/local/taskTracker;mkdir -p /data/hadoop/data12/mapred/local/taskTracker" > slave_cmd1_20141202_createDir.log 2>&1
sh bin/slave_cmd2_nocheck.sh "mkdir -p /data/hadoop/data1/mapred/local/taskTracker;mkdir -p /data/hadoop/data2/mapred/local/taskTracker;mkdir -p /data/hadoop/data3/mapred/local/taskTracker;mkdir -p /data/hadoop/data4/mapred/local/taskTracker;mkdir -p /data/hadoop/data5/mapred/local/taskTracker;mkdir -p /data/hadoop/data6/mapred/local/taskTracker;mkdir -p /data/hadoop/data7/mapred/local/taskTracker;mkdir -p /data/hadoop/data8/mapred/local/taskTracker;mkdir -p /data/hadoop/data9/mapred/local/taskTracker;mkdir -p /data/hadoop/data10/mapred/local/taskTracker;mkdir -p /data/hadoop/data11/mapred/local/taskTracker;mkdir -p /data/hadoop/data12/mapred/local/taskTracker" > slave_cmd2_20141202_createDir.log 2>&1 &
sh bin/slave_cmd3_nocheck.sh "mkdir -p /data/hadoop/data1/mapred/local/taskTracker;mkdir -p /data/hadoop/data2/mapred/local/taskTracker;mkdir -p /data/hadoop/data3/mapred/local/taskTracker;mkdir -p /data/hadoop/data4/mapred/local/taskTracker;mkdir -p /data/hadoop/data5/mapred/local/taskTracker;mkdir -p /data/hadoop/data6/mapred/local/taskTracker;mkdir -p /data/hadoop/data7/mapred/local/taskTracker;mkdir -p /data/hadoop/data8/mapred/local/taskTracker;mkdir -p /data/hadoop/data9/mapred/local/taskTracker;mkdir -p /data/hadoop/data10/mapred/local/taskTracker;mkdir -p /data/hadoop/data11/mapred/local/taskTracker;mkdir -p /data/hadoop/data12/mapred/local/taskTracker" > slave_cmd3_20141202_createDir.log 2>&1 &

二、成功启动服务器，启动相关服务
hbase-daemon.sh start master
hadoop-daemon.sh start namenode
hadoop-daemon.sh start zkfc
hadoop-daemon.sh start jobtracker
nohup sh bin/slave_cmd1_nocheck.sh "hadoop-daemon.sh start tasktracker" > slave_cmd1_20141202_start_tasktracker.log
nohup sh bin/slave_cmd2_nocheck.sh "hadoop-daemon.sh start tasktracker" > slave_cmd2_20141202_start_tasktracker.log
nohup sh bin/slave_cmd3_nocheck.sh "hadoop-daemon.sh start tasktracker" > slave_cmd3_20141202_start_tasktracker.log

#启动crontab
三、失败之后的方案：
1.[root@hadoopNN2 ~]# cd /home/hadoop/setup

1修改/etc/hosts.allow 允许10.180.92.198访问
sh bin/slave_cmd1_nocheck.sh "echo 'sshd:10.180.92.198' >> /etc/hosts.allow"
sh bin/slave_cmd2_nocheck.sh "echo 'sshd:10.180.92.198' >> /etc/hosts.allow"
sh bin/slave_cmd3_nocheck.sh "echo 'sshd:10.180.92.198' >> /etc/hosts.allow"

2.修改/etc/hosts并分发到各服务器。
sed -i "s/10.180.92.198/10.100.91.36/" /etc/hosts
sed -i "s/10.100.91.36/10.180.92.198/" /etc/hosts
./upgrade.sh distribute /usr/local/hadoop/conf/slaves /etc/hosts /etc

以hadoop用户进入新服务器 /usr/local/hadoop
#已经启动jobtracker.
nohup sh bin/slave_cmd1_nocheck.sh "hadoop-daemon.sh start tasktracker" > slave_cmd1_20150129_start_tasktracker.log 2>&1 &
nohup sh bin/slave_cmd2_nocheck.sh "hadoop-daemon.sh start tasktracker" > slave_cmd2_20150129_start_tasktracker.log 2>&1 &
nohup sh bin/slave_cmd3_nocheck.sh "hadoop-daemon.sh start tasktracker" > slave_cmd3_20150129_start_tasktracker.log 2>&1 &
hbase-daemon.sh start master
hadoop-daemon.sh start namenode -bootstrapStandby
jps
hadoop-daemon.sh start zkfc

3.设置crontab
*/10 * * * * /home/hadoop/cluster-monitor/monitor.sh >/dev/null 2>&1
0 12 * * * /home/hadoop/cluster-monitor/daily-monitor.sh >/dev/null 2>&1