hadoop配置使用到的脚本

1.ip_hostname.sh 查看ip和hostname

[root@master conf]# vi ip_hostname.sh

#!/bin/bash
# input ip
ip=`ifconfig eno16777736| grep "inet" | awk '{ print $2}'`
echo "本机的IP是$ip"
# input hostname
hostname=`hostname`
echo "本机的主机名是$hostname"
ip=`ifconfig -a|grep inet|grep -v 127.0.0.1|grep -v inet6|awk '{print $2}'|tr -d "addr:"`
echo "本机的IP是$ip"

#1.ifconfig -a  　　　　   和window下执行此命令一样道理，返回本机所有ip信息
#2.grep inet               截取包含ip的行
#3.grep -v 127.0.0.1       去掉本地指向的那行
#4.grep -v inet6           去掉包含inet6的行
#5.awk { print $2}         $2 表示默认以空格分割的第二组 同理 $1表示第一组​ 
#6.tr -d "addr:            删除"addr:"这个字符串

[root@master conf]# ./ip_hostname.sh 
本机的IP是192.168.121.151
fe80::20c:29ff:fe38:f04
本机的主机名是master
本机的IP是192.168.121.151
192.168.122.1

[root@master conf]# sh ip_hostname.sh 
本机的IP是192.168.121.151
fe80::20c:29ff:fe38:f04
本机的主机名是master
本机的IP是192.168.121.151
192.168.122.1

2.genpasswd.sh 本地生成密钥

内容如下

#!/bin/bash
for i in master slaver1 slaver2
do
  echo "========$i生成密钥========"
  ssh $i "ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa"
done

执行

[root@master conf]# ./genpasswd.sh 
========master生成密钥========
The authenticity of host 'master (192.168.121.151)' can't be established.
ECDSA key fingerprint is 7e:62:21:88:c3:41:ba:83:44:54:a9:63:6e:93:8f:67.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'master,192.168.121.151' (ECDSA) to the list of known hosts.
root@master's password: 
Generating public/private rsa key pair.
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
fb:59:35:82:bb:b7:12:e0:56:6c:b3:9d:a9:98:86:46 root@master
The key's randomart image is:
+--[ RSA 2048]----+
|                 |
|                 |
|         .       |
|        . =.     |
|       .S+.+.oo  |
|      E o.o.+o . |
|     . o.o.o.    |
|      o +.o+.    |
|     . .  +o..   |
+-----------------+
========slaver1生成密钥========
root@slaver1's password: 
Generating public/private rsa key pair.
Created directory '/root/.ssh'.
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
32:70:48:56:d2:78:80:ff:73:8d:81:52:b4:85:7a:b4 root@slaver1
The key's randomart image is:
+--[ RSA 2048]----+
|   .=*o..        |
|  .o.o*o         |
|   .o=oo         |
|    +oE .        |
|     +o S+       |
|      ooo .      |
|       o         |
|                 |
|                 |
+-----------------+
========slaver2生成密钥========
root@slaver2's password: 
Generating public/private rsa key pair.
Created directory '/root/.ssh'.
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
b0:a5:6d:47:41:ed:7e:2b:b5:88:a0:59:a0:0a:bb:a5 root@slaver2
The key's randomart image is:
+--[ RSA 2048]----+
|         .o.     |
|           ..    |
|      . . ..     |
|      .* .  .    |
|     .o.S ..     |
|.   .  .o.  . o  |
| o..   + . . + o |
|.o.   o   . o o  |
|E.           .   |
+-----------------+

3. 上传密钥

master,slaver1,slaver2都执行以下三行代码

ssh-copy-id -i ~/.ssh/id_rsa.pub master
ssh-copy-id -i ~/.ssh/id_rsa.pub slaver1
ssh-copy-id -i ~/.ssh/id_rsa.pub slaver2

4.delpasswd.sh 删除密钥目录

[root@master conf]# cat delpasswd.sh 
#!/bin/bash
for i in master slaver1 slaver2
do
  echo "========$i生成密钥========"
  ssh $i "rm -rf ~/.ssh"
done
rm -rf ~/.ssh

5.jps.sh 查看jps

#!/bin/bash
for i in master slaver1 slaver2
do
  echo "========$i的JPS========"
  ssh $i "jps"
done

6.hadoop.sh 重新格式化单步启动hadoop集群

#!/bin/bash
echo -e "\n"
for i in master slaver1 slaver2
do
  echo "========删除$i的进程========"
  ssh $i "jps |awk '{print $1}' |xargs kill -9"
  echo "========查看$i的进程========"
  ssh $i "jps"
done

echo -e "\n"
for i in master slaver1 slaver2
do
  echo "========删除$i的hadoop的log和tmp========"
  ssh $i "rm -rf /export/data/hadoop /export/servers/hadoop-2.7.3/logs"
done

echo "========在master上格式化namenode节点========"
ssh master "hadoop namenode -format"
echo -e "\n"

echo "========在master上启动NameNode========"
ssh master "hadoop-daemon.sh start namenode"
echo -e "\n"

echo "========在slaver2上启动SecondaryNameNode========"
ssh slaver2 "hadoop-daemon.sh start secondarynamenode"
echo -e "\n"

for i in master slaver1 slaver2
do
  echo "========在$i上启动DataNode节点========"
  ssh $i "hadoop-daemon.sh start datanode"
done
echo -e "\n"

echo "========在slaver1启动ResourceManager========"
ssh slaver1 "yarn-daemon.sh start resourcemanager"
echo -e "\n"

for i in master slaver1 slaver2
do
  echo "========在$i上启动NodeManger========"
  ssh $i "yarn-daemon.sh start nodemanager"
done
echo -e "\n"

for i in master slaver1 slaver2
do
  echo "========$i的JPS========"
  ssh $i "jps"
done

7.start.sh 重新格式化启动集群

#!/bin/bash
echo -e "\n"
for i in master slaver1 slaver2
do
  echo "========删除$i的进程========"
  ssh $i "jps |awk '{print $1}' |xargs kill -9"
  echo "========查看$i的进程========"
  ssh $i "jps"
done

echo -e "\n"
for i in master slaver1 slaver2
do
  echo "========删除$i的hadoop的log和tmp========"
  ssh $i "rm -rf /export/data/hadoop /export/servers/hadoop-2.7.3/logs"
done

echo "========在master上格式化namenode节点========"
ssh master "hadoop namenode -format"
echo -e "\n"

echo "========在master上启动hdfs所有节点========"
ssh master "start-dfs.sh"
echo -e "\n"

echo "========在slaver1上启动yarn所有节点========"
ssh slaver1 "start-yarn.sh"
echo -e "\n"

for i in master slaver1 slaver2
do
  echo "========$i的JPS========"
  ssh $i "jps"
done

如果resourcemanager和namenode都在master上可以使用starct-all.sh启动

#!/bin/bash
echo -e "\n"
for i in master slaver1 slaver2
do
  echo "========删除$i的进程========"
  ssh $i "jps |awk '{print $1}' |xargs kill -9"
  echo "========查看$i的进程========"
  ssh $i "jps"
done

echo -e "\n"
for i in master slaver1 slaver2
do
  echo "========删除$i的hadoop的log和tmp========"
  ssh $i "rm -rf /export/data/hadoop /export/servers/hadoop-2.7.3/logs"
done

echo "========在master上格式化namenode节点========"
ssh master "hadoop namenode -format"
echo -e "\n"

echo "========在master上启动所有节点========"
ssh master "start-all.sh"
echo -e "\n"

for i in master slaver1 slaver2
do
  echo "========$i的JPS========"
  ssh $i "jps"
done