脚本
节点之间同步文件
#!/bin/bash
if [ $# -lt 1 ]
then
echo not enough argument!
exit;
fi
for host in hadoop1 hadoop2 hadoop3
do
echo =======================$host=====================
for file in $@
do
if [ -e $file ]
then
pdir=$(cd -P $(dirname $file); pwd)
fname=$(basename $file)
ssh $host "mkdir -p $pdir"
rsync -av $pdir/$fname $host:$pdir
else
echo $file does not exist!
fi
done
done
启动集群的hdfs 、yarn、历史记录服务器
#!/bin/bash
if [ $# -lt 1 ]
then echo "not enought arguments"
exit;
fi
case $1 in
"start")
echo "=====================start hadoop================="
echo "=====================start HDFS==================="
ssh hadoop1 "/opt/module/hadoop-3.1.4/sbin/start-dfs.sh"
echo "=====================start yarn==================="
ssh hadoop2 "/opt/module/hadoop-3.1.4/sbin/start-yarn.sh"
echo "=====================start history server=============="
ssh hadoop2 "/opt/module/hadoop-3.1.4/bin/mapred --daemon start historyserver"
;;
"stop")
echo "====================stop hadoop====================="
echo "====================stop history server================="
ssh hadoop2 "/opt/module/hadoop-3.1.4/bin/mapred --daemon stop historyserver"
echo "====================stop yarn==========================="
ssh hadoop2 "/opt/module/hadoop-3.1.4/sbin/stop-yarn.sh"
echo "====================stop HDFS============================"
ssh hadoop1 "/opt/module/hadoop-3.1.4/sbin/stop-dfs.sh"
;;
*)
echo "wrong input arguments"
;;
esac
看所有节点上的java进程
#!/bin/bash
for host in hadoop1 hadoop2 hadoop3
do
echo "================$host=================="
ssh $host "jps"
done
端口号
- hadoop3:
HDFS内部通信常用端口:8020/9000/9820
HDFS对用户的web端口:9870
Yarn查看任务的web端口:8088
历史服务器的web端口:19888
- hadoop2:
HDFS内部通信常用端口:8020/9000
HDFS对用户的web端口:50070
Yarn查看任务的web端口:8088
历史服务器的web端口:19888
配置文件
版本 | 常用配置文件 |
---|---|
hadoop3 | core-site.xml hdfs-site.xml yarn-site.xml mapred-site.xml workers |
hadoop2 | core-site.xml hdfs-site.xml yarn-site.xml mapred-site.xml slaves |