一键安装hadoop完全分布式shell脚本

最新推荐文章于 2025-01-24 11:12:56 发布
bankq
最新推荐文章于 2025-01-24 11:12:56 发布
阅读量1.7k
点赞数 2
分类专栏：大数据文章标签： hadoop
本文链接：https://blog.youkuaiyun.com/bankq/article/details/90517247
版权
大数据专栏收录该内容
7 篇文章
订阅专栏
博客围绕Hadoop展开，但具体内容缺失，推测可能涉及Hadoop在大数据开发中的应用、技术特点等信息技术相关内容。
摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >
#!/bin/bash
#配置hadoop的安装目录 修改的地方1 脚本可以自己创建
##在windows编写的代码可能运行有问题执行以下 1>vim redisshell.sh  2>:set ff=unix

echo -e "请输入hadoop的安装目录,不存在脚本自动创建,最后一个/不要写 /bigdata/install"
read esinstallpath

#创建hadoop安装的目录
if [ ! -d $esinstallpath ]; then
   mkdir -p $esinstallpath
fi 
if [ ! -d $esinstallpath ]; then
  echo "创建目录$esinstallpat失败！请检查目录是否有权限"
  exit
fi

#解压tar包
currentdir=$(cd $(dirname $0); pwd)
ls | grep 'hadoop-.*[gz]$'
if [ $? -ne 0 ]; then
   #当前目录没有hadoop的压缩包
   echo "在$currentdir下没有发现hadoop-*.tar.gz,请自行上传!"
   exit
else
   #解压
   tar -zxvf $currentdir/$(ls | grep 'hadoop-.*[gz]$') -C $esinstallpath
fi

esbanben=`ls $esinstallpath| grep 'hadoop-.*'`

#PATH设置
#末行插入
echo "">>~/.bash_profile
echo '#HADOOP'>>~/.bash_profile
echo "export HADOOP_HOME=$esinstallpath/$esbanben">>~/.bash_profile
echo 'export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin'>>~/.bash_profile
source ~/.bash_profile

hadooppath=$esinstallpath/$esbanben
confpath=$esinstallpath/$esbanben/etc/hadoop

javahome=`echo $JAVA_HOME`
bak_dir='export JAVA_HOME=${JAVA_HOME}'
new_dir='export JAVA_HOME='$javahome
sed -i "s!${bak_dir}!${new_dir}!g" $confpath/hadoop-env.sh


#修改core-site.xml
echo -e "请输入hadoop集群服务器名称 例如 mycluster"
read mycluster
sed -i '/<\/configuration>/i\<!--配置hadoop集群服务器名称 -->' $confpath/core-site.xml
sed -i '/<\/configuration>/i\<property>' $confpath/core-site.xml
sed -i '/<\/configuration>/i\  <name>fs.defaultFS</name>' $confpath/core-site.xml
sed -i "/<\/configuration>/i\  <value>hdfs://$mycluster</value>" $confpath/core-site.xml
sed -i "/<\/configuration>/i\</property>" $confpath/core-site.xml

echo -e "请输入hadoop运行时产生文件的存储目录 不需要自己创建目录 集群自己会创建 指定一个空间大的 用来保存hadoop数据 例如 /bigdata/hadoop"
read hadooptmpdir
sed -i '/<\/configuration>/i\<!--hadoop运行时产生文件的存储目录 -->' $confpath/core-site.xml
sed -i '/<\/configuration>/i\<property>' $confpath/core-site.xml
sed -i '/<\/configuration>/i\  <name>hadoop.tmp.dir</name>' $confpath/core-site.xml
sed -i "/<\/configuration>/i\  <value>$hadooptmpdir</value>" $confpath/core-site.xml
sed -i "/<\/configuration>/i\</property>" $confpath/core-site.xml

echo -e "请输入zk集群 例如 cdh01:2181,cdh02:2181,cdh03:2181"
read zkhosts
sed -i '/<\/configuration>/i\<!--配置zookeeper集群，容灾 -->' $confpath/core-site.xml
sed -i '/<\/configuration>/i\<property>' $confpath/core-site.xml
sed -i '/<\/configuration>/i\  <name>ha.zookeeper.quorum</name>' $confpath/core-site.xml
sed -i "/<\/configuration>/i\  <value>$zkhosts</value>" $confpath/core-site.xml
sed -i "/<\/configuration>/i\</property>" $confpath/core-site.xml

echo "core-site.xml 配置如下"

cat $confpath/core-site.xml

echo "core-site.xml 配置完成"
sleep 1
#开始配置hdfs-site.xml
sed -i '/<\/configuration>/i\<!-- 集群服务名称  -->' $confpath/hdfs-site.xml
sed -i '/<\/configuration>/i\<property>' $confpath/hdfs-site.xml
sed -i '/<\/configuration>/i\  <name>dfs.nameservices</name>' $confpath/hdfs-site.xml
sed -i "/<\/configuration>/i\  <value>$mycluster</value>" $confpath/hdfs-site.xml
sed -i "/<\/configuration>/i\</property>" $confpath/hdfs-site.xml

sed -i '/<\/configuration>/i\<!-- 名称节点配置  -->' $confpath/hdfs-site.xml
sed -i '/<\/configuration>/i\<property>' $confpath/hdfs-site.xml
sed -i "/<\/configuration>/i\  <name>dfs.ha.namenodes.$mycluster</name>" $confpath/hdfs-site.xml
sed -i "/<\/configuration>/i\  <value>nn1,nn2</value>" $confpath/hdfs-site.xml
sed -i "/<\/configuration>/i\</property>" $confpath/hdfs-site.xml


echo  "请输入两个nameNode的地址,按照空格分开 例如 cdh01 cdh02" 
read nameNodes
array=(`echo $nameNodes | tr ' ' ' '` )

for i  in `seq 0 $((${#array[@]}-1))`
do
 sed -i "/<\/configuration>/i\<!-- 指定nn$((${i}+1))机器的RPC服务完整监听地址  -->" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\<property>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\  <name>dfs.namenode.rpc-address.$mycluster.nn$((${i}+1))</name>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\  <value>${array[${i}]}:8020</value>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\</property>" $confpath/hdfs-site.xml
 
 sed -i "/<\/configuration>/i\<!-- 指定nn$((${i}+1))机器的http服务地址  -->" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\<property>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\  <name>dfs.namenode.http-address.$mycluster.nn$((${i}+1))</name>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\  <value>${array[${i}]}:50070</value>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\</property>" $confpath/hdfs-site.xml
 
 #配置yarn的环境
 sed -i "/<\/configuration>/i\<property>" $confpath/yarn-site.xml
 sed -i "/<\/configuration>/i\  <name>yarn.resourcemanager.hostname.rm$((${i}+1))</name>" $confpath/yarn-site.xml
 sed -i "/<\/configuration>/i\  <value>${array[${i}]}</value>" $confpath/yarn-site.xml
 sed -i "/<\/configuration>/i\</property>" $confpath/yarn-site.xml
 
 sed -i "/<\/configuration>/i\<property>" $confpath/yarn-site.xml
 sed -i "/<\/configuration>/i\  <name>yarn.resourcemanager.webapp.address.rm$((${i}+1))</name>" $confpath/yarn-site.xml
 sed -i "/<\/configuration>/i\  <value>${array[${i}]}:8088</value>" $confpath/yarn-site.xml
 sed -i "/<\/configuration>/i\</property>" $confpath/yarn-site.xml
 
 
done

echo  "指定journalnode的配置,所有机器 例如 cdh01:8485;cdh02:8485;cdh03:8485;cdh04:8485;cdh05:8485" 
read journalnode
 sed -i "/<\/configuration>/i\<!-- 指定journalnode  -->" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\<property>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\  <name>dfs.namenode.shared.edits.dir</name>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\  <value>qjournal://$journalnode/$mycluster</value>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\</property>" $confpath/hdfs-site.xml
 
 
 sed -i "/<\/configuration>/i\<!--   -->" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\<property>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\  <name>dfs.client.failover.proxy.provider.$mycluster</name>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\  <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\</property>" $confpath/hdfs-site.xml
 
 
 
 sed -i "/<\/configuration>/i\<!--  在故障切换期间将用于隔离Active NameNode的脚本 -->" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\<property>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\  <name>dfs.ha.fencing.methods</name>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\  <value>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\  sshfence" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\  shell(/bin/true)" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\  </value>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\</property>" $confpath/hdfs-site.xml
 
 sed -i "/<\/configuration>/i\<!-- 免密  -->" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\<property>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\  <name>dfs.ha.fencing.ssh.private-key-files</name>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\  <value>/home/.ssh/id_rsa</value>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\</property>" $confpath/hdfs-site.xml
 
 
 sed -i "/<\/configuration>/i\<!-- Journal Node文件存储地址 -->" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\<property>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\  <name>dfs.journalnode.edits.dir</name>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\  <value>$hadooptmpdir/journal</value>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\</property>" $confpath/hdfs-site.xml
 
 sed -i "/<\/configuration>/i\<!--是否开启自动failover机制-->" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\<property>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\  <name>dfs.ha.automatic-failover.enabled</name>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\  <value>true</value>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\</property>" $confpath/hdfs-site.xml
 
 sed -i "/<\/configuration>/i\<!--HDFS文件系统每个block冗余备份数-->" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\<property>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\  <name>dfs.replication</name>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\  <value>3</value>" $confpath/hdfs-site.xml
 sed -i "/<\/configuration>/i\</property>" $confpath/hdfs-site.xml
 
 
echo "hdfs-site.xml 配置如下"
cat $confpath/hdfs-site.xml
echo "hdfs-site.xml 配置完成"
sleep 1

#开始配置mapred-site.xml
#mapred-site.xml
cp $confpath/mapred-site.xml.template $confpath/mapred-site.xml
 sed -i "/<\/configuration>/i\<!--HDFS文件系统每个block冗余备份数-->" $confpath/mapred-site.xml
 sed -i "/<\/configuration>/i\<property>" $confpath/mapred-site.xml
 sed -i "/<\/configuration>/i\  <name>mapreduce.framework.name</name>" $confpath/mapred-site.xml
 sed -i "/<\/configuration>/i\  <value>yarn</value>" $confpath/mapred-site.xml
 sed -i "/<\/configuration>/i\</property>" $confpath/mapred-site.xml

echo "mapred-site.xml 配置如下"
cat $confpath/mapred-site.xml
echo "mapred-site.xml 配置完成"
sleep 1


#开始配置yarn-site.xml
 sed -i "/<\/configuration>/i\<property>" $confpath/yarn-site.xml
 sed -i "/<\/configuration>/i\  <name>yarn.nodemanager.aux-services</name>" $confpath/yarn-site.xml
 sed -i "/<\/configuration>/i\  <value>mapreduce_shuffle</value>" $confpath/yarn-site.xml
 sed -i "/<\/configuration>/i\</property>" $confpath/yarn-site.xml
 
 sed -i "/<\/configuration>/i\<property>" $confpath/yarn-site.xml
 sed -i "/<\/configuration>/i\  <name>yarn.resourcemanager.ha.enabled</name>" $confpath/yarn-site.xml
 sed -i "/<\/configuration>/i\  <value>true</value>" $confpath/yarn-site.xml
 sed -i "/<\/configuration>/i\</property>" $confpath/yarn-site.xml
 
 sed -i "/<\/configuration>/i\<property>" $confpath/yarn-site.xml
 sed -i "/<\/configuration>/i\  <name>yarn.resourcemanager.ha.rm-ids</name>" $confpath/yarn-site.xml
 sed -i "/<\/configuration>/i\  <value>rm1,rm2</value>" $confpath/yarn-site.xml
 sed -i "/<\/configuration>/i\</property>" $confpath/yarn-site.xml
 
 
 sed -i "/<\/configuration>/i\<property>" $confpath/yarn-site.xml
 sed -i "/<\/configuration>/i\  <name>yarn.resourcemanager.zk-address</name>" $confpath/yarn-site.xml
 sed -i "/<\/configuration>/i\  <value>$zkhosts</value>" $confpath/yarn-site.xml
 sed -i "/<\/configuration>/i\</property>" $confpath/yarn-site.xml
 
 sed -i "/<\/configuration>/i\<property>" $confpath/yarn-site.xml
 sed -i "/<\/configuration>/i\  <name>yarn.resourcemanager.cluster-id</name>" $confpath/yarn-site.xml
 sed -i "/<\/configuration>/i\  <value>mycluster1</value>" $confpath/yarn-site.xml
 sed -i "/<\/configuration>/i\</property>" $confpath/yarn-site.xml

echo "yarn-site.xml 配置如下"
cat $confpath/yarn-site.xml
echo "yarn-site.xml 配置完成"
sleep 1
 
 #开始配置slaves
 
echo  "请输入所有的datanode,按照空格分开 例如 cdh01 cdh02 cdh03" 
read datanodes
array=(`echo $datanodes | tr ' ' ' '` )
touch  $confpath/slaves
for datanode in ${array[@]}
do
 echo $datanode >>  $confpath/slaves
done
 
echo "slaves 配置如下"
cat $confpath/slaves
echo "slaves 配置完成"
sleep 1

#分发hadoop安装文件
echo "以下输入的节点必须做免密登录"
echo  "请输入所有的hadoop节点,按照空格分开,当前机器不用输入(当前机器是cdh01) 例如 cdh02 cdh03 cdh04 cdh05" 
read allnodes
user=`whoami`
array=(`echo $allnodes | tr ' ' ' '` )
for allnode in ${array[@]}
do
 echo ======= $allnode  =======
 ssh $allnode "echo ''>>~/.bash_profile"
 ssh $allnode "echo '#HADOOP'>>~/.bash_profile"
 ssh $allnode "echo 'export HADOOP_HOME=$hadooppath'>>~/.bash_profile"
 ssh $allnode 'echo "export PATH=\$PATH:\$HADOOP_HOME/bin:\$HADOOP_HOME/sbin">>~/.bash_profile'
 ssh $allnode "source ~/.bash_profile"
 ssh $allnode "rm -rf $hadooppath"
 ssh $allnode "mkdir -p $hadooppath"
 scp -r $hadooppath/* ${user}@$allnode:$hadooppath/
 echo ======= $allnode 复制完成  =======
done

for allnode in ${array[@]}
do
 echo ======= 在 $allnode 手动执行 source ~/.bash_profile 在通过 hadoop version 查看是否安装成功 =======
done