大数据集群一键启动脚本

1.一键启动

大数据集群启动脚本，包括zk，hadoop，spark，kafka，hbase，redisclickhouse。

vi startall.sh

#!/bin/bash
SPARK_HOME=/opt/module/spark
REDIS_HOME=/opt/module/redis
/bin/zk_shtd.sh start
/bin/hadoop_shtd.sh start
/bin/hive_shtd.sh start
$SPARK_HOME/sbin/start-history-server.sh
/bin/kafka_shtd.sh start
/bin/hbase_shtd.sh start
$REDIS_HOME/bin/redis-server /opt/module/redis/etc/redis.conf
/bin/clickhouse-server_shtd.sh start
/bin/jpsall

2.zk

vi /bin/zk_shtd.sh

#!/bin/bash
ZK_HOME=/opt/module/zookeeper
names=(master slave1 slave2)
if [ $# -lt 1 ]
then
    echo "No Args Input..."
    exit ;
fi

case $1 in
"start")
        for i in ${names[@]}
    do
        echo "=====================  $i  ======================="
        ssh $i "${ZK_HOME}/bin/zkServer.sh start"
    done
;;
"stop")
        for i in ${names[@]}
    do
        echo "=====================  $i  ======================="
        ssh $i "${ZK_HOME}/bin/zkServer.sh stop"
    done
;;
"status")
        for i in ${names[@]}
    do
        echo "=====================  $i  ======================="
        ssh $i "${ZK_HOME}/bin/zkServer.sh status"
    done
;;
*)
    echo "Input Args Error..."
;;
esac

3. hadoop

vi /bin/hadoop_shtd.sh

#!/bin/bash
HADOOP_HOME=/opt/module/hadoop
names=(master slave1 slave2)
if [ $# -lt 1 ]
then
    echo "No Args Input..."
    exit ;
fi

case $1 in
"start")
        echo " =================== START hadoop ==================="

        echo " --------------- start  hdfs ---------------"
        ssh  ${names[0]} "$HADOOP_HOME/sbin/start-dfs.sh"
        echo " --------------- start yarn ---------------"
        ssh ${names[1]}  "$HADOOP_HOME/sbin/start-yarn.sh"
        echo " --------------- start historyserver ---------------"
        ssh ${names[0]}  "$HADOOP_HOME/bin/mapred --daemon start historyserver"
;;
"stop")
        echo " =================== STOP hadoop ==================="

        echo " --------------- stop historyserver ---------------"
        ssh ${names[0]}  "$HADOOP_HOME/bin/mapred --daemon stop historyserver"
        echo " ---------------stop yarn ---------------"
        ssh ${names[1]}  "$HADOOP_HOME/sbin/stop-yarn.sh"
        echo " ---------------stop hdfs ---------------"
        ssh ${names[0]}  "$HADOOP_HOME/sbin/stop-dfs.sh"
;;
*)
    echo "Input Args Error..."
;;
esac

4. hive

vi /bin/hive_shtd.sh

#!/bin/bash
HIVE_HOME=/opt/module/hive
HIVE_LOG_DIR=$HIVE_HOME/logs
if [ ! -d $HIVE_LOG_DIR ]
then
        mkdir -p $HIVE_LOG_DIR
fi
function check_process()
{
    pid=$(ps -ef 2>/dev/null | grep -v grep | grep -i $1 | awk '{print $2}')
    ppid=$(netstat -nltp 2>/dev/null | grep $2 | awk '{print $7}' | cut -d '/' -f 1)
    echo $pid
    [[ "$pid" =~ "$ppid" ]] && [ "$ppid" ] && return 0 || return 1
}

function hive_start()
{
    metapid=$(check_process HiveMetastore 9083)
    cmd="nohup hive --service metastore >$HIVE_LOG_DIR/metastore.log 2>&1 &"
    cmd=$cmd"hdfs dfsadmin -safemode wait >/dev/null 2>&1"
    [ -z "$metapid" ] && eval $cmd || echo "Metastroe START"
    server2pid=$(check_process HiveServer2 10000)
    cmd="nohup hive --service hiveserver2 >$HIVE_LOG_DIR/hiveServer2.log 2>&1 &"
    [ -z "$server2pid" ] && eval $cmd || echo "HiveServer2 START"
}

function hive_stop()
{
    metapid=$(check_process HiveMetastore 9083)
    [ "$metapid" ] && kill $metapid || echo "Metastore stop"
    server2pid=$(check_process HiveServer2 10000)
    [ "$server2pid" ] && kill $server2pid || echo "HiveServer2 stop"
}

case $1 in
"start")
    hive_start
    ;;
"stop")
    hive_stop
    ;;
"restart")
    hive_stop
    sleep 2
    hive_start
    ;;
"status")
    check_process HiveMetastore 9083 >/dev/null && echo "Metastore status" || echo "Metastore status"
    check_process HiveServer2 10000 >/dev/null && echo "HiveServer2 status" || echo "HiveServer2 status"
    ;;
*)
    echo Invalid Args!
    echo 'Usage: '$(basename $0)' start|stop|restart|status'
    ;;
esac


--------------------------------------------------------
第二种，只有启动
#!/bin/bash
HIVE_HOME=/opt/module/hive
#start metastore
${HIVE_HOME}/bin/hive --service metastore >>${HIVE_HOME}/logs/metastore.log 2>&1 &
echo "start metastore"
${HIVE_HOME}/bin/hiveserver2 >>${HIVE_HOME}/logs/hiveserver2.log 2>&1 &
echo "start hiveserver2"

5.kafka

vi /bin/kafka_shtd.sh

#!/bin/bash
KAFKA_HOME=/opt/module/kafka
names=(master slave1 slave2)
if [ $# -lt 1 ]
then
  echo "Input Args Error....."
  exit
fi
case $1 in
start)
 sleep 10
;;
esac
for i in ${names[@]}
do

case $1 in
start)
  echo "==================START $i KAFKA==================="
  ssh $i $KAFKA_HOME/bin/kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties
;;
stop)
  echo "==================STOP $i KAFKA==================="
  ssh $i $KAFKA_HOME/bin/kafka-server-stop.sh stop
;;

*)
 echo "Input Args Error....."
 exit
;;
esac

done

6.hbase

vi /bin/hbase_shtd.sh

#!/bin/bash
HBASE_HOME=/opt/module/hbase
if [ $# -lt 1 ]
then
  echo "Input Args Error....."
  exit
fi


case $1 in
start)
 sleep 10
;;
esac


case $1 in
start)
  echo "==================START HBASE==================="
  $HBASE_HOME/bin/start-hbase.sh
;;
stop)
  echo "==================STOP HBASE==================="
  $HBASE_HOME/bin/stop-hbase.sh
;;

*)
 echo "Input Args Error....."
 exit
;;
esac

7.clickhouse

vi /bin/clickhouse-server_shtd.sh

#!/bin/bash
if [ $# -lt 1 ]
then
  echo "Input Args Error....."
  exit
fi

case $1 in
start)
  echo "==================START clickhouse==================="
  /etc/init.d/clickhouse-server start
;;
stop)
  echo "==================STOP clickhouse==================="
  /etc/init.d/clickhouse-server stop
;;
status)
  echo "==================STOP clickhouse==================="
  systemctl status clickhouse-server
;;

*)
 echo "Input Args Error....."
 exit
;;
esac

8.xsync

vi /bin/xsync
rsync: command not found:yum install rsync -y

#!/bin/bash
names=(master slave1 slave2)
if [ $# -lt 1 ]
then
  echo not enough argument1
  exit;
fi
for host in ${names[@]}
do
  echo ========$host=========
  for file in $@
  do
    if [ -e $file ]
    then
      pdir=$(cd -P $(dirname $file);pwd)
      fname=$(basename $file)
      ssh $host "mkdir -p $pdir"
      rsync -av $pdir/$fname $host:$pdir
    else
      echo $file does not exists!
    fi
  done
done

9.jpsall

vi /bin/jpsall

#!/bin/bash
names=(master slave1 slave2)
for host in ${names[@]}
do
  echo "========$host========="
  ssh ${host} "jps"
done

10.maxwell

#!/bin/bash
MAXWELL_HOME=/opt/module/maxwell
status_maxwell(){
        pid=`ps -ef |grep maxwell | grep -v grep | grep maxwell | awk '{print $2}'`
	if [[ -z $pid ]]
	then
		echo "maxwell not running !!!"
	else
	        echo $pid
	fi
}
start_maxwell(){
	
	if [[ -z $pid ]]
	then
		
		$MAXWELL_HOME/bin/maxwell --config $MAXWELL_HOME/config.properties --daemon
		echo "=====maxwell start!!====="
	else
		echo "maxwell 启动了，不需要再启动。。。"
	fi
}


stop_maxwell(){
	if [[ -z $pid ]]
	then
		echo '====maxwell 没启动===='
	else
		kill -9 $pid
		echo 'kill 掉了maxwell===$pid'
	fi
}

if [ $# -lt 1 ]
then
	start_maxwell
else
	pid=`status_maxwell`	
	case $1 in
	"start")
		start_maxwell
		;;
	"stop")
		stop_maxwell
		;;
	"status")
		echo $pid
		;;

	*)
		echo "start,stop,status"	
		;;
	esac
fi