spark集群app运行状态监控

部署运行机器：bdp-40

*/1 * * * * /bd/spark_job_monitor/spark-job-monitor.sh

spark-job-monitor.sh 脚本代码：

#!/bin/bash
2
3 # spark job name array
4 job_name_array=(
5 'app_sparkToMg-1114'
6 'sparkToDB-1016'
7 'sparkToMg-0709'
8 'sparkToMg_0822_online'
9 )
10 
11 job_name_array_01=(
12 'sparkToMg_0319_online'
13 )
14 
15 length=${#job_name_array[@]}
16 for((i=0;i<$length;i++))
17 do
18 app_name=${job_name_array[$i]}
19 #echo $app_name
20 application_id=$(yarn application -list|grep -P "${app_name}\\t"|awk '{print $1}')
21 #echo $application_id
22 application_status=$(yarn application -status ${application_id}|grep "State : \(RUNNING\|ACCEPTED\)")
23 #echo $application_status
24 dt=`date +"%Y-%m-%d %H:%M:%S"`
25 if [ -n "$application_status" ];then
26 echo "${dt} Spark Cluster : ${app_name} status ok."
27 #|mailx -s "spark cluster job monitor" test@163.com
28 else
29 echo "${dt} Spark Cluster : ${app_name} status fail."
30 echo "${dt} Spark Cluster : ${app_name} status execption , please timely check !" |mailx -s "spark cluster job monitor" test@163.com
31 fi
32 #yarn application -status `yarn application -list|grep $app_name|awk '{print $1}'`|grep "State : \(RUNNING\|ACCEPTED\)"

33 done