Spark监控之StreamingListener

nice try

引言

关于spark的报警，我们是怎么做的呢？
1、程序本身报警，使用的是StreamingListener，基本上满足需求
- 程序出错报警
- 数据挤压报警
- 数据量异常报警
2、任务端口存活情况报警
spark-submit提交任务的时候指定端口号，–conf spark.ui.port=4083，corntab定时ping就可以了。

源码

package org.apache.spark.streaming.scheduler

@DeveloperApi
trait StreamingListener {
   

  /** Called when the streaming has been started */
  def onStreamingStarted(streamingStarted: StreamingListenerStreamingStarted) {
    }

  /** Called when a receiver has been started */
  def onReceiverStarted(receiverStarted: StreamingListenerReceiverStarted) {
    }

  /** Called when a receiver has reported an error */
  def onReceiverError(receiverError: StreamingListenerReceiverError) {
    }

  /** Called when a receiver has been stopped */
  def onReceiverStopped(receiverStopped: StreamingListenerReceiverStopped) {
    }

  /** Called when a batch of jobs has been submitted for processing. */
  def onBatchSubmitted(batchSubmitted: StreamingListenerBatchSubmitted) {
    }

  /** Called when processing of a batch of jobs has started.  */
  def onBatchStarted(batchStarted: StreamingListenerBatchStarted) {
    }

  /** Called when processing of a batch of jobs has completed. */
  def onBatchCompleted(batchCompleted: StreamingListenerBatchCompleted) {
    }

  /** Called when processing of a job of a batch has started. */
  def onOutputOperationStarted(
      outputOperationStarted: StreamingListenerOutputOperationStarted) {
    }

  /** Called when processing of a job of a batch has completed. */
  def onOutputOperationCompleted(
      outputOperationCompleted: StreamingListenerOutputOperationCompleted) {
    }
}

使用

import java.text.SimpleDateFormat


import org.apache.