Spark任务跑完将结果以邮件的方式进行发送_spark driver 节点发邮件-优快云博客

本文链接：https://blog.youkuaiyun.com/qq_28844767/article/details/93901981

本文介绍了一种使用Scala编写的Spark任务结果即时通知方案，通过整合play-mailer库，实现任务完成后立即发送邮件通知，包括邮件正文和附件，提高了工作效率。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

工作中，一般是晚上提交任务，第二天等结果，等跑完再走等问题阻碍着我们的步伐，现开发提交完任务就可以第一时间知道，并可以发送给领导，很方便

以下代码为scala编写，主要应用于跑spark任务后的结果发送。

开始需要：构建maven项目：

添加依赖pom.xml:

  <!-- https://mvnrepository.com/artifact/com.typesafe.play/play-mailer -->
        <dependency>
            <groupId>com.typesafe.play</groupId>
            <artifactId>play-mailer_2.11</artifactId>
            <version>7.0.0</version>
        </dependency>

里面主要包含

生成邮件
创建发邮件者
创建附件并添加
快速发送简单信息等。

package utils

import java.io.File

import com.typesafe.config.ConfigFactory
import org.apache.spark.rdd.RDD
import play.api.libs.mailer._


/**
  * @program: route
  * @class_name TaskSendMail
  * @author: rk
  * @create: 2019-06-24 09:57
  * @Description:
  **/
object TaskSendMail {

  /**
    * 定义一个发邮件的人
    * @param host STMP服务地址
    * @param port STMP服务端口号
    * @param user STMP服务用户邮箱
    * @param password STMP服务邮箱密码
    * @param timeout setSocketTomeout 默认: 60s
    * @param connectionTimeout setSocketConnectionTimeout 默认：60s
    * @return  返回一个可以发邮件的用户
    */
  def createMailer(host:String, port: Int, user: String, password: String, timeout:Int = 10000, connectionTimeout:Int = 10000):SMTPMailer ={
    // STMP服务SMTPConfiguration
    val configuration = new SMTPConfiguration(
      host, port, false, false, false,
      Option(user), Option(password), false, timeout = Option(timeout),
      connectionTimeout = Option(connectionTimeout), ConfigFactory.empty(), false
    )
    val mailer: SMTPMailer = new SMTPMailer(configuration)
    mailer
  }


  /**
    * 生成一封邮件
    * @param subject 邮件主题
    * @param from 邮件发送地址
    * @param to 邮件接收地址
    * @param bodyText 邮件内容
    * @param bodyHtml 邮件的超文本内容
    * @param charset 字符编码 默认： utf-8
    * @param attachments 邮件的附件
    * @return 一封邮件
    */
  def createEmail(subject:String, from:String, to:Seq[String], bodyText:String = "ok", bodyHtml:String = "", charset:String = "utf-8", attachments:Seq[Attachment] = Seq.empty): Email = {

    val email = Email(subject, from, to,
      bodyText = Option[String](bodyText), bodyHtml = Option[String](bodyHtml),
      charset= Option[String](charset),attachments = attachments

    )
    email

  }

  /**
    *  生成一个附件
    * @param name 附件的名字
    * @param fileStr 以本地文件为附件相关参数
    * @param rdd 以hdfs文件或rdd或df为附件相关参数
    * @return
    */
  def createAttachments(name: String, fileStr: String = "", rdd:RDD[String] = null): Attachment  = {
    var attachment: Attachment = null
    if(fileStr.contains(":")){
      val file: File = new File(fileStr)
      attachment = AttachmentFile(name, file)
    }else{
      val data: Array[Byte] = rdd.collect().mkString("\n").getBytes()
      // 根据文件类型选择MimeTypes对应的值
      val mimetype = "text/plain"
      attachment = AttachmentData(name, data, mimetype)
    }
    attachment
  }


  /**
    *  主要针对日常简单结果的快速发送
    * @param subject 邮件主题名字
    * @param toStr 邮件的接收人，多名以,分割
    * @param bodyText 邮件的内容
    * @return 用户设备 <510109769.1.1561635225728@RAN>
    */
  def dailyEmail(subject:String, toStr:String, bodyText: String):String={
    val to = toStr.split(",").toList
    // 阿里云企业 邮箱
    val host = "smtp.mxhichina.com"
    val port = 25
    val user = "@.com"
    val password = ""

    val from = user


    val mailer: SMTPMailer = TaskSendMail.createMailer(host, port, user, password)
    val email: Email = TaskSendMail.createEmail(subject, from, to, bodyText = bodyText)
    val userdev: String = mailer.send(email)
    userdev
  }
  

}

进行测试：

package email

import org.apache.spark.rdd.RDD
import org.apache.spark.sql.SparkSession
import play.api.libs.mailer.{Attachment, Email, SMTPMailer}
import utils.TaskSendMail

/**
  * @program: route
  * @class_name EmailTest
  * @author: rk
  * @create: 2019-06-24 11:10
  * @Description:
  **/
object EmailTest {
  def main(args: Array[String]): Unit = {


    val subject = "ais-scala发邮件"
    val from = "@.com"
    val to = Seq("@.com","@qq.com")
    val bodyText = "发个邮件"
    val bodyHtml =
      """
        |<html>
        |<body>
        |<h1 align="center">发邮件成功啦</h1>
        |</body>
        |</html>
      """.stripMargin
    val charset = "utf-8"
    //    val emails: Email = email(subject, from, to)
    //    val emails: Email = email(subject, from, to, bodyText = bodyText, charset =charset)
    //    val emails: Email = email(subject, from, to, bodyHtml = bodyHtml)

    val name1 = "附件1"
    val file = "d:/loong/data/ais.csv"
    val attachment1 = TaskSendMail.createAttachments(name1, file)
    //    val emails: Email = email(subject, from, to, attachments = Seq(attachment1))

    val name2 = "附件2"
    val session: SparkSession = SparkSession.builder().master("local[*]")
      .appName(this.getClass.getSimpleName).getOrCreate()
    val ais_rdd: RDD[String] = session.sparkContext.textFile("d:/loong/data/ais.csv")
    val attavhment2: Attachment = TaskSendMail.createAttachments(name2, rdd = ais_rdd)


    val emails: Email = TaskSendMail.createEmail(subject, from, to)
//    val emails: Email = TaskSendMail.createEmail(subject, from, to, attachments = Seq(attachment1, attavhment2))



//    val host = "smtp.qq.com"
////    val port = 587
////    val user = "@qq.com"
////    val password = "jsjvokandbvxbbch"

//  阿里云企业 邮箱
    val host = "smtp.mxhichina.com"
    val port = 25
    val user = "@.com"
    val password = ""


    val timeout = Option(10000)
    val connectionTimeout = Option(10000)
    val mailerop: SMTPMailer = TaskSendMail.createMailer(host, port, user, password)
    mailerop.send(emails)

    println("发送成功！！！")

  }


}

日常邮件发送测试：

package email

import utils.TaskSendMail

/**
  * @program: route
  * @class_name DailyEmail
  * @author: rk
  * @create: 2019-06-27 19:11
  * @Description:
  **/
object DailyEmail {
  def main(args: Array[String]): Unit = {
    val subject = "吃过饭了"
    val to = "@qq.com"
    val bodyText = "真的吃过了！！！"
    TaskSendMail.dailyEmail(subject, to, bodyText)
    println("发送成功！！！")

  }

}

注：