【spark】spark structrued streaming读写kafka 使用kerberos认证

spark版本:2.4.0
官网
Spark --files使用总结
Spark --files理解

一、编写jar

import org.apache.kafka.clients.CommonClientConfigs
import org.apache.kafka.common.config.SaslConfigs
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.streaming.Trigger
import org.apache.spark.streaming.{
   Seconds, StreamingContext}
import org.slf4j.{
   Logger, LoggerFactory}

import scala.collection.mutable

/**
 * 在集群使用以下脚本发送和接受消息！
 * kafka-console-producer.sh --bootstrap-server xxx.01.com:6667,xxx.02.com,xxx.03.com:6667 --topic tp-read  --producer-property security.protocol=SASL_PLAINTEXT
 * kafka-console-consumer.sh --bootstrap-server xxx.01.com:6667,xxx.02.com,xxx.03.com:6667 --topic tp-write --from-beginning  --consumer-property security.protocol=SASL_PLAINTEXT --group tester
 */
object SparkKafka {
   
   val LOG = LoggerFactory.getLogger(classOf[SparkSession])
  val bootstrapServers = "xxx.01.com:6667,xxx.02.com,xxx.03.com:6667"
  val readTopic = "tp-read"
  val writeTopic = "tp-write"

  def main(args: Array[String]): Unit = {
   

    val SparkKafkaProps: mutable.Map[String, String] = mutable.Map.empty

    // Kafka’s own configurations can be set via DataStreamReader.option with kafka. prefix, e.g, stream.option("kafka.bootstrap.servers", "host:port").
    // For possible kafka parameters, see Kafka consumer config docs for parameters related to reading data, and Kafka producer config docs for parameters related to writing data.
    SparkKafkaProps.put("kafka." + CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT")
    SparkKafkaProps.put(