pyspark kafka mysql_使用sparksql往kafka推送数据-优快云博客

该博客展示了如何使用PySpark从MySQL查询数据，然后将结果转换为JSON格式并推送到Kafka主题。首先，它加载了配置属性，初始化Kafka生产者，接着从Hive支持的SparkSession中执行SQL查询，最后遍历结果数据并将每条记录作为Kafka消息发送。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

packagekafkaimportjava.io.InputStreamimportjava.text.SimpleDateFormatimportjava.util.{Date, HashMap, Properties}importcom.google.gson.JsonObjectimportorg.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig, ProducerRecord}importorg.apache.spark.SparkConfimportorg.apache.spark.sql.SparkSession

object DakaMemProducer {

val prop= newProperties()

val is: InputStream= this.getClass().getResourceAsStream("/conf.properties")

prop.load(is)

val environment_broker_list= "brokers_list";private val brokers =prop.getProperty(environment_broker_list)//Zookeeper connection properties

private val props = newHashMap[String, Object]()

props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokers)

props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer")

props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer")private val producer = new KafkaProducer[String, String](this.props)

def main(args: Array[String]): Unit={

val sparkConf= new SparkConf().setAppName("DakaMemProducer")

val spark=SparkSession.builder().config(sparkConf).enableHiveSupport().getOrCreate()

val date= new Date(new Date().getTime - 86400000L)

val dateFormat= new SimpleDateFormat("yyyyMMdd")

val statisDate=dateFormat.format(date)

val querySql1= "select member_id,flag,nick_name,nick_type from xxx_db.xxx_table where statis_date = " + statisDate //离线数据

val resultDF1 =spark.sql(querySql1)

resultDF1.rdd.foreach(row=>{

val member_id: String= row.getAs[String]("member_id").toString()

val flag: String= row.getAs[String]("flag").toString()

val nick_name: String= row.getAs[String]("nick_name").toString()

val nick_type: String= row.getAs[String]("nick_type").toString()

val json= newJsonObject()

json.addProperty("memberId", member_id).toString

json.addProperty("flag", flag).toString

json.addProperty("nickName", nick_name).toString

json.addProperty("nickType", nick_type).toString

kafkaProducerSend(json.toString)

})if(!environment_broker_list.contains("prd")){

resultDF1.show(100)

}

def kafkaProducerSend(args: String) {if (args != null) {

val topic= "last_member_info"val message= new ProducerRecord[String, String](topic, null, args)

producer.send(message)

}