streaming通过sql实现wordcount代码

最新推荐文章于 2023-04-04 20:58:21 发布

原创最新推荐文章于 2023-04-04 20:58:21 发布 · 227 阅读

0 ·

CC 4.0 BY-SA版权

大数据生态圈专栏收录该内容

61 篇文章

订阅专栏

博客主要提及了代码测试及测试结果，但未给出具体内容。代码测试在信息技术领域是保障软件质量的重要环节。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

代码

package com.badou.sql

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.{Time, Seconds, StreamingContext}
import org.apache.spark.util.IntParam
import org.apache.spark.sql.SQLContext
import org.apache.spark.storage.StorageLevel

case class Record(word: String)

object SQLContextSingleton {
  @transient  private var instance: SQLContext = _
  def getInstance(sparkContext: SparkContext): SQLContext = {
    if (instance == null) {
      instance = new SQLContext(sparkContext)
    }
    instance
  }
}

object sqlAndStreamingWC {
  def main(args: Array[String]) {
    if (args.length < 2) {
      System.err.println("Usage: NetworkWordCount <hostname> <port>")
      System.exit(1)
    }

    val sparkConf = new SparkConf().setMaster("local[2]").setAppName("sqlAndStreamingWC")
    val sc = new SparkContext(sparkConf)
    val ssc = new StreamingContext(sc, Seconds(30))

    val lines = ssc.socketTextStream(args(0), args(1).toInt, StorageLevel.MEMORY_AND_DISK_SER)
    val words = lines.flatMap(_.split(" "))

    words.foreachRDD((rdd: RDD[String], time: Time) => {

      val sqlContext = SQLContextSingleton.getInstance(rdd.sparkContext)
      import sqlContext.implicits._

      val wordsDataFrame = rdd.map(w => Record(w)).toDF()
      wordsDataFrame.registerTempTable("words")
      val wordCountsDataFrame =
        sqlContext.sql("select word, count(*) as total from words group by word")
      println(s"========= $time =========")
      wordCountsDataFrame.show()


    })

    ssc.start()
    ssc.awaitTermination()
  }
}

测试：