spark streaming去读取kafka各个topic中的文件数据

最新推荐文章于 2025-05-21 15:52:24 发布

原创最新推荐文章于 2025-05-21 15:52:24 发布 · 370 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#spark #kafka

本文介绍了如何在Spark Streaming中使用Apache Kafka作为消息源，并展示了如何从多个主题中读取数据，过滤和处理event_attendees文件，最后将结果输出到控制台。

pom文件添加的依赖：

  <dependencies>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>4.11</version>
      <scope>test</scope>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core -->
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_2.11</artifactId>
      <version>2.1.0</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql -->
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-sql_2.11</artifactId>
      <version>2.1.0</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming -->
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-streaming_2.11</artifactId>
      <version>2.1.0</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka -->
    <dependency>
      <groupId>org.apache.kafka</groupId>
      <artifactId>kafka_2.11</artifactId>
      <version>2.0.0</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka-clients -->
    <dependency>
      <groupId>org.apache.kafka</groupId>
      <artifactId>kafka-clients</artifactId>
      <version>2.0.0</version>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming-kafka-0-10 -->
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-streaming-kafka-0-10_2.11</artifactId>
      <version>2.0.0</version>
    </dependency>
  </dependencies>

import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord}
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.InputDStream
import org.apache.spark.streaming.kafka010.{ConsumerStrategies, KafkaUtils, LocationStrategies}
import org.apache.spark.streaming.{Seconds, StreamingContext}


object  ReadkafkaTopic {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local[*]").
      setAppName("readtest")
      .set("spark.serializer","org.apache.spark.serializer.KryoSerializer")
    val ssc = new StreamingContext(conf, Seconds(1))
    ssc.checkpoint("d:/ck")
    val kafkaParams = Map(
      ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> "192.168.181.132:9092",
      ConsumerConfig.GROUP_ID_CONFIG -> "zjy",
      ConsumerConfig.MAX_POLL_RECORDS_CONFIG->"500",
      ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG -> classOf[StringDeserializer],
      ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG -> classOf[StringDeserializer],
      ConsumerConfig.AUTO_OFFSET_RESET_CONFIG -> "earliest"
    )
//读取user_friends的文件

//    val ku: InputDStream[ConsumerRecord[String, String]] = KafkaUtils.createDirectStream(ssc,
//      LocationStrategies.PreferConsistent,
//      ConsumerStrategies.Subscribe[String, String](Set("user_friends"), kafkaParams)
//    )
//过滤掉friend列为空的行数
//    ku.filter(ln=>{
//      val reg = ",$".r
//      val it = reg.findAllMatchIn(ln.value())
//      !it.hasNext
//    }).flatMap(line=>{
//      val info = line.value().split(",")
//      info(1).split(" ").map(fid=>{
//        (info(0),fid)
//      })
//    }).foreachRDD(rdd=>rdd.foreach(println))

//读取文件event_attendees
    val kuea: InputDStream[ConsumerRecord[String, String]] = KafkaUtils.createDirectStream(ssc,
      LocationStrategies.PreferConsistent,
      ConsumerStrategies.Subscribe[String, String](Set("event_attendees"), kafkaParams))

//并对event_attendees文件进行处理后再控制台输出
    val yes = kuea.map(line => {
      val info = line.value().split(",", -1)
      (info(0), info(1))
    }).filter(_._2 != "").flatMap(x => {
      x._2.split(" ").map((x._1, _, "yes"))
    })

    val maybe = kuea.map(line => {
      val info = line.value().split(",", -1)
      (info(0), info(2))
    }).filter(_._2 != "").flatMap(x => {
      x._2.split(" ").map((x._1, _, "maybe"))
    })

    val invited = kuea.map(line => {
      val info = line.value().split(",", -1)
      (info(0), info(3))
    }).filter(_._2 != "").flatMap(x => {
      x._2.split(" ").map((x._1, _, "invited"))
    })

    val no = kuea.map(line => {
      val info = line.value().split(",", -1)
      (info(0), info(4))
    }).filter(_._2 != "").flatMap(x => {
      x._2.split(" ").map((x._1, _, "no"))
    })

    yes.union(maybe).union(invited).union(no).foreachRDD(x=>x.foreach(println))



    ssc.start()
    ssc.awaitTermination()
  }

}