sparkStreaming整合flume 拉模式Poll

package com.spark.streaming

import java.net.InetSocketAddress

import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.flume.FlumeUtils
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}

/**
  * sparkStreaming整合flume 拉模式Poll

  */
object SparkStreamingFlumePoll {
  System.setProperty("hadoop.home.dir", "d://soft//hadoop//hadoop-2.7.3")
  //newValues 表示当前批次汇总成的(word,1)中相同单词的所有的1
  //runningCount 历史的所有相同key的value总和
  def updateFunction(newValues: Seq[Int], runningCount: Option[Int]): Option[Int] = {
    val newCount =runningCount.getOrElse(0)+newValues.sum
    Some(newCount)
  }

  def main(args: Array[String]): Unit = {
    //配置sparkConf参数
    val sparkConf = new SparkConf().setAppName("SparkStreaming_Flume_Poll").setMaster("local[2]")
    //构建sparkContext对象
    val sc = new SparkContext(sparkConf)
    sc.setLogLevel("WARN")
    //构建StreamingContext对象，每个批处理的时间间隔
    val scc = new StreamingContext(sc, Seconds(5))

    //设置checkpoint
    scc.checkpoint("./")
    //设置flume的地址，可以设置多台
    val address = Seq(new InetSocketAddress("star.com",8888))
    // 从flume中拉取数据
    val flumeStream = FlumeUtils.createPollingStream(scc,address,StorageLevel.MEMORY_AND_DISK)

    //获取flume中数据，数据存在event的body中，转化为String
    val lineStream = flumeStream.map(x=>new String(x.event.getBody.array()))
    //实现单词汇总
    val result = lineStream.flatMap(_.split(" ")).map((_,1)).updateStateByKey(updateFunction)

    result.print()
    scc.start()
    scc.awaitTermination()
  }

}

测试和配置信息https://blog.youkuaiyun.com/star5610/article/details/106522455