Flink2-从集合 文件 kafka 自定义source中读取数据

文件
sensor_1,1547718199,35.8
sensor_6,1547718201,15.4
sensor_7,1547718202,6.7
sensor_10,1547718205,38.1
sensor_1,1547718129,29.8
sensor_1,1547718158,5.8
sensor_1,1547718140,40.8
sensor_1,1547718111,11.8

 

package com.apitest

import java.util.Properties

import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.streaming.api.functions.source.SourceFunction
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011

import scala.util.Random

//定义样例类，温度传感器
case class SensorReading(id:String,timestamp:Long,temperature:Double)

object SourceTest {
  def main(args: Array[String]): Unit = {
    //创建执行环境
    val  env = StreamExecutionEnvironment.getExecutionEnvironment;
    // 1.从集合中读取数据
    val dataList=List(
      SensorReading("sensor_1", 1547718199, 35.8),
      SensorReading("sensor_6", 1547718201, 15.4),
      SensorReading("sensor_7", 1547718202, 6.7),
      SensorReading("sensor_10", 1547718205, 38.1)
    )
 //   val stream1 = env.fromCollection(dataList)
   // stream1.print()

    //2.从文件中读取
    val input ="D:\\workspace\\ideastudy\\flinkstudy\\src\\main\\scala\\com\\apitest\\sensor.txt"
    //val stream2 = env.readTextFile(input)
    //stream2.print()

    //3. 从kafka中读取数据
    val properties=new Properties()
    properties.setProperty("zookeeper.connect", "192.168.31.174:2181")
    properties.setProperty("bootstrap.servers","192.168.31.174:9092")
    properties.setProperty("group.id", "consumer-group")
    val stream3= env.addSource(new FlinkKafkaConsumer011[String]("first_topic",new SimpleStringSchema(),properties))
    //stream3.print()

    //4.自定义数据源

    val stream4= env.addSource(new MySensorSource())
    stream4.print()
    //执行
    env.execute("source test")
  }
}
//自定义数据源
class MySensorSource() extends SourceFunction[SensorReading]{

  //定义一个标识flag,用来标识数据源是否正常发出数据
  var running:Boolean = true

  override def cancel(): Unit = running = false

  override def run(sourceContext: SourceFunction.SourceContext[SensorReading]): Unit = {
    //定义一个随机数发生器
    val rand=new Random()

    //随机生成组(10个) 传感器的初始温度

    var  curTemp=1.to(10).map(i => ("sensor_" + i,rand.nextDouble()*100))
    //定义无限循环,不停的产生数据，除非被cancel
    while(running){
        // 在上次数据基础上,微调更新温度值
        curTemp=curTemp.map(
          data=>(data._1,data._2 + rand.nextGaussian())
        )
      //获取当前时间戳，加入到数据中,调用sourceContext发出数据
      val curTim=System.currentTimeMillis()

      curTemp.foreach(
        data=>sourceContext.collect(SensorReading(data._1,curTim,data._2))
      )
      //间隔一段时间
      Thread.sleep(500)
    }
  }

}