flink4 - 输出到文件/kafka/redis/mysql

数据
sensor_1,1547718199,35.8
sensor_6,1547718201,15.4
sensor_7,1547718202,6.7
sensor_10,1547718205,38.1
sensor_1,1547718129,29.8
sensor_1,1547718158,5.8
sensor_1,1547718140,40.8
sensor_1,1547718111,11.8

package com.sinktest

import java.sql.{Connection, Driver, DriverManager, PreparedStatement}

import org.apache.flink.api.common.serialization.{SimpleStringEncoder, SimpleStringSchema}
import org.apache.flink.configuration.Configuration
import org.apache.flink.core.fs.Path
import org.apache.flink.streaming.api.functions.sink.{RichSinkFunction, SinkFunction}
import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011
import org.apache.flink.streaming.connectors.redis.RedisSink
import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisPoolConfig
import org.apache.flink.streaming.connectors.redis.common.mapper.{RedisCommand, RedisCommandDescription, RedisMapper}
//定义样例类，温度传感器
case class SensorReading3(id:String,timestamp:Long,temperature:Double)
object FileSink {
  def main(args: Array[String]): Unit = {
    val env=StreamExecutionEnvironment.getExecutionEnvironment
    val inputPath ="D:\\workspace\\ideastudy\\flinkstudy\\src\\main\\scala\\com\\apitest\\sensor.txt"
    env.setParallelism(1)//为了测试保持读取数据的顺序
    val inputStream = env.readTextFile(inputPath)
    //1.先转换成样例类类型
    val dataStream= inputStream.map(data=>{
      val arr = data.split(",")
      SensorReading3(arr(0),arr(1).toLong,arr(2).toDouble)
    })
    //1.输出到文件
    //dataStream.writeAsText("D:\\workspace\\ideastudy\\flinkstudy\\src\\main\\resources\\hello2.txt")
    //2.文件写入第二种方式
//    dataStream.addSink(StreamingFileSink.forRowFormat(
//      new Path("D:\\workspace\\ideastudy\\flinkstudy\\src\\main\\resources\\hello3.txt"),
//      new SimpleStringEncoder[SensorReading3]()
//    ).build()
//    )

    //3.往kafka中写数据
    val kafkaSink=dataStream.map(data=>data.toString)
    //kafkaSink.addSink(new FlinkKafkaProducer011[String]("localhost:9092","sinktest",new SimpleStringSchema()))

    //4.往redis中写数据
    val conf = new FlinkJedisPoolConfig.Builder()
        .setHost("localhost")
        .setPort(6379)
        .build()

   // dataStream.addSink(new RedisSink[SensorReading3](conf,new MyRedisMapper))

    //5.自定义sink
    dataStream.addSink(new MyJdbcSinkFunc)
    env.execute("file sink test")
  }
}

//定义一个RedisMapper
class MyRedisMapper extends RedisMapper[SensorReading3]{
  //定义保存数据写入redis的命令
  override def getCommandDescription: RedisCommandDescription = {
    //Hset 表示为hash表 ,第二个参数表示为表名
    new RedisCommandDescription(RedisCommand.HSET,"sensor_temp")
  }
  //将温度值指定为value
  override def getKeyFromData(t: SensorReading3): String = {
    t.temperature.toString
  }

  //将id指定为key
  override def getValueFromData(t: SensorReading3): String = {
    t.id
  }
}

class MyJdbcSinkFunc extends RichSinkFunction[SensorReading3]{
  //定义连接、预编译语句
  var conn:Connection= _
  var insertStmt:PreparedStatement=_
  var updateStmt:PreparedStatement=_

  override def open(parameters: Configuration): Unit = {
    conn=DriverManager.getConnection("jdbc:mysql://localhost/test","root","root")
    insertStmt=conn.prepareStatement("insert into test(id,temp) values(?,?)")
    updateStmt=conn.prepareStatement("update test set temp=? where id=?")
  }

  //每来一条数据走这个方法
  override def invoke(value: SensorReading3, context: SinkFunction.Context[_]): Unit = {
      //先执行更新操作，查到就更新
    updateStmt.setDouble(1,value.temperature);
    updateStmt.setString(2,value.id);
    updateStmt.execute();
    if(updateStmt.getUpdateCount == 0){
      insertStmt.setString(1,value.id);
      insertStmt.setDouble(2,value.temperature);
      insertStmt.execute();
    }
  }

  override def close(): Unit = {
    insertStmt.close()
    updateStmt.close()
    conn.close()
  }
}