Flink之自定义JDBC Sink 写入mysql数据

最新推荐文章于 2025-10-07 14:38:01 发布

原创最新推荐文章于 2025-10-07 14:38:01 发布 · 4.7k 阅读

4 ·

CC 4.0 BY-SA版权

flink 专栏收录该内容

4 篇文章

订阅专栏

本文介绍如何使用Apache Flink处理传感器数据，并通过JDBC将处理后的数据写入MySQL数据库。具体步骤包括：从文件读取数据，转换数据为SensorReading对象，使用自定义的JDBC Sink Function将数据写入数据库，实现数据的实时存储。

先建表
在这里插入图片描述
数据

sensor_1, 1547718199, 35.80018327300259
sensor_6, 1547718201, 15.402984393403084
sensor_7, 1547718202, 6.720945201171228
sensor_10, 1547718205, 38.101067604893444
sensor_1, 1547718206, 35.1
sensor_1, 1547718207, 35.6

结果
在这里插入图片描述

package com.flink.sourceAndSink.sink

import java.sql.{Connection, DriverManager, PreparedStatement}

import com.flink.sourceAndSink.SensorReading
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.sink.{RichSinkFunction, SinkFunction}
import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}

/**
 * Created by Shi shuai RollerQing on 2019/12/19 15:16
 */
object JdbcSink {
  def main(args: Array[String]): Unit = {
    val env = StreamExecutionEnvironment.getExecutionEnvironment
    env.setParallelism(1)
    import org.apache.flink.api.scala._
    //1.读取文件到kafka的sinkTest的topic 一般只是测试使用
    val streamFromFile: DataStream[String] = env.readTextFile("C:\\Users\\HP\\IdeaProjects\\sparkCore\\flink\\src\\main\\resources\\sensor.txt")
    val dataStream: DataStream[SensorReading] = streamFromFile.map(data => {
      val dataArray = data.split(",")
      SensorReading(dataArray(0).trim, dataArray(1).trim.toLong, dataArray(2).trim.toDouble)
    })

    //sink
    dataStream.addSink(new MyJdbcSink())

    env.execute("mysql sink test")
  }
}

//因为SinkFunction的功能比较少 所以使用rich的SinkFunction
class MyJdbcSink() extends RichSinkFunction[SensorReading] {
  //定义sql连接、预编译器
  var conn: Connection = _
  var insertStmt: PreparedStatement = _
  var updateStmt: PreparedStatement = _

  //初始化 、 创建连接 、 和 预编译语句
  override def open(parameters: Configuration): Unit = {
    super.open(parameters)
    conn = DriverManager.getConnection("jdbc:mysql://hadoop01:3306/test", "root", "root")
    insertStmt = conn.prepareStatement("insert into temperatures (sensor, temp) values (? , ?)")
    updateStmt = conn.prepareStatement("update temperatures set temp  = ? where sensor = ?")
  }

  // 调用连接 执行sql
  override def invoke(value: SensorReading, context: SinkFunction.Context[_]): Unit = {
    // 执行更新语句
    updateStmt.setDouble(1, value.temperature)
    updateStmt.setString(2, value.id)
    updateStmt.execute()
    //如果update没有更新 即 没有查询到数据 即 没有该id 那么执行插入
    if (updateStmt.getUpdateCount == 0) {
      insertStmt.setString(1, value.id)
      insertStmt.setDouble(2, value.temperature)
      insertStmt.execute()
    }
  }

  //关闭时做清理工作
  override def close(): Unit = {
    insertStmt.close()
    updateStmt.close()
    conn.close()
  }
}