在数据量小的情况下，flink基于事件驱动，处理更新、删除、追加历史数据

实现思想，每来一条数据，按照数据 更新、删除、追加三种情况直接用JDBC的方式更新数据库，然后在faltmap函数中，用最新的所有数据通过JDBC的处理，得到最新的结果，样例如下

package com.bai

import java.sql
import java.sql.{
   
   DriverManager, PreparedStatement, ResultSet, ResultSetMetaData}

import com.alibaba.fastjson.{
   
   JSON, JSONObject}
import org.apache.flink.api.common.functions.RichFlatMapFunction
import org.apache.flink.streaming.api.scala._
import org.apache.flink.util.Collector

object JDBCtest {
   
   
  var conn: sql.Connection = _
  var selectStmt: PreparedStatement = _
  var insertStmt: PreparedStatement = _

  def main(args: Array[String]): Unit = {
   
   
    val env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI()
    val ncStream = env.socketTextStream("hadoop104", 7777)
    ncStream.print("before")
    val value1: DataStream[String] = ncStream.filter(_.matches("(\\d)+,(.)*"))
    value1.print("after")
    val value2: DataStream[Student] = value1.flatMap(new RichFlatMapFunction[String, Student] {
   
   
      override def flatMap(value: String, out: Collector[Student]) = {
   
   
        val arr: Array[String] = value.split(",",2)
        conn = DriverManager.getConnection("jdbc:mysql://hadoop103:3306/test", "root", "123456")
          println(arr.toList)
        insertStmt = conn.prepareStatement("insert into input (id, name) values (?,?)")
        insertStmt.setInt(1,arr(0).toInt)
        insertStmt.setString(2,arr(1))
        insertStmt.execute()

        selectStmt = conn.prepareStatement("select * from   input")
        val data: ResultSetMetaData = selectStmt.getMetaData
        val resultSet: ResultSet = selectStmt.executeQuery()

        while (resultSet.next()) {
   
   
          val map = new java.util.HashMap[String, String]()
          val jSONObject = new JSONObject()
          for (i <- 1 to data.getColumnCount) {
   
   
            val key: String = data.getColumnName(i)
            val value: String = resultSet.getString(i)
            map.put(key, value)
            jSONObject.put(key, value)
          }
          //        println(jSONObject)
          val a: Student = JSON.parseObject(jSONObject.toString(), classOf[Student])

          out.collect(a)
        }
        selectStmt.close()
        conn.close()
      }
    })
    value2.print("dd")

//    value1.map(r => {
   
   
//      conn = DriverManager.getConnection("jdbc:mysql://hadoop103:3306/test", "root", "123456")
//      selectStmt = conn.prepareStatement("select * from   input")
//      val data: ResultSetMetaData = selectStmt.getMetaData
//      val resultSet: ResultSet = selectStmt.executeQuery()

//      while (resultSet.next()) {
   
   
//        val map = new java.util.HashMap[String, String]()
//        val jSONObject = new JSONObject()
//        for (i <- 1 to data.getColumnCount) {
   
   
//          val key: String = data.getColumnName(i)
//          val value: String = resultSet.getString(i)
//          map.put(key, value)
//          jSONObject.put(key, value)
//        }
//        //        println(jSONObject)
//        val value: Student = JSON.parseObject(jSONObject.toString(), classOf[Student])
//        println(value + "dd")
//        println(jSONObject.toString())
//      }
//      selectStmt.close()
//      conn.close()
//      r
//    }
//    )
    //    inputstream.addSink(new )
    env.execute()
  }

}

case class Student(id: Int, name: String)

maven 依赖如下

<properties>
		<project.build.sourceEncoding>UTF-8