flink-ProcessFunction API 底层API

最新推荐文章于 2025-05-10 23:10:42 发布

东哥2020

最新推荐文章于 2025-05-10 23:10:42 发布

阅读量174

点赞数

分类专栏： flink 文章标签： flink

本文链接：https://blog.youkuaiyun.com/dongfengpo25/article/details/105251177

版权

flink 专栏收录该内容

6 篇文章

订阅专栏

本文详细介绍Apache Flink在实时流处理场景中的应用，包括如何使用Flink进行数据流读取、乱序数据处理、状态管理和定时器设置，以及通过案例演示温度变化报警功能的实现。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

在这里插入图片描述

package com.dong.flink.func

import org.apache.flink.api.scala._
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import com.dong.flink.SensorReading
import org.apache.flink.streaming.runtime.operators.windowing.WindowOperator.Timer
import org.apache.flink.streaming.api.windowing.time.Time
import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor
import org.apache.flink.streaming.api.functions.KeyedProcessFunction
import org.apache.flink.util.Collector
import org.apache.flink.api.common.state.ValueState
import org.apache.flink.api.common.state.ValueStateDescriptor
import org.apache.flink.configuration.Configuration
import org.apache.flink.api.common.functions.RichFlatMapFunction

object ProcessFunctionTest {
  
  def main(args: Array[String]): Unit = {

    //创建一个批处理的执行环境
    val env = StreamExecutionEnvironment.getExecutionEnvironment
    env.setParallelism(1)

    //让例子更清晰，默认的processTime不够清晰
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
    //默认200ms
    env.getConfig.setAutoWatermarkInterval(100L)

    val s = env.socketTextStream("localhost", 7777)

    val ds = s.map(data => {
      val array = data.split(",")
      SensorReading(array(0).trim, array(1).trim.toLong, array(2).trim.toDouble)
    })
      //乱序场景
      .assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor[SensorReading](Time.seconds(1)) {
        override def extractTimestamp(e: SensorReading): Long = e.timestamp * 1000
      })
    
     val processedStream = ds.keyBy(_.id)
      .process( new TempIncreAlert() )

    val processedStream2 = ds.keyBy(_.id)
//      .process( new TempChangeAlert(10.0) )
      .flatMap( new TempChangeAlert(10.0) )

    val processedStream3 = ds.keyBy(_.id)
      .flatMapWithState[(String, Double, Double), Double]{
      // 如果没有状态的话，也就是没有数据来过，那么就将当前数据温度值存入状态
      case ( input: SensorReading, None ) => ( List.empty, Some(input.temperature) )
      // 如果有状态，就应该与上次的温度值比较差值，如果大于阈值就输出报警
      case ( input: SensorReading, lastTemp: Some[Double] ) =>
        val diff = ( input.temperature - lastTemp.get ).abs
        if( diff > 10.0 ){
          ( List((input.id, lastTemp.get, input.temperature)), Some(input.temperature) )
        } else
          ( List.empty, Some(input.temperature) )
    }

    ds.print("input data")
    processedStream3.print("processed data")

    env.execute("window test")
  }

}

class TempIncreAlert() extends KeyedProcessFunction[String, SensorReading, String] {
   // 定义一个状态，用来保存上一个数据的温度值
  lazy val lastTemp: ValueState[Double] = getRuntimeContext.getState( new ValueStateDescriptor[Double]("lastTemp", classOf[Double]) )
  // 定义一个状态，用来保存定时器的时间戳
  lazy val currentTimer: ValueState[Long] = getRuntimeContext.getState( new ValueStateDescriptor[Long]("currentTimer", classOf[Long]) )

  override def processElement(value: SensorReading, ctx: KeyedProcessFunction[String, SensorReading, String]#Context, out: Collector[String]): Unit = {
    // 先取出上一个温度值
    val preTemp = lastTemp.value()
    // 更新温度值
    lastTemp.update( value.temperature )

    val curTimerTs = currentTimer.value()


    if( value.temperature < preTemp || preTemp == 0.0 ){
      // 如果温度下降，或是第一条数据，删除定时器并清空状态
      ctx.timerService().deleteProcessingTimeTimer( curTimerTs )
      currentTimer.clear()
    } else if ( value.temperature > preTemp && curTimerTs == 0 ){
      // 温度上升且没有设过定时器，则注册定时器
      val timerTs = ctx.timerService().currentProcessingTime() + 5000L
      ctx.timerService().registerProcessingTimeTimer( timerTs )
      currentTimer.update( timerTs )
    }
  }

  override def onTimer(timestamp: Long, ctx: KeyedProcessFunction[String, SensorReading, String]#OnTimerContext, out: Collector[String]): Unit = {
    // 输出报警信息
    out.collect( ctx.getCurrentKey + " 温度连续上升" )
    currentTimer.clear()
  }
}

class TempChangeAlert(threshold: Double) extends RichFlatMapFunction[SensorReading, (String, Double, Double)]{

  private var lastTempState: ValueState[Double] = _

  override def open(parameters: Configuration): Unit = {
    // 初始化的时候声明state变量
    lastTempState = getRuntimeContext.getState(new ValueStateDescriptor[Double]("lastTemp", classOf[Double]))
  }

  override def flatMap(value: SensorReading, out: Collector[(String, Double, Double)]): Unit = {
    // 获取上次的温度值
    val lastTemp = lastTempState.value()
    // 用当前的温度值和上次的求差，如果大于阈值，输出报警信息
    val diff = (value.temperature - lastTemp).abs
    if(diff > threshold){
      out.collect( (value.id, lastTemp, value.temperature) )
    }
    lastTempState.update(value.temperature)
  }

}


class TempChangeAlert2(threshold: Double) extends KeyedProcessFunction[String, SensorReading, (String, Double, Double)]{
  // 定义一个状态变量，保存上次的温度值
  lazy val lastTempState: ValueState[Double] = getRuntimeContext.getState( new ValueStateDescriptor[Double]("lastTemp", classOf[Double]) )

  override def processElement(value: SensorReading, ctx: KeyedProcessFunction[String, SensorReading, (String, Double, Double)]#Context, out: Collector[(String, Double, Double)]): Unit = {
    // 获取上次的温度值
    val lastTemp = lastTempState.value()
    // 用当前的温度值和上次的求差，如果大于阈值，输出报警信息
    val diff = (value.temperature - lastTemp).abs
    if(diff > threshold){
      out.collect( (value.id, lastTemp, value.temperature) )
    }
    lastTempState.update(value.temperature)
  }
}