package apiTest
import org.apache.flink.streaming.api.scala._
object TransformTest {
def main(args: Array[String]): Unit = {
val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
env.setParallelism(1)
//0.读取数据
val inputPath = "D:\\Mywork\\workspace\\Project_idea\\flink-2021\\src\\main\\resources\\sensor.txt"
val inputStream = env.readTextFile(inputPath)
// 1.先转换成样例类类型(简单转换操作)
val dataStream = inputStream
.map( data => {
val arr = data.split(",")
SensorReading(arr(0), arr(1).toLong, arr(2).toDouble)
} )
// 2.分组聚合,输出每个传感器当前最小值
val aggStream = dataStream.keyBy(_.id).minBy("temperature")
// 3.需要输出当前最小的温度值,以及最近的时间戳,要用reduce; curState(当前状态)、newData(新数据)
val resultStream = dataStream.keyBy(_.id).reduce((curState, newData) =>
SensorReading(curState.id, newData.timestamp, curState.temperature.min(newData.temperature))
)
// 4. 多流转换操作
// 4.1 分流,将传感器温度数据分成低温、高温两条流
val splitStream = dataStream.split(data =>
if (data.temperature > 30.0) Seq("high") else Seq("low"))
val highTempStream = splitStream.select("high")
val lowTempStream = splitStream.select("low")
val allTempStream = splitStream.select("all", "low")
// highTempStream.print("high")
// lowTempStream.print("low")
// 4.2 合流,connect
val warningStream = highTempStream.map( data => (data.id, data.temperature) )
val connectedStreams: ConnectedStreams[(String, Double), SensorReading] = warningStream.connect(lowTempStream)
// 用coMap对数据进行分别处理
val coMapResultStream: DataStream[Product with Serializable] = connectedStreams
.map(
warn => (warn._1, warn._2, "warning"),
low => (low.id, "healthy")
)
coMapResultStream.print("coMap")
// 4.3 union合流
val unionStream = highTempStream.union(lowTempStream, allTempStream)
//unionStream.print("union")
env.execute()
}
}
flink 的Transform算子
最新推荐文章于 2025-03-05 18:55:42 发布