Claroja

package com.claroja.window;import com.claroja.model.SensorReading;import com.claroja.readwrite.SensorSource;import org.apache.flink.streaming.api.datastream.DataStream;import org.apache.flink.streaming.api.datastream.KeyedStream;import org.apache.fli

package com.claroja.watermark;import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;import org.apache.flink.api.common.eventtime.WatermarkStrategy;import org.apache.flink.api.common.typeinfo.TypeHint;import org.apache.flink.api.ja

package com.claroja.function;import org.apache.flink.api.common.functions.RichMapFunction;import org.apache.flink.api.common.functions.RuntimeContext;import org.apache.flink.configuration.Configuration;import org.apache.flink.streaming.api.datastream.

package com.claroja.function;import com.claroja.model.SensorReading;import com.claroja.readwrite.SensorSource;import org.apache.flink.streaming.api.datastream.DataStream;import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;pu

package com.claroja;import org.apache.flink.streaming.api.datastream.DataStreamSource;import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;import org.apache.flink.table.annotation.DataTypeHint;import org.apache.flink.table.anno

<dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-cep_${scala.binary.version}</artifactId> <version>${flink.version}</version> <scope>${scp}</scope> </dependency>..

<dependency> <groupId>mysql</groupId> <artifactId>mysql-connector-java</artifactId> <version>8.0.21</version> </dependency>package com.claroja;import org.apache.flink.configuration.Configurat..

<dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-connector-elasticsearch7_2.11</artifactId> <version>1.11.0</version> </dependency>package com.claroja;import org.apache.flin..

package com.claroja;import org.apache.flink.streaming.api.datastream.DataStream;import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;import org.apache.flink.streaming.connectors.redis.RedisSink;import org.apache.flink.streaming

package com.claroja;import org.apache.flink.api.common.serialization.SimpleStringSchema;import org.apache.flink.streaming.api.datastream.DataStreamSource;import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;import org.apache.fl

package com.claroja;import org.apache.flink.api.common.serialization.SimpleStringSchema;import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011;import java.

1.at-most-once: 故障发生之后，计数结果可能丢失。同样的还有udp。2.at-least-once: 计数程序在发生故障后可能多算，但是绝不会少算。3.exactly-once: 这指的是系统保证在发生故障后得到的计数结果与正确值一致。Flink的一个重大价值在于，它既保证了exactly-once，也具有低延迟和高吞吐的处理能力。1.内部保证 —— 依赖checkpoint2.source 端 —— 需要外部源可重设数据的读取位置3.sink 端 —— 需..

键控状态数据结构1.值状态（ Value State ） new ValueStateDescriptor(“last-temp”, Types.of[T])• 将状态表示为单个的值2.列表状态（ List State ）• 将状态表示为⼀组数据的列表3.映射状态（ Map State ，字典状态，哈希表状态） new MapStateDescriptor• 将状态表示为⼀组 Key-Value 对4.聚合状态（ Reducing State & Aggregating Stat

1.MemoryStateBackend(Default)• 内存级的状态后端，会将键控状态作为内存中的对象进⾏管理，将它们存储在 TaskManager 的JVM 堆上，⽽将 checkpoint 存储在 JobManager 的内存中• 特点：快速、低延迟，但不稳定2.FsStateBackend• 将 checkpoint 存到远程的持久化⽂件系统（ FileSystem ）上，⽽对于本地状态，跟MemoryStateBackend ⼀样，也会存在 TaskMana.

package com.claroja;import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;import org.apache.flink.api.common.eventtime.WatermarkStrategy;import org.apache.flink.api.common.functions.JoinFunction;import org.apache.flink.api.java.tu

package com.claroja;import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;import org.apache.flink.api.common.eventtime.WatermarkStrategy;import org.apache.flink.api.java.tuple.Tuple3;import org.apache.flink.streaming.api.TimeChara

package com.claroja;import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;import org.apache.flink.api.common.eventtime.WatermarkStrategy;import org.apache.flink.api.common.functions.MapFunction;import org.apache.flink.api.common.s

package com.claroja;import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;import org.apache.flink.api.common.eventtime.WatermarkStrategy;import org.apache.flink.api.common.functions.MapFunction;import org.apache.flink.api.common.s

package com.claroja;import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;import org.apache.flink.api.common.eventtime.WatermarkStrategy;import org.apache.flink.api.common.typeinfo.TypeHint;import org.apache.flink.api.java.tuple.T

package com.claroja;import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;import org.apache.flink.api.common.eventtime.WatermarkStrategy;import org.apache.flink.api.common.typeinfo.TypeHint;import org.apache.flink.api.java.tuple.T

package com.claroja;import org.apache.flink.streaming.api.datastream.DataStream;import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;import org.apache.

a = 1a = 100a = 101每次都发送最小值

1.合流发送最小的水位线 22.4覆盖后,发送最小的33.7覆盖后,发送最小的34.6覆盖后,发送最小的4

自定义生成水位线package com.claroja;import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;import org.apache.flink.api.common.eventtime.WatermarkStrategy;import org.apache.flink.api.common.functions.MapFunction;import org.apache.flink.api

• Event Time （事件时间）：事件创建的时间（必须包含在数据源中的元素⾥⾯）• Ingestion Time （摄⼊时间）：数据进⼊ Flink 的 source 算⼦的时间，与机器相关• Processing Time （处理时间）：执⾏操作算⼦的本地系统时间，与机器相关⽔位线：系统认为时间戳⼩于⽔位线的事件都已经到达了因此， Window 的执⾏也是由 Watermark 触发的（⽔位线 >= 窗⼝结束时间）⽔位线产⽣的公式：⽔位线 = 系统观察到的最⼤事件时间

package com.claroja;import org.apache.flink.api.common.functions.AggregateFunction;import org.apache.flink.api.java.tuple.Tuple3;import org.apache.flink.streaming.api.datastream.DataStreamSource;import org.apache.flink.streaming.api.environment.Stream

全窗口聚合函数（full window functions）1.先把窗口所有数据收集起来2.窗口关闭时,遍历所有数据ProcessWindowFunction全窗口函数相比累加窗口函数好的优势是可以获得窗口的上下文信息,比如窗口的开始时间和结束时间package com.claroja;import org.apache.flink.streaming.api.datastream.DataStreamSource;import org.apache.flink.streaming.api.

增量聚合函数（incremental aggregation functions）1.来一条数据计算一次2.每次计算,保持一个简单的状态（累加器）3.当窗口闭合时,增量聚合完成ReduceFunction, AggregateFunctionpackage com.claroja;import org.apache.flink.api.common.functions.AggregateFunction;import org.apache.flink.api.java.tuple.Tuple

Flink使用类型信息的概念来表示数据类型，并为每个数据类型生成特定的序列化器、反序列化器和比较器。但是，在某些情况下，例如lambda函数或泛型类型，需要显式地提供类型信息，才能使应用程序正常工作或提高其性能。1.Java基础类型Int, Double, Long, String, …2.Java元组（Tuples）3.Java简单对象（POJOs）public class Person {public String name;public int age; public Person(

setParallelism最好不设置全局并行度，而设置每个算子的并行度。如果需要source算子，设置为1，可以保证读取的顺序。package com.claroja;import org.apache.flink.streaming.api.datastream.DataStream;import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;public class Parallelism {

package com.claroja;import org.apache.flink.api.common.functions.MapFunction;import org.apache.flink.streaming.api.datastream.DataStream;import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;public class MapOperator { publi

package com.claroja;import org.apache.flink.streaming.api.datastream.DataStream;import org.apache.flink.streaming.api.datastream.KeyedStream;import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;public class KeyedStreamOperator

package com.claroja;import org.apache.flink.api.common.functions.FlatMapFunction;import org.apache.flink.streaming.api.datastream.DataStream;import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;import org.apache.flink.util.Coll

package com.claroja;import org.apache.flink.api.common.functions.FilterFunction;import org.apache.flink.streaming.api.datastream.DataStream;import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;public class FilterOperator {

package com.claroja;import org.apache.flink.api.java.tuple.Tuple2;import org.apache.flink.streaming.api.datastream.ConnectedStreams;import org.apache.flink.streaming.api.datastream.DataStream;import org.apache.flink.streaming.api.environment.StreamExe

package com.claroja;import org.apache.flink.streaming.api.datastream.DataStream;import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;public class ConsumerFromSensorSource { public static void main(String[] args) throws Exce

1.模拟数据生成package com.claroja;import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;import java.util.Random;public class SensorSource extends RichParallelSourceFunction<SensorReading> { Boolean running = true;

Flink 算子