文章目录
IDE环境准备
准备依赖和打包插件
Flink版本:1.13
<!-- 配置 -->
<properties>
<flink.version>1.13.1</flink.version>
<scala.binary.version>2.12</scala.binary.version>
<slf4j.version>1.7.30</slf4j.version>
</properties>
<!-- 依赖 -->
<dependencies>
<!-- Flink -->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>${flink.version}</version>
<scope>provided</scope> <!-- 参与编译,不参与打包 -->
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-runtime-web_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
<!-- 日志 -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>${slf4j.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
<version>2.14.0</version>
<scope>provided</scope>
</dependency>
</dependencies>
<!-- 打包插件 -->
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.4</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<artifactSet>
<excludes>
<exclude>com.google.code.findbugs:jsr305</exclude>
<exclude>org.slf4j:*</exclude>
<exclude>log4j:*</exclude>
</excludes>
</artifactSet>
<filters>
<filter>
<!-- Do not copy the signatures in the META-INF folder.
Otherwise, this might cause SecurityExceptions when using the JAR. -->
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
<transformers combine.children="append">
<transformer
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer">
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
日志配置:创建log4j.properties
log4j.rootLogger=error, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
编辑IDEA运行时配置
Include dependencies with “Provided” scope
开发步骤

1·Environment
- Job提交前,需要先建立与Flink框架之间的联系,也就是当前的Flink运行环境
// 批处理环境
ExecutionEnvironment benv = ExecutionEnvironment.getExecutionEnvironment();
// 流处理环境
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
2·Source
- 获取数据
从Java的集合中读取数据
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import java.util.ArrayList;
public class Hello {
public static void main(String[] args) throws Exception {
//创建集合
ArrayList<Integer> al = new ArrayList<Integer>();
al.add(1);
al.add(2);
//创建执行环境
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
//从集合中获取数据并打印
env.fromCollection(al).print();
//执行
env.execute();
}
}
从Kafka获取数据
创建Kafka主题
kafka-topics.sh \
--replication-factor 1 \
--partitions 1 \
--zookeeper hadoop102:2181/kafka \
--create \
--topic flink
Flink的Kafka依赖
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka_2.12</artifactId>
<version>1.13.1</version>
</dependency>
运行Java代码
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import java.util.Properties;
public class Hello {
public static void main(String[] args) throws Exception {
//Kafka配置
Properties properties = new Properties();
properties.setProperty("bootstrap.servers", "hadoop100:9092,hadoop101:9092,hadoop102:9092");
properties.setProperty("group.id", "g1");
properties.setProperty("auto.offset.reset", "latest");
//创建执行环境
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env
.addSource(new FlinkKafkaConsumer<>("flink", new SimpleStringSchema(), properties))
.print("Kafka");
//执行
env.execute();
}
}
生产数据
kafka-producer-perf-test.sh \
--throughput -1 \
--print-metrics \
--num-records 10000 \
--record-size 1024 \
--producer-props bootstrap.servers=hadoop102:9092 \
--topic flink
自定义数据源
- 实现
SourceFunction接口 - 覆写
run和cancel方法
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import java.util.ArrayList;
public class Hello {
public static void main(String[] args) throws Exception {
//创建执行环境
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
//创建自定义数据源
ArrayList<Integer> al = new ArrayList<>();
al.add(333);
al.add(4444);
al.add(55555);
MySource mySource = new MySource(al);
//加入数据源
env.addSource(mySource).print();
//执行
env.execute();
}
public static class MySource implements SourceFunction<Integer> {
private final ArrayList<Integer> al;
public MySource(ArrayList<Integer> al) {
this.al = al;
}
@Override
public void run(SourceContext<Integer> sc) {
for (Integer integer : al) {
sc.collect(integer);
}
}
@Override
public void cancel() {
}
}
}
3·Transform
- 数据转换
有map、flatMap、filter、keyBy、shuffle、reduce……
下面以map为例
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
public class Hello {
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.fromElements(1, 2, 3, 4, 5).map(new MapFunction<Integer, Integer>() {
@Override
public Integer map(Integer i) {
return i * i;
}
}).print();
env.execute();
}
}
使用lambda表达式
.map((MapFunction<Integer, Integer>) i -> i * i)
4·Sink
KafkaSink
Flink的Kafka生产者类:FlinkKafkaProducer
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;
public class Hello {
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment().setParallelism(1);
env
.fromElements("a", "bb", "ccc")
.addSink(
new FlinkKafkaProducer<>("hadoop102:9092", "flink",
new SimpleStringSchema())
);
env.execute();
}
}
查看有没有写进去
kafka-console-consumer.sh \
--bootstrap-server hadoop102:9092 \
--from-beginning \
--topic flink
自定义Sink
实现RichSinkFunction
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
public class Hello {
public static void main(String[] args) throws Exception {
//创建执行环境
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
//自定义Sink
env.fromElements(1, 2, 3, 4).addSink(new RichSinkFunction<Integer>() {
@Override
public void invoke(Integer value, Context context) {
System.out.println("value: " + value);
System.out.println("context: " + context);
}
});
//执行
env.execute();
}
}
打印结果
Execution Mode(执行模式)
- 流式API特性:可根据使用情况和Job特点,来选择不同的运行时执行模式
- 流式API默认是使用的STREAMING执行模式,持续在线,常用于无界流,也可用于有界数据
还有一个BATCH执行模式,用于有界数据
配置方式
命令行方式配置
bin/flink run -Dexecution.runtime-mode=BATCH ...
代码方式配置
env.setRuntimeMode(RuntimeExecutionMode.BATCH);
建议
代码方式配置 不灵活,建议用 命令行方式配置 来随时切换有界数据和无界数据
有界数据用STREAMING和BATCH的区别
STREAMING模式:来一条数据输出一次结果
BATCH模式:数据处理完之后,一次性输出结果


1839



被折叠的 条评论
为什么被折叠?



