第一个Flink SQL作业：WordCount的现代流式实现

最新推荐文章于 2025-12-03 15:33:15 发布

原创最新推荐文章于 2025-12-03 15:33:15 发布 · 704 阅读

10 ·

CC 4.0 BY-SA版权

文章标签：

#flink #sql #c#

1. 从批处理到流式处理的演进

1.1 传统WordCount的局限性

经典的WordCount示例通常以批处理方式实现，但在实时数据场景下存在明显不足：

批处理WordCount（伪代码）

// 读取完整文件后统计
textFile.flatMap(line => line.split(" "))
       .map(word => (word, 1))
       .reduceByKey(_ + _)

流式处理的挑战

数据持续到达，无法等待"文件结束"
需要处理无界数据流
要求低延迟的实时结果更新

1.2 流式WordCount设计思路

现代流式WordCount需要解决：

如何定义"计数窗口"
如何处理乱序数据
如何持续更新统计结果
如何保证精确一次语义

2. 数据流模拟与源表定义

2.1 模拟实时文本数据流

使用Flink内置的DataGen连接器创建文本流：

-- 创建模拟文本数据源表
CREATE TABLE text_stream (
    line_id INT,
    content STRING,
    event_time TIMESTAMP(3),
    WATERMARK FOR event_time AS event_time - INTERVAL '3' SECOND
) WITH (
    'connector' = 'datagen',
    'rows-per-second' = '10',
    'fields.line_id.kind' = 'sequence',
    'fields.line_id.start' = '1',
    'fields.line_id.end' = '10000',
    'fields.content.length' = '20'
);

2.2 数据格式说明

模拟数据示例：

(1, "hello world flink sql", 2023-10-01 10:00:00)
(2, "stream processing with flink", 2023-10-01 10:00:01)  
(3, "sql query on data streams", 2023-10-01 10:00:02)

3.流式WordCount完整实现

3.1 单词拆分与标准化处理

使用Flink SQL的字符串函数进行文本预处理：

-- 创建单词拆分视图
CREATE VIEW word_stream AS
SELECT 
    line_id,
    LOWER(TRIM(word)) as word,
    event_time
FROM text_stream,
    LATERAL TABLE(SPLIT(content)) AS T(word)
WHERE word <> '';

3.2 滚动窗口单词计数

基于事件时间的窗口聚合：

-- 每分钟单词计数
CREATE TABLE minute_word_count (
    window_start TIMESTAMP(3),
    window_end TIMESTAMP(3),
    word STRING,
    word_count BIGINT
) WITH (
    'connector' = 'print'
);

INSERT INTO minute_word_count
SELECT
    window_start,
    window_end,
    word,
    COUNT(*) AS word_count
FROM TUMBLE(TABLE word_stream, DESCRIPTOR(event_time), INTERVAL '1' MINUTES)
GROUP BY 
    window_start,
    window_end,
    word;

3.3 滑动窗口实时热点词检测

检测短期内的热门词汇：

-- 每10秒统计最近1分钟的热点词
CREATE TABLE hot_words AS
SELECT
    window_start,
    window_end,
    word,
    COUNT(*) AS word_count
FROM HOP(TABLE word_stream, DESCRIPTOR(event_time), INTERVAL '10' SECOND, INTERVAL '1' MINUTES)
GROUP BY 
    window_start,
    window_end,
    word
HAVING COUNT(*) > 5;  -- 只输出出现5次以上的词汇

4. 高级特性：状态管理与容错

4.1 全局单词计数（无界流）

使用OVER窗口实现累积统计：

-- 从流开始至今的单词总计数
CREATE TABLE total_word_count AS
SELECT
    word,
    COUNT(*) OVER (
        PARTITION BY word
        ORDER BY event_time
        ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
    ) AS total_count
FROM word_stream;

5. 结果输出与可视化

5.1 多目标输出配置

同时输出到控制台和文件：

-- 控制台输出（调试用）
CREATE TABLE print_sink (
    window_start TIMESTAMP(3),
    window_end TIMESTAMP(3), 
    word STRING,
    word_count BIGINT
) WITH ('connector' = 'print');

-- 文件输出（持久化）
CREATE TABLE file_sink (
    window_start TIMESTAMP(3),
    window_end TIMESTAMP(3),
    word STRING, 
    word_count BIGINT
) WITH (
    'connector' = 'filesystem',
    'path' = 'file:///tmp/flink-output',
    'format' = 'json'
);

-- 同时写入多个目标
INSERT INTO print_sink
SELECT window_start, window_end, word, word_count 
FROM minute_word_count;

INSERT INTO file_sink  
SELECT window_start, window_end, word, word_count
FROM minute_word_count;

5.2 实时仪表板数据准备

为可视化工具准备标准化数据：

-- 为Grafana等工具准备数据
CREATE TABLE dashboard_data (
    window_start STRING,
    word STRING,
    word_count BIGINT,
    update_time TIMESTAMP(3)
) WITH (
    'connector' = 'jdbc',
    'url' = 'jdbc:mysql://localhost:3306/metrics',
    'table-name' = 'word_metrics'
);

INSERT INTO dashboard_data
SELECT
    DATE_FORMAT(window_start, 'yyyy-MM-dd HH:mm:ss'),
    word,
    word_count,
    CURRENT_TIMESTAMP
FROM minute_word_count;

6. 作业提交与监控

6.1 SQL Client作业提交

-- 设置作业名称和配置
SET 'pipeline.name' = 'Streaming-WordCount-Demo';
SET 'parallelism.default' = '2';

-- 执行完整的WordCount管道
BEGIN STATEMENT SET;

INSERT INTO minute_word_count ...;
INSERT INTO hot_words ...;
INSERT INTO dashboard_data ...;

END;

6.2 命令行作业提交

# 将SQL保存为文件
cat > wordcount.sql << 'EOF'
CREATE TABLE text_stream ...;
CREATE VIEW word_stream ...; 
INSERT INTO minute_word_count ...;
EOF

# 提交作业
./bin/sql-client.sh -f wordcount.sql

# 监控作业状态
./bin/flink list
./bin/flink webui

6.3 Web UI监控要点

在http://localhost:8081监控关键指标：

吞吐量：Records Received/Sent per Second
延迟：Checkpoint Duration、Watermark Lag
背压：Task Back Pressure指标
状态：State Size、Checkpoint完成率

7. 故障恢复与一致性保证

7.1 检查点配置优化

-- 启用精确一次语义
SET 'execution.checkpointing.interval' = '30s';
SET 'execution.checkpointing.mode' = 'EXACTLY_ONCE';
SET 'execution.checkpointing.timeout' = '5min';

-- 状态后端配置
SET 'state.backend' = 'rocksdb';
SET 'state.checkpoints.dir' = 'file:///tmp/checkpoints';

7.2 从检查点恢复作业

#从指定检查点恢复
./bin/flink run \
    -s hdfs:///checkpoints/wordcount/chk-12345 \
    -c org.apache.flink.table.client.SqlClient \
    ./lib/sql-client.jar

8. 性能优化实战

8.1 并行度调优

-- 根据数据特征设置并行度
SET 'parallelism.default' = '4';

-- 源表并行度（根据分区数设置）
CREATE TABLE text_stream (...) WITH (
    'connector' = 'datagen',
    'scan.parallelism' = '2',
    ...
);

-- 窗口聚合并行度
CREATE TABLE minute_word_count (...) WITH (
    'sink.parallelism' = '4',
    ...
);

8.2 状态优化策略

-- 设置状态TTL
SET 'table.exec.state.ttl' = '1h';

-- 使用增量检查点减少IO
SET 'state.backend.incremental' = 'true';

9. 测试与验证

9.1 数据验证查询

-- 验证数据完整性
SELECT COUNT(*) as total_words FROM word_stream;

-- 验证窗口计算正确性
SELECT 
    window_start,
    COUNT(DISTINCT word) as distinct_words,
    SUM(word_count) as total_occurrences
FROM minute_word_count
GROUP BY window_start;

-- 验证水位线进度
SELECT CURRENT_WATERMARK(event_time) as current_watermark
FROM text_stream;

9.2 端到端测试用例

-- 测试完整流程
CREATE TABLE test_source (...);
CREATE TABLE test_sink (...);

-- 插入测试数据
INSERT INTO test_source VALUES 
(1, 'test flink sql', '2023-10-01 10:00:00'),
(2, 'stream processing', '2023-10-01 10:00:01');

-- 验证输出结果
SELECT * FROM test_sink;