实践数据湖iceberg 第二十五课后台运行flink sql 增删改的效果

原创

已于 2022-04-28 14:51:05 修改 · 2.4k 阅读

4 ·

CC 4.0 BY-SA版权

文章标签：

#hadoop #hdfs #kafka

于 2022-04-01 15:04:44 首次发布

该系列文章详细介绍了如何使用Flink SQL进行数据湖操作，特别是针对MySQL的变更数据捕获（CDC）到Iceberg的数据湖。涵盖了从创建Iceberg表、读取Kafka数据、处理CDC事件到实时写入、元数据管理等多个环节，并通过Java后台代码展示了Flink SQL执行增删改操作的全过程。内容深入且实用，适合大数据处理和Flink实践者参考。

系列文章目录

前言

代码中展示FLINK SQL 执行增删改的效果

一、JAVA 后台代码

1.代码

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;

public class FlinkSqlCDC {
   
   

    public static void main(String[] args) throws Exception {
   
   
        //TODO 1.准备环境
        //1.1流处理环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        //1.2 表执行环境
        StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);

        String createSql = "CREATE TABLE stock_basic_source(\n" +
                "  `i`  INT NOT NULL,\n" +
                "  `ts_code`     CHAR(10) NOT NULL,\n" +
                "  `symbol`   CHAR(10) NOT NULL,\n" +
                "  `name` char(10) NOT NULL,\n" +
                "  `area`   CHAR(20) NOT NULL,\n" +
                "  `industry`   CHAR(20) NOT NULL,\n" +
                "  `list_date`   CHAR(10) NOT NULL,\n" +
                "  `actural_controller`   CHAR(100),\n" +
                "    PRIMARY KEY(i) NOT ENFORCED\n" +
                ") WITH (\n" +
                "  'connector' = 'mysql-cdc',\n" +
                "  'hostname' = 'hadoop103',\n" +
                "  'port' = '3306',\n" +
                "  'username' = 'hive',\n" +
                "  'password' = '123456',\n" +
                "  'database-name' = 'xxzh_stock',\n" +
                "  'table-name' = 'stock_basic'\n" +
                ")" ;
        //TODO 2.创建动态表
        tableEnv.executeSql(createSql);

        tableEnv.executeSql("select * from stock_basic_source").print();

        //TODO 6.执行任务
        env.execute();
    }

}

2.pom

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

    <artifactId>flink-iceberg-learning</artifactId>
    <groupId>org.example</groupId>
    <version>1.0-SNAPSHOT</version>

    <modelVersion>4.0.0</modelVersion>

    <name>flink-iceberg-learning</name>
    <!-- FIXME change it to the project's website -->
    <url>http://www.example.com</url>

    <properties>
        <!-- project compiler -->
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <maven.compiler.source>1.8</maven.compiler.source>
        <maven.compiler.target>1.8</maven.compiler.target>
        <!-- maven compiler-->
        <scala.maven.plugin.version>3.2.2</scala.maven.plugin.version>
        <maven.compiler.plugin.version>3.8.1</maven.compiler.plugin.version>
        <maven.assembly.plugin.version>3.1.1</maven.assembly.plugin.version>
        <!-- sdk -->
        <java.version>1.8</java.version>
        <scala.version>2.12.12</scala.version>
        <scala.binary.version>2.12</scala.binary.version>
        <!-- engine-->
        <hadoop.version>2.7.2</hadoop.version>
        <flink.version>1.13.5</flink.version>
        <flink.cdc.version>2.0.2</flink.cdc.version>
        <iceberg.version>0.12.1</iceberg.version>
        <hive.version>2.3.6</hive.version>
        <!-- <scope.type>provided</scope.type>-->
        <scope.type>compile</scope.type>
    </properties>
    <dependencies>
        <!-- scala -->
        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-library</artifactId>
            <version>${
   
   scala.version}</version>
            <scope>${
   
   scope.type}</scope>
        </dependency>
        <!-- flink Dependency --