Flink Iceberg 测试

组件版本

组件	版本
Java	1.8.251
Scala	1.12.14
Flink	1.12.5
Iceberg	0.12.0
Hadoop	2.9.2
Hive	2.3.6

将hdfs-site.xml，core-site.xml，hive-site.xml放入resources下

hadoop_catalog

object TestFlinkSQLOptIcebergHadoopCatalog {
  def main(args: Array[String]): Unit = {
    //   val env = StreamExecutionEnvironment.getExecutionEnvironment
    val settings = EnvironmentSettings
      .newInstance()
      .useBlinkPlanner()
      .inBatchMode()
      .build()

    val tableEnv = TableEnvironment.create(settings)

    val DDL =
      """
        |create catalog hadoop_catalog with (
        | 'type' = 'iceberg',
        | 'catalog-type' = 'hadoop',
        | 'property-version' = '1',
        | 'warehouse' = 'hdfs:///user/hive/warehouse/'
        |)
        |""".stripMargin
    tableEnv.executeSql(DDL)

    //    两种写法
    //    tableEnv.executeSql("use catalog hadoop_catalog")
    tableEnv.useCatalog("hadoop_catalog")

    tableEnv.executeSql("create database if not exists iceberg_db")
    //    tableEnv.executeSql("use iceberg_db")
    tableEnv.useDatabase("iceberg_db")

    tableEnv.executeSql("show databases").print()
    tableEnv.executeSql("show tables").print()

    // 1. 创建表
    //    val tableDDL =
    //      """
    //        |create table if not exists iceberg_test_table (
    //        | id bigint comment 'unique id',
    //        | data string
    //        |) comment 'iceberg test table'
    //        | partitioned by (data)
    //        |""".stripMargin
    //    tableEnv.executeSql(tableDDL)
    //    tableEnv.executeSql("show tables").print()

    // *** 2. 修改表名，暂不支持 hadoop catalog 更改表名等操作,当前仅支持更新设置属性和删除
    //    tableEnv.executeSql("alter table iceberg_test_table rename to iceberg_test_table2")
    //    tableEnv.executeSql("show tables").print()

    // 3. 删除表
    //    tableEnv.executeSql("drop table if exists iceberg_test_table")
    //    tableEnv.executeSql("show tables").print()

    // 4. 查询表
    //    tableEnv.executeSql("show tables").print()

    // 5. like 根据已有表创建新表
    tableEnv.executeSql("create table iceberg_test_like like iceberg_test_table")
    tableEnv.executeSql("show tables").print()

    // 6. 修改表属性
    // flink1.11后支持
    //    tableEnv.executeSql("""alter table test_like set ('write.format.default'='avro')""")

    // 7. 写入
    //    tableEnv.executeSql("insert into test_hadoop_table values (1, 'a')")
    //    tableEnv.executeSql("insert overwrite test_hadoop_table values (2, 'a') ")
    //    tableEnv.executeSql("insert overwrite test_table PARTITION(data='b') SELECT 6")

    // 8. 读取数据
    //    tableEnv.executeSql("select * from test_hadoop_table").print()

    // 9. 写入数据
    //    val insert =
    //      """
    //        |insert into test_like
    //        |select
    //        | id, data
    //        |from test_hadoop_table
    //        |""".stripMargin
    //    tableEnv.executeSql(insert)
  }
}

hive_catalog

object TestFlinkSQLOptIcebergHiveCatalog {
  private var logger: org.slf4j.Logger = _

  def main(args: Array[String]): Unit = {
    logger = LoggerFactory.getLogger(this.getClass.getSimpleName)
    Logger.getLogger("org.apache").setLevel(Level.INFO)
    Logger.getLogger("hive.metastore").setLevel(Level.INFO)
    Logger.getLogger("akka").setLevel(Level.INFO)

    val tableEnv = FlinkUtils.initStreamTableEnvironment()

    //    val env = StreamExecutionEnvironment.getExecutionEnvironment
    //
    //    val settings = EnvironmentSettings
    //      .newInstance()
    //      .useBlinkPlanner()
    //      .inStreamingMode()
    //      .build()
    //
      // streamTable 环境
    //    val tableEnv = StreamTableEnvironment.create(env, settings)

    // batchTable 环境
    //    val settings = EnvironmentSettings
    //      .newInstance()
    //      .useBlinkPlanner()
    //      .inBatchMode()
    //      .build()
    //    val tableEnv = TableEnvironment.create(settings)

    //        val catalog_name = "hive_catalog"
    //        val database = "iceberg_test_db"
    //        val hiveConf = "F:\\workspace\\realtime-lakehouse\\test\\src\\main\\resources"
    //
    //        val hiveCatalog = new HiveCatalog(
    //          catalog_name,
    //          null,
    //          hiveConf
    //        )
    //        tableEnv.registerCatalog(catalog_name, hiveCatalog)
    //        tableEnv.getConfig.setSqlDialect(SqlDialect.HIVE)

    // catalog
    //    val catalogDDL =
    //      """
    //        |create catalog hive_catalog with (
    //        | 'type' = 'iceberg',
    //        | 'catalog-type' = 'hive',
    //        | 'uri' = 'thrift://test-lakehouse:9083',
    //        | 'clients' = '5',
    //        | 'property-version' = '1',
    //        | 'warehouse' = 'hdfs://test-lakehouse:9000/user/hive/warehouse/'
    //        |)
    //        |""".stripMargin
    //    tableEnv.executeSql(catalogDDL)

    //    两种写法
    //    //    tableEnv.executeSql("use catalog hive_catalog")
    //    tableEnv.useCatalog("hive_catalog")
    //    tableEnv.executeSql("show catalogs").print()
    //
    //    val databaseDDL = "create database if not exists iceberg_test_db"
    //    tableEnv.executeSql(databaseDDL)
    //
    //    tableEnv.useDatabase("iceberg_test_db")
    //    println(s"current database: ${tableEnv.getCurrentDatabase}")

    //    tableEnv.executeSql("show databases").print()

    //    println("list catalogs:")
    //    tableEnv.listCatalogs().foreach(println)
    //    tableEnv.listDatabases()

    //