Flink-doris-connector sql写入doris on zeppelin优化版

要是觉得写得不错，请务必点个赞，谢谢啊！！！

 

目录

要是觉得写得不错，请务必点个赞，谢谢啊！！！

一，代码开发

1，写入doris的案例代码

2，指定DorisDynamicTableSourceFactory

3，DorisDynamicTableSourceFactory 具体实现

4，DorisSinkFunction具体实现

5，队列类实现

6，doris sink代码，这个比较简单。注意传参就ok了。

二，打包细节

三，zeppelin执行

1，引入依赖：

2，创建语句

3，多个sql执行

四，执行效果

五，总结

---

 

一，代码开发

1，写入doris的案例代码

public class DorisSqlTest2 {
    private  static final String DORIS_SQL =  "CREATE TABLE doris_test (\n"+
            " order_number STRING,\n" +
            " order_key STRING,\n" +
            " stock_order STRING,\n" +
            " stock_code STRING,\n" +
            " intermediary STRING,\n" +
            " intermediary_name STRING,\n" +
            " intermediary_accountid STRING,\n" +
            " intermediary_phone STRING,\n" +
            " canal_type STRING\n" +
            " ) WITH (\n"+
            "   'connector' = 'doris',\n"+
            "   'doris.host' = '192.168.6.143',\n"+
            "   'doris.port' = '8030',\n"+
            "   'database-name' = 'example_db',\n"+
            "   'table-name' = 'assure_orders2',\n"+
            "   'username' = 'root',\n"+
            "   'password' = 'root',\n"+
            "   'max.batch' = '500'\n"+
            " )";



    private static final String DATA_GEN =  "CREATE TABLE datagen (\n" +
            " id STRING,\n" +
            " name STRING,\n" +
            " user_age INT,\n" +
            " user_other STRING,\n" +
            " ts AS localtimestamp\n" +
            ") WITH (\n" +
            " 'connector' = 'datagen',\n" +
            " 'rows-per-second'='500',\n" +
            " 'fields.id.length'='7',\n" +
            " 'fields.user_age.min'='1',\n" +
            " 'fields.user_age.max'='100',\n" +
            " 'fields.name.length'='2',\n" +
            " 'fields.user_other.length'='10'\n" +
            ")";



    private  static final String KAFKA_SQL = "CREATE TABLE kafka_test (\n" +
           " order_number STRING,\n" +
            " order_key STRING,\n" +
            " stock_order STRING,\n" +
            " stock_code STRING,\n" +
            " intermediary STRING,\n" +
            " intermediary_name STRING,\n" +
            " intermediary_accountid STRING,\n" +
            " intermediary_phone STRING,\n" +
            " canal_type STRING\n" +
            ") WITH (\n" +
            " 'connector' = 'kafka',\n" +
            " 'topic' = 'dwd.rds_core.plateform_stable.assure_orders',\n" +
            " 'properties.bootstrap.servers' = 'dev-ct6-dc-worker01:9092,dev-ct6-dc-worker02:9092,dev-ct6-dc-worker03:9092',\n" +
            " 'properties.group.id' = 'testGroup',\n" +
            " 'format' = 'json',\n" +
            " 'scan.startup.mode' = 'earliest-offset'\n" +
            ")";

    public static void main(String[] args) {

//        System.out.println("DORIS_SQL = " + DORIS_SQL);
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(3);
        env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

        EnvironmentSettings bsSettings = EnvironmentSettings.newInstance().useBlinkPlanner().inStreamingMode().build();
//		StreamTableEnvironment bsTableEnv = StreamTableEnvironment.create(env, bsSettings);
        StreamTableEnvironment bsTableEnv = StreamTableEnvironment.create(env);
        bsTableEnv.executeSql(DORIS_SQL);
        bsTableEnv.executeSql(DATA_GEN);
        bsTableEnv.executeSql(KAFKA_SQL);

        bsTableEnv.executeSql("insert into doris_test select order_number,order_key,stock_order,stock_code,intermediary,intermediary_name,intermediary_accountid,intermediary_phone,canal_type from kafka_test");
//        bsTableEnv.executeSql("insert into kafkaTable select id,name from datagen");
    }
}

2，指定DorisDynamicTableSourceFactory

创建相对应的路径，并且创建类

 

 

3，DorisDynamicTableSourceFactory 具体实现

对应的就是sql里面的参数和类型

package org.apache.flink.connector.doris.table;

import org.apache.flink.configuration.ConfigOption;
import org.apache.flink.configuration.ConfigOptions;
import org.apache.flink.configuration.ReadableConfig;
import org.apache.flink.table.api.TableSchema;
import org.apache.flink.table.connector.sink.DynamicTableSink;
import org.apache.flink.table.factories.DynamicTableSinkFactory;
import org.apache.flink.table.factories.FactoryUtil;
import org.apache.flink.table.utils.TableSchemaUtils;

import java.time.Duration;
import java.util.HashSet;
import java.util.Set;

/**
 * @program: flink-neiwang-dev
 * @description: 1
 * @author: Mr.Wang
 * @create: 2020-11-12 14:04
 **/
public class DorisDynamicTableSourceFactory implements DynamicTableSinkFactory {
    //todo 名称叫doris
    public static final String IDENTIFIER = "doris";

    public static final ConfigOption<String> DORIS_HOST = ConfigOptions
            .key("doris.host")
            .stringType()
            .noDefaultValue()
            .withDescription("the doris database url.");


    public static final ConfigOption<String> DORIS_HTTP_PORT = ConfigOptions
            .key("doris.port")
            .stringType()
            .noDefaultValue()
            .withDescription("the doris database url.");


    public static final ConfigOption<String> DATABASE_NAME = ConfigOptions
            .key("database-name")
            .stringType()
            .noDefaultValue()
            .withDescription("the database name.");


    public static final ConfigOption<String> TABLE_NAME = ConfigOptions
            .key("table-name")
            .stringType()
            .noDefaultValue()
            .withDescription("the jdbc table name.");
    public static final ConfigOption<String> USERNAME = ConfigOptions
            .key("username")
            .stringType()
            .noDefaultValue()
            .withDescription("the jdbc user name.");
    public static final ConfigOption<String&g