Java版本Flink写Clickhouse

最新推荐文章于 2025-07-06 14:41:43 发布

原创最新推荐文章于 2025-07-06 14:41:43 发布 · 427 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#flink #clickhouse #java #大数据 #数据仓库

flink 专栏收录该内容

3 篇文章

订阅专栏

该文章展示了如何在Java中使用ApacheFlink将数据流连接到Clickhouse数据库。通过引入Flink和Clickhouse的Maven依赖，创建JdbcSink，设置SQL语句和连接选项，将数据从Socket源转换并写入Clickhouse表。

Java版本Flink写Clickhouse

引入maven依赖

<dependency>
	<groupId>org.apache.flink</groupId>
	<artifactId>flink-connector-jdbc_${scala.binary.version}</artifactId>
	<version>${flink.version}</version>
</dependency>
<dependency>
	<groupId>com.clickhouse</groupId>
	<artifactId>clickhouse-jdbc</artifactId>
	<version>0.4.0</version>
</dependency>

代码实现

import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.connector.jdbc.JdbcConnectionOptions;
import org.apache.flink.connector.jdbc.JdbcExecutionOptions;
import org.apache.flink.connector.jdbc.JdbcSink;
import org.apache.flink.connector.jdbc.JdbcStatementBuilder;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;

public class FLinkClickHouseSinkApp {

    public static void main(String[] args) throws Exception {

        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        env.setParallelism(1);

        // 通过socket接受输入数据
        DataStreamSource<String> stream = env.socketTextStream("192.168.49.194", 9999);

        // 数据转换
        stream.map(new MapFunction<String, User>() {
            @Override
            public User map(String value) throws Exception {
                String[] split = value.split(",");
                return new User(Integer.parseInt(split[0]), Integer.parseInt(split[1]), split[2]);
            }
        }).addSink(getJdbcSink("insert into test_user values(?,?,?)"));

        // 执行
        env.execute();
    }

    private static SinkFunction<User> getJdbcSink(String sql) {
        // SQL预编译
        JdbcStatementBuilder<User> statementBuilder = (statement, user) -> {
            statement.setInt(1, user.id);
            statement.setInt(2, user.age);
            statement.setString(3, user.userName);
        };

        // 配置
        JdbcExecutionOptions options = JdbcExecutionOptions.builder()
                .withBatchSize(1500)
                .withBatchIntervalMs(250)
                .withMaxRetries(5)
                .build();

        // 连接信息
        JdbcConnectionOptions connectionOptions = new JdbcConnectionOptions.JdbcConnectionOptionsBuilder()
                .withUrl("jdbc:ch:http://192.168.49.194:8123/db_test")
                .withDriverName("com.clickhouse.jdbc.ClickHouseDriver")
                .build();

        return JdbcSink.sink(sql, statementBuilder, options, connectionOptions);
    }


    // 自定义实体类
    public static class User {

        public int id;

        public int age;

        public String userName;

        public User() {
        }

        public User(int id, int age, String userName) {
            this.id = id;
            this.age = age;
            this.userName = userName;
        }
    }
}