flink jdbc实战

最新推荐文章于 2025-01-24 09:31:08 发布

DogTips

最新推荐文章于 2025-01-24 09:31:08 发布

阅读量295

点赞数

CC 4.0 BY-SA版权

文章标签： flink java 大数据

本文链接：https://blog.youkuaiyun.com/DogTips/article/details/127003312

本文详细介绍了如何使用Apache Flink进行JDBC操作，包括数据源配置、数据读写及实战案例，深入理解Flink在大数据处理中的应用。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

public class MySinkUtil {
    
    public static <T> SinkFunction<T> getClickHouseSink(String db,
                                                                   String table,
                                                                   Class<T> tClass) {
        
        String url = "jdbc:clickhouse://hadoop162:8123/" + db;
        String driver = "ru.yandex.clickhouse.ClickHouseDriver";
        
        // insert into visitor_stats_2021(c1, c2, c3,...) values(?,?,?...)
        StringBuilder sql = new StringBuilder();
        sql.append("insert into ").append(table).append("(");
        // TODO
        final Field[] fields = tClass.getDeclaredFields();
        for (Field field : fields) {
            // 如果属性被transient修饰, 则表示不会写入到ClickHouse中
            if (field.getAnnotation(TransientSink.class) == null) {
                final String fieldName = field.getName();
                sql.append(fieldName).append(",");
            }
        }
        // 去掉最后一个逗号
        sql.deleteCharAt(sql.length() - 1);
        sql.append(" ) values ( ");
        for (Field field : fields) {
            if (field.getAnnotation(TransientSink.class) == null) {
                sql.append("?,");
            }
        }
        sql.deleteCharAt(sql.length() - 1);
        sql.append(" )");
        
        return getJDBCSink(url, driver, sql.toString());
    }
    
    public static <T> SinkFunction<T> getJDBCSink(String url,
                                                  String driver,
                                                  String sql) {
        // jdbc connector 只支持sink
        return JdbcSink.<T>sink(sql,
                                new JdbcStatementBuilder<T>() {
                                    @Override
                                    public void accept(PreparedStatement ps,
                                                       T t) throws SQLException {
                                        // 3种获取class:  A.class  Class.forName(字符串类名)  对象.getClass
                                        final Field[] fields = t.getClass().getDeclaredFields();
                                        try {
                                            for (int i = 0, position = 1; i < fields.length; i++) {
                                                if (fields[i].getAnnotation(TransientSink.class) == null) {
                                                    fields[i].setAccessible(true);
                                                    ps.setObject(position++, fields[i].get(t));
                                                }
                                            }
                                        } catch (IllegalAccessException e) {
                                            e.printStackTrace();
                                        }
                                    }
                                },
                                new JdbcConnectionOptions.JdbcConnectionOptionsBuilder()
                                    .withUrl(url)
                                    .withDriverName(driver)
                                    .build());
    }
    
}