Flink 自定义UDTF

最新推荐文章于 2024-10-09 23:50:47 发布

訾零

最新推荐文章于 2024-10-09 23:50:47 发布

阅读量1.8k

点赞数

分类专栏： Flink 文章标签： flink

本文链接：https://blog.youkuaiyun.com/lingeio/article/details/119599245

版权

本文档展示了如何在Flink中实现一个Java和Scala版本的自定义函数（UDF），用于将Json格式的数据转换为Row类型。通过示例代码，解释了如何处理Json数组并提取字段，以及如何在Flink SQL中注册和使用这些函数。此外，还提供了完整的测试用例和依赖管理配置。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

Flink自定义函数实现列传行，数据格式为Json数据：[{"key1":"value1","key2":"value2"...}]

代码

Java

@FunctionHint(output = @DataTypeHint("ROW<drugUniversalName string, specifications string, goodsUnit string, " +
        "location string, instruction string, consumption string, consumptionUnit string, frequency string, " +
        "prescriptionsAmount string, prescriptionsUnit string, goodsNo string>"))
public class JsonArrayParseUDF extends TableFunction<Row> {
    private static final Logger logger = Logger.getLogger(JsonArrayParseUDF.class);

    public void eval(String json) {
        if (StringUtils.isNullOrWhitespaceOnly(json)) {
            return;
        }
        String drugUniversalName = null;
        String specifications = null;
        String goodsUnit = null;
        String location = null;
        String instruction = null;
        String consumption = null;
        String consumptionUnit = null;
        String frequency = null;
        String prescriptionsAmount = null;
        String prescriptionsUnit = null;
        String goodsNo = null;

        try {
            Gson gson = new Gson();
            JsonArray jsonArray = gson.fromJson(json, JsonArray.class);

            for (JsonElement jsonElement : jsonArray) {
                JsonObject jsonObject = jsonElement.getAsJsonObject();

                JsonElement drugUniversalNameTmp = jsonObject.get("drugUniversalName");
                drugUniversalName = invalidate(drugUniversalNameTmp);

                JsonElement specificationsTmp = jsonObject.get("specifications");
                specifications = invalidate(specificationsTmp);

                JsonElement goodsUnitTmp = jsonObject.get("goodsUnit");
                goodsUnit = invalidate(goodsUnitTmp);

                JsonElement locationTmp = jsonObject.get("location");
                location = invalidate(locationTmp);

                JsonElement instructionTmp = jsonObject.get("instruction");
                instruction = invalidate(instructionTmp);

                JsonElement consumptionTmp = jsonObject.get("consumption");
                consumption = invalidate(consumptionTmp);

                JsonElement consumptionUnitTmp = jsonObject.get("consumptionUnit");
                consumptionUnit = invalidate(consumptionUnitTmp);

                JsonElement frequencyTmp = jsonObject.get("frequency");
                frequency = invalidate(frequencyTmp);

                JsonElement prescriptionsAmountTmp = jsonObject.get("prescriptionsAmount");
                prescriptionsAmount = invalidate(prescriptionsAmountTmp);

                JsonElement prescriptionsUnitTmp = jsonObject.get("prescriptionsUnit");
                prescriptionsUnit = invalidate(prescriptionsUnitTmp);

                JsonElement goodsNoTmp = jsonObject.get("goodsNo");
                goodsNo = invalidate(goodsNoTmp);

                Row row = Row.of(drugUniversalName, specifications, goodsUnit, location, instruction, consumption,
                        consumptionUnit, frequency, prescriptionsAmount, prescriptionsUnit, goodsNo);
                // System.out.println(row);
                collect(row);
            }

        } catch (Exception e) {
            logger.error("json parser failed :" + e.getMessage());
        }
    }

    public String invalidate(JsonElement jsonElement) {
        if (jsonElement != null) {
            return jsonElement.getAsString();
        } else {
            return "";
        }
    }

//    public static void main(String[] args) {
//        JsonArrayParseUDF parseUDF = new JsonArrayParseUDF();
//        String str = "[{\"drugUniversalName\":\"达格列净片\",\"specifications\":\"10mg*10片*3板\",\"goodsUnit\":\"盒\",\"location\":\"\",\"instruction\":\"口服\",\"consumption\":\"2.0\",\"consumptionUnit\":\"片\",\"frequency\":\"1日1次\",\"prescriptionsAmount\":\"5\",\"prescriptionsUnit\":\"盒\",\"countDosage\":\"\",\"description\":\"(安达唐)达格列净片10mg*10片*3板阿斯利康\",\"goodsNo\":\"1029990\"}]\n";