Kettle CsvInput--filterRows--FiltOutput

最新推荐文章于 2024-04-07 14:44:07 发布

原创最新推荐文章于 2024-04-07 14:44:07 发布 · 1.2k 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#java #kettle

本文介绍了一个使用Java实现的CSV文件处理流程，包括读取CSV文件、应用过滤规则筛选数据及输出处理后的CSV文件。该流程利用了Kettle的ETL功能进行数据处理，并通过条件配置实现了灵活的数据过滤。

在图形化界面中：Csv文件读入--记录过滤--Csv输出过程

在java后台实现：

csvInput

public class CsvInput {
    private String filePath;

	//获取表头注入
    private String[] headers;

	//获取每一列注入

private List<String[]> body; public List<String[]> getBody() { return body; } public void setBody(List<String[]> body) { this.body = body; } public CsvInput(String filePath, String[] headers) { this.filePath = filePath; this.headers = headers; } public StepMeta init() { CsvInputMeta csvInputMeta = new CsvInputMeta(); csvInputMeta.setDefault(); csvInputMeta.setDelimiter(","); csvInputMeta.setFilename(filePath); csvInputMeta.setHeaderPresent(true); TextFileInputField[] textFileInputFields = new TextFileInputField[headers.length]; Integer[] integers = new Integer[headers.length]; for (int i = 0; i < headers.length; i++) { textFileInputFields[i] = new TextFileInputField(); textFileInputFields[i].setName(headers[i]); if(body!=null){

		//设置每列样本，判断每列类型

                textFileInputFields[i].setSamples(body.get(i));
            }
            textFileInputFields[i].guessType();
        }
        csvInputMeta.setEncoding("utf-8");
        csvInputMeta.setInputFields(textFileInputFields);
        return new StepMeta("CsvInput", "csvInput", (StepMetaInterface) csvInputMeta);
    }

}

filterRows:

    public StepMeta init() throws KettleException {

        FilterRowsMeta filterRowsMeta = new FilterRowsMeta();
        filterRowsMeta.setDefault();
        filterRowsMeta.setFalseStepname("false");
        filterRowsMeta.setTrueStepname("success");

	//设置过滤条件
        Condition condition = new Condition();
        condition.setLeftValuename(filter.leftName);
        condition.setFunction(Condition.getFunction(filter.function));
        if (filter.getType() == 2 || filter.getType() == 3) {
            condition.setRightExact(new ValueMetaAndData("constant", filter.getRightName()));
        } else {
            condition.setRightExact(new ValueMetaAndData("constant", Double.parseDouble(filter.getRightName())));
        }
        filterRowsMeta.setCondition(condition);
        return new StepMeta("FilterRows", "filterRows", (StepMetaInterface) filterRowsMeta);
    }

类型对应数字说明：

1--Number

2--string

3--data

4--Boolean

5--Integete

6--BigNumber

8--Binary

FileOutput:

    public StepMeta init() {
        TextFileOutputMeta textFileOutputMeta = new TextFileOutputMeta();
        textFileOutputMeta.setDefault();
        textFileOutputMeta.setFileName(filePath);
        textFileOutputMeta.setSeparator(",");
        textFileOutputMeta.setExtension("csv");
        textFileOutputMeta.setEncoding("utf-8");
        return new StepMeta("TextFileOutput", stepname, (StepMetaInterface) textFileOutputMeta);
    }

执行Kettle转换：

    private void executeKettle(Object pluginClass, String[] headers, String file, String filePath) throws KettleException, IOException {
        KettleEnvironment.init();
        TransMeta transMeta = new TransMeta();
        CsvInput csvInput = new CsvInput(file, headers);
        csvInput.setBody(body);

	//初始化csvInput步骤元
        StepMeta stepMeta1 = csvInput.init();

	//初始化filterRows步骤元
        StepMeta stepMeta2 = (StepMeta) pluginClass;

	//初始化FileOutput步骤元
        StepMeta stepMeta3 = new FileOutput(file, "success").init();

        transMeta.addStep(stepMeta1);
        transMeta.addStep(stepMeta2);
        transMeta.addStep(stepMeta3);

        transMeta.addTransHop(new TransHopMeta(stepMeta1, stepMeta2));//连接两个步骤
        transMeta.addTransHop(new TransHopMeta(stepMeta2, stepMeta3));

        if (this.getSaveFilter()) {
            StepMeta stepMeta4 = new FileOutput(this.getFalsePath(), "false").init();
            transMeta.addStep(stepMeta4);
            transMeta.addTransHop(new TransHopMeta(stepMeta2, stepMeta4));
        }

        String xml = transMeta.getXML();
        //ktr文件缓存位置
        String ktrPath = filePath + "ktrPath" + File.separator;
        if (!new File(ktrPath).exists()) {
            new File(ktrPath).mkdirs();
        }
        String ktr = ktrPath + UUID.randomUUID().toString() + ".ktr";
        File ktrFile = new File(ktr);
        FileUtils.writeStringToFile(ktrFile, xml, "UTF-8");
        TransMeta transMeta1 = new TransMeta(ktr);
        Trans trans = new Trans(transMeta1);
        trans.prepareExecution(null);
        trans.startThreads();
        trans.waitUntilFinished();
        FileUtils.deleteQuietly(ktrFile);
    }