Kettle CsvInput--filterRows--FiltOutput

最新推荐文章于 2025-02-12 00:24:12 发布

qq_35152018

最新推荐文章于 2025-02-12 00:24:12 发布

阅读量1.2k

点赞数 1

文章标签： java kettle

本文链接：https://blog.youkuaiyun.com/qq_35152018/article/details/72903014

版权

在图形化界面中：Csv文件读入--记录过滤--Csv输出过程

在java后台实现：

csvInput

public class CsvInput {
    private String filePath;

	//获取表头注入
    private String[] headers;

	//获取每一列注入 private List<String[]> body; public List<String[]> getBody() { return body; } public void setBody(List<String[]> body) { this.body = body; } public CsvInput(String filePath, String[] headers) { this.filePath
 = filePath; this.headers = headers; } public StepMeta init() { CsvInputMeta csvInputMeta = new CsvInputMeta(); csvInputMeta.setDefault(); csvInputMeta.setDelimiter(","); csvInputMeta.setFilename(filePath); csvInputMeta.setHeaderPresent(true); TextFileInputField[]
 textFileInputFields = new TextFileInputField[headers.length]; Integer[] integers = new Integer[headers.length]; for (int i = 0; i < headers.length; i++) { textFileInputFields[i] = new TextFileInputField(); textFileInputFields[i].setName(headers[i]); if(body!=null){

		//设置每列样本，判断每列类型
                textFileInputFields[i].setSamples(body.get(i));
            }
            textFileInputFields[i].guessType();
        }
        csvInputMeta.setEncoding("utf-8");
        csvInputMeta.setInputFields(textFileInputFields);
        return new StepMeta("CsvInput", "csvInput", (StepMetaInterface) csvInputMeta);
    }

}


filterRows:

    public StepMeta init() throws KettleException {

        FilterRowsMeta filterRowsMeta = new FilterRowsMeta();
        filterRowsMeta.setDefault();
        filterRowsMeta.setFalseStepname("false");
        filterRowsMeta.setTrueStepname("success");
	//设置过滤条件
        Condition condition = new Condition();
        condition.setLeftValuename(filter.leftName);
        condition.setFunction(Condition.getFunction(filter.function));
        if (filter.getType() == 2 || filter.getType() == 3) {
            condition.setRightExact(new ValueMetaAndData("constant", filter.getRightName()));
        } else {
            condition.setRightExact(new ValueMetaAndData("constant", Double.parseDouble(filter.getRightName())));
        }
        filterRowsMeta.setCondition(condition);
        return new StepMeta("FilterRows", "filterRows", (StepMetaInterface) filterRowsMeta);
    }
类型对应数字说明：

1--Number
2--string
3--data
4--Boolean
5--Integete
6--BigNumber
8--Binary
FileOutput:
    public StepMeta init() {
        TextFileOutputMeta textFileOutputMeta = new TextFileOutputMeta();
        textFileOutputMeta.setDefault();
        textFileOutputMeta.setFileName(filePath);
        textFileOutputMeta.setSeparator(",");
        textFileOutputMeta.setExtension("csv");
        textFileOutputMeta.setEncoding("utf-8");
        return new StepMeta("TextFileOutput", stepname, (StepMetaInterface) textFileOutputMeta);
    }


执行Kettle转换：
    private void executeKettle(Object pluginClass, String[] headers, String file, String filePath) throws KettleException, IOException {
        KettleEnvironment.init();
        TransMeta transMeta = new TransMeta();
        CsvInput csvInput = new CsvInput(file, headers);
        csvInput.setBody(body);
	//初始化csvInput步骤元
        StepMeta stepMeta1 = csvInput.init();
	//初始化filterRows步骤元
        StepMeta stepMeta2 = (StepMeta) pluginClass;
	//初始化FileOutput步骤元
        StepMeta stepMeta3 = new FileOutput(file, "success").init();

        transMeta.addStep(stepMeta1);
        transMeta.addStep(stepMeta2);
        transMeta.addStep(stepMeta3);

        transMeta.addTransHop(new TransHopMeta(stepMeta1, stepMeta2));//连接两个步骤
        transMeta.addTransHop(new TransHopMeta(stepMeta2, stepMeta3));

        if (this.getSaveFilter()) {
            StepMeta stepMeta4 = new FileOutput(this.getFalsePath(), "false").init();
            transMeta.addStep(stepMeta4);
            transMeta.addTransHop(new TransHopMeta(stepMeta2, stepMeta4));
        }

        String xml = transMeta.getXML();
        //ktr文件缓存位置
        String ktrPath = filePath + "ktrPath" + File.separator;
        if (!new File(ktrPath).exists()) {
            new File(ktrPath).mkdirs();
        }
        String ktr = ktrPath + UUID.randomUUID().toString() + ".ktr";
        File ktrFile = new File(ktr);
        FileUtils.writeStringToFile(ktrFile, xml, "UTF-8");
        TransMeta transMeta1 = new TransMeta(ktr);
        Trans trans = new Trans(transMeta1);
        trans.prepareExecution(null);
        trans.startThreads();
        trans.waitUntilFinished();
        FileUtils.deleteQuietly(ktrFile);
    }