java实现kettle资源库任务执行

Java调用Kettle执行数据转换任务

原创已于 2022-06-24 11:58:38 修改 · 664 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#java #数据库 #大数据

于 2022-06-24 11:51:26 首次发布

本文介绍如何使用Java编程方式与Kettle（Pentaho Data Integration）交互，实现对Kettle资源库中的数据转换任务进行执行，进而完成大数据处理工作。通过Java API，可以灵活地在应用程序中集成Kettle的ETL能力。

/**
     * 2.通过文件方式执行job
     *
     * @throws KettleException
     */
    public boolean runWithDb(String path, String transName, StringBuffer outFileName, String isTrans) throws KettleException {
        synchronized (KettelUtil.class){
            KettleEnvironment.init();
        }

        //创建DB资源库
        KettleDatabaseRepository repository = new KettleDatabaseRepository();
//        DatabaseMeta databaseMeta=new DatabaseMeta("kettle","mysql","jdbc","localhost","kettle","3306","root","root");
        DatabaseMeta databaseMeta = new MyDatabaseMeta(kettleConfig.getRepositoryName(), kettleConfig.getJdbcType()
                , kettleConfig.getAccess(), kettleConfig.getHost(), kettleConfig.getDb()
                , kettleConfig.getPort(), kettleConfig.getUser(), kettleConfig.getPass());
        //选择资源库
        KettleDatabaseRepositoryMeta kettleDatabaseRepositoryMeta = new KettleDatabaseRepositoryMeta(kettleConfig.getRepositoryId()
                , kettleConfig.getRepositoryName(), kettleConfig.getRepositoryDescription(), databaseMeta);
        repository.init(kettleDatabaseRepositoryMeta);
        //连接资源库
        repository.connect(kettleConfig.getRepositoryUserName(), kettleConfig.getRepositoryPass());
        RepositoryDirectoryInterface directoryInterface = repository.findDirectory(path);
        //选择转换 transMeta.hops.get(1).to_step.getStepMetaInterface()
        if ("1".equals(isTrans)) {
            TransMeta transMeta = repository.loadTransformation(transName, directoryInterface, null, true, null);
            Trans trans = new Trans(transMeta);
            //默认参数
            trans.setParameterValue("today", DateUtil.today().replace("-", ""));
           
            //查找输出文件
            findOutFileName(trans, outFileName);

            trans.execute(null);
            trans.waitUntilFinished();//等待直到数据结束
            if (trans.getErrors() > 0) {
                return false;
            } else {
                return true;
            }
        } else {
            JobMeta jobMeta = repository.loadJob(transName, directoryInterface, null, null);
            Job job = new Job(repository, jobMeta);
            jobMeta.setParameterValue("today", DateUtil.today().replace("-", ""));
            findOutFileName(job,outFileName,repository);
            job.start();
            job.waitUntilFinished();
            if (job.getErrors() > 0) {
                return false;
            }
            return true;
        }

    }

 //解决kettle低版本兼容mysql8
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.exception.KettleDatabaseException;

public class MyDatabaseMeta  extends DatabaseMeta {

    public MyDatabaseMeta(String name, String type, String access, String host, String db, String port, String user, String pass) {
        super(name, type, access, host, db, port, user, pass);
    }

    public MyDatabaseMeta() {
      super();
    }

    @Override
    public String getDriverClass() {
        String driverClass = super.getDriverClass();
        return driverClass.replace("org.gjt.mm.mysql.Driver","com.mysql.cj.jdbc.Driver");
    }

    @Override
    public String getURL() throws KettleDatabaseException {
        return super.getURL()+"&useSSL=false&allowPublicKeyRetrieval=true";
    }


}