Java执行Datax脚本-MySQL写入Doris

1、生成DataX json模板

    public static void josnTemplate(String source,String target,String datasetId) {
        String classPath = "D:/software/datax";
        String jobFilePath = "D:/software/datax/job/";
        System.setProperty("datax.home", classPath);

        String[] arguments = new String[]{"python", classPath+"/bin/datax.py", "-r", source.toLowerCase()+"reader", "-w", target.toLowerCase()+"writer"};
        try {
            Process process = Runtime.getRuntime().exec(arguments);
            BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream(), "UTF-8"));
            BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(
                    new FileOutputStream(jobFilePath + "dataset_"+datasetId), "UTF-8"));

            String line = null;
            int count = 0;
            while ((line = br.readLine()) != null) {
                count++;
                if (count > 15) {
                    //System.out.println(line);

                    bw.write(line);
                    bw.newLine();
                    bw.flush();
                }
            }
            bw.close();
            br.close();
            //java代码中的process.waitFor()返回值为0表示我们调用python脚本成功，
            //返回值为1表示调用python脚本失败，这和我们通常意义上见到的0与1定义正好相反
            int re = process.waitFor();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

2、组装DataX json模板

    public static String getJson(String filePath) {
        String json = "";
        try {
            File file = new File(filePath);
            FileReader reader = new FileReader(file);
            FileInputStream inputStream = new FileInputStream(file);
            Reader stream = new InputStreamReader(inputStream, Charset.defaultCharset());

            int ch = 0;
            StringBuilder sb = new StringBuilder();
            while ((ch = reader.read()) != -1) {
                sb.append((char) ch);
            }
            reader.close();
            stream.close();

            JSONObject jobj = JSONObject.parseObject(sb.toString());
            json = jobj.toJSONString();
        } catch (Exception e) {
            e.printStackTrace();
        }
        return json;
    }

    public static String getDataXJson(String filePath) throws FileNotFoundException {
        String json = getJson(filePath);
        JSONObject jobj = JSONObject.parseObject(json);
        //组装reader
        JSONObject reader = jobj.getJSONObject("job").getJSONArray("content").getJSONObject(0).getJSONObject("reader");
        JSONObject rp = reader.getJSONObject("parameter");
        rp.put("username", "root");
        rp.put("password", "123456");
        rp.remove("column");//删除column元素

        JSONObject rc = rp.getJSONArray("connection").getJSONObject(0);
        JSONArray jdbcUrl = new JSONArray();
        jdbcUrl.add("jdbc:mysql://IP:3306/dataset?useUnicode=true&characterEncoding=utf-8&useSSL=false&autoReconnect=true&allowMultiQueries=true&serverTimezone=Asia/Shanghai");
        rc.put("jdbcUrl", jdbcUrl);
        rc.remove("table");//删除table元素
        JSONArray querySql = new JSONArray();
        querySql.add("select uuid() as fusion_uuid,a,b,c,d,e,f,h from t_number");
        rc.put("querySql", querySql);
        //组装writer
        JSONObject writer = jobj.getJSONObject("job").getJSONArray("content").getJSONObject(0).getJSONObject("writer");
        JSONObject wp = writer.getJSONObject("parameter");
        wp.put("username", "root");
        wp.put("password", "123456");
        JSONArray column = new JSONArray();
        column.add("*");
        wp.put("column", column);
        JSONArray preSql = new JSONArray();
        preSql.add("truncate table dataset_1619888763184439297");
        wp.put("preSql", preSql);
        JSONArray loadUrl = new JSONArray();
        loadUrl.add("IP:8030");
        wp.put("loadUrl", loadUrl);

        JSONObject wc = wp.getJSONArray("connection").getJSONObject(0);
        wc.put("jdbcUrl", "jdbc:mysql://IP:9030/fusion");
        JSONArray table = new JSONArray();
        table.add("dataset_1619888763184439297");
        wc.put("table", table);
        wc.put("selectedDatabase", "fusion");
        //组装setting
        JSONObject setting = jobj.getJSONObject("job").getJSONObject("setting");
        JSONObject speed = setting.getJSONObject("speed");
        speed.put("channel", "1");
		//组装errorLimit
        JSONObject errorlimit = new JSONObject();
        errorlimit.put("percentage",0.02);
        errorlimit.put("record",0);
        setting.put("errorLimit",errorlimit);
        //写入文本
        PrintStream stream= new PrintStream(filePath,"UTF-8");
        //stream.println(jobj);
        
        return jobj.toString();
    }

3、执行DataX job

    public static void execute(String filePath) throws Exception {
        String dataxHome = "D:/software/datax";
        System.setProperty("datax.home", dataxHome);

        String dataXJson = getDataXJson(filePath);
        //PrintStream stream= new PrintStream(filePath);
        //stream.println(dataXJson);

        //String[] datxArgs = {"-job", getClasspath() + "/datax/stream.json", "-mode", "standalone", "-jobid", "-1"};
        String[] dataXArgs = {"-job", filePath, "-mode", "standalone", "-jobid", "-1"};

        try {
            Engine.entry(dataXArgs);
        }catch (DataXException e){
            e.printStackTrace();
        }
        catch (Throwable throwable) {
            /**
             * 拿到异常信息入库.
             */
            log.error("\n\n经DataX智能分析,该任务最可能的错误原因::\n" + ExceptionTracker.trace(throwable));
        }

    }

    public static void main(String[] args) throws Exception {
        josnTemplate("mysql","doris","1619879625037537281");
        getDataXJson("D:/software/datax/job/dataset_1619879625037537281");
        execute("D:/software/datax/job/dataset_1619879625037537281");

    }