flink java提交任务，根据jobname获取jobid，rest获取任务的状态

原创已于 2022-06-19 12:38:42 修改 · 8.3w 阅读

40 ·

CC 4.0 BY-SA版权

文章标签：

#flink

于 2020-08-03 17:27:54 首次发布

flink 专栏收录该内容

29 篇文章

订阅专栏

本文介绍如何在Apache Flink中使用Java编程实现任务的创建、上传与控制，包括通过REST API获取任务ID及监测任务状态的方法。通过具体代码示例，详细展示了如何定义并执行Flink流处理任务，以及如何利用Apache HttpClient组件通过REST API进行任务状态的实时监测与任务的终止操作。

1.传入jobname:
在这里插入图片描述

public class ReduceDemo {

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        DataStreamSource<String> words = env.socketTextStream("localhost", 8888);
        SingleOutputStreamOperator<Tuple2<String, Integer>> wordAndOne = words.map(w -> Tuple2.of(w, 1)).returns(Types.TUPLE(Types.STRING, Types.INT));
        KeyedStream<Tuple2<String, Integer>, Tuple> keyBy = wordAndOne.keyBy(0);
        /*sum*/
        SingleOutputStreamOperator<Tuple2<String, Integer>> reduce = keyBy.reduce(new ReduceFunction<Tuple2<String, Integer>>() {
            @Override
            public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2)  {
                String key = value1.f0;
                Integer count1 = value1.f1;
                Integer count2 = value2.f1;
                int count = count1 + count2;
                return Tuple2.of(key, count);
            }
        });
        reduce.print();
        env.execute("cz");
    }
}

2.上传任务:

public class Upjar {
    public static void main(String[] args) throws IOException {
        List<String> arg = new ArrayList<>();
        arg.add("sh");
        arg.add("-c");
        arg.add("./flink run -p 2  -C file://绝对路径 -C file://绝对路径 -C file://绝对路径 -c 主类路径 主类所在jar包的绝对路径\n");
        ProcessBuilder pb = new ProcessBuilder(arg);
        //设置工作目录
        pb.directory(new File("/home/flink/flink-1.10.0/bin"));
        //redirectErrorStream 属性默认值为false，意思是子进程的标准输出和错误输出被发送给两个独立的流，可通过 Process.getInputStream() 和 Process.getErrorStream() 方法来访问
        //值设置为 true，标准错误将与标准输出合并。合并的数据可从 Process.getInputStream() 返回的流读取，而从 Process.getErrorStream() 返回的流读取将直接到达文件尾
        pb.redirectErrorStream(true);
        File log = new File("/home/flink/flink-1.10.0/logs");
        pb.redirectOutput(ProcessBuilder.Redirect.appendTo(log));
        Process p = pb.start();
        assert pb.redirectInput() == ProcessBuilder.Redirect.PIPE;
        //重定向标准输出到日志
        assert pb.redirectOutput().file() == log;
        assert p.getInputStream().read() == -1;

    }
}

3.获取jobid
在这里插入图片描述
通过rest访问根据jobname获取jobid,可使用 apache的httpcomponents,根据返回值获取jobid

public class HttpComponentsForRest {
    static String jobName = "97c5216de4ef4b15b16de3a3ab53fd78";
    static String urlPrefix = "http://192.168.1.127:8081";

    public static void main(String[] args) throws IOException {
        String jobID = getJobID(urlPrefix, jobName);
//        getJobState(urlPrefix, jobID);
//        stop(urlPrefix, jobID);


        try {
            while (true) {
                getJobState(urlPrefix, jobID);
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            System.out.println("pipeline stop");
        }


    }

    /*若任务的状态为running，则调用api结束任务*/
    public static void stop(String urlPrefix, String jid) throws IOException {
        CloseableHttpResponse stopJob = send(new HttpPatch(urlPrefix + "/jobs/" + jid));
        StatusLine statusLine = stopJob.getStatusLine();
        int statusCode = statusLine.getStatusCode();
        /*202代表执行成功，关闭pipeline*/
        if (statusCode != 202) {
            throw new RuntimeException("当前任务未能正确关闭");
        }

    }

    /*判断当前任务的状态   "state": "RUNNING" */
    public static void getJobState(String urlPrefix, String jid) throws IOException {
        CloseableHttpResponse getJobState = send(new HttpGet(urlPrefix + "/jobs/" + jid));
        String getJobStateRes = transform(getJobState);
        JSONObject jsonObject = JSON.parseObject(getJobStateRes);
        String state = jsonObject.getString("state");
        if (!"RUNNING".equals(state)) {
            throw new RuntimeException("当前任务处于停止或故障状态");
        }
    }

    /*根据jobname获取jobid*/
    public static String getJobID(String urlPrefix, String jobName) throws IOException {
        CloseableHttpResponse getJobID = send(new HttpGet(urlPrefix + "/jobs/overview"));
        String getJobIDRes = transform(getJobID);
        JSONObject res = JSON.parseObject(getJobIDRes);
        JSONArray jobs = res.getJSONArray("jobs");
        String jid = null;
        for (int i = 0; i < jobs.size(); i++) {
            JSONObject jsonObject = (JSONObject) jobs.get(i);
            String name = (String) jsonObject.get("name");
            if (name.equals(jobName)) {
                jid = (String) jsonObject.get("jid");
                break;
            }
        }
        if (jid == null || jid.equals("")) {
            throw new RuntimeException("pipeline在flink中没有对应的任务");
        }
        return jid;
    }

    public static CloseableHttpResponse send(HttpRequestBase httpRequestBase) throws IOException {
        CloseableHttpClient httpclient = HttpClients.createDefault();
        CloseableHttpResponse response = httpclient.execute(httpRequestBase);
        return response;
    }

    public static String transform(CloseableHttpResponse response) throws IOException {
        HttpEntity entity = response.getEntity();
        String result = EntityUtils.toString(entity);
        return result;
    }


}

关于flink中任务通过rest访问控制状态，可参考官方文档