yarn编码获取application列表

最新推荐文章于 2022-09-29 11:30:00 发布

nma_123456

最新推荐文章于 2022-09-29 11:30:00 发布

阅读量9.3k

点赞数 1

CC 4.0 BY-SA版权

分类专栏：大数据文章标签： yarn application

本文链接：https://blog.youkuaiyun.com/nma_123456/article/details/51692721

大数据专栏收录该内容

8 篇文章

订阅专栏

本文档介绍了如何通过编程方式获取YARN集群上的应用程序列表。主要关注的是使用YARN的API来实现这一功能。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

1、新建maven工程

2、编写代码

package Yarn.Yarn;

import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.nio.charset.Charset;
import java.text.DecimalFormat;
import java.util.EnumSet;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobID;
import org.apache.hadoop.mapred.TaskReport;
import org.apache.hadoop.mapred.JobStatus;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationReportPBImpl;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.cli.ApplicationCLI;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;

public class YarnAppManager {

	private YarnClient client;
	protected PrintStream sysout = System.out;
	private static final String APPLICATIONS_PATTERN = "%30s\t%20s\t%20s\t%10s\t%10s\t%18s\t%18s\t%15s\t%35s"
			+ System.getProperty("line.separator");
	public static void main(String[] args) {
		// TODO Auto-generated method stub
		try {
			YarnAppManager app = new YarnAppManager();
			app.testAppState();
		} catch (Exception ex) {
			ex.printStackTrace();
		}
	}
	private void getJobState(){
		Configuration conf = new Configuration();
		client = YarnClient.createYarnClient();
		client.init(conf);
		client.start();
	}

	private void testAppState() throws YarnException, IOException, InterruptedException, ClassNotFoundException {
		Configuration conf = new Configuration();
		client = YarnClient.createYarnClient();
		client.init(conf);
		client.start();
	
		EnumSet<YarnApplicationState> appStates = EnumSet.noneOf(YarnApplicationState.class);
		if (appStates.isEmpty()) {
			appStates.add(YarnApplicationState.RUNNING);
			appStates.add(YarnApplicationState.ACCEPTED);
			appStates.add(YarnApplicationState.SUBMITTED);
		}
		List<ApplicationReport> appsReport = client.getApplications(appStates);

		PrintWriter writer = new PrintWriter(new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
		for (ApplicationReport appReport : appsReport) {
			ApplicationReportPBImpl app = (ApplicationReportPBImpl) appReport;
			DecimalFormat formatter = new DecimalFormat("###.##%");
			String progress = formatter.format(appReport.getProgress());
			writer.printf(APPLICATIONS_PATTERN, appReport.getApplicationId(), appReport.getName(),
					appReport.getApplicationType(), appReport.getUser(), appReport.getQueue(),
					appReport.getYarnApplicationState(), appReport.getFinalApplicationStatus(), progress,
					appReport.getOriginalTrackingUrl());
		}
		writer.flush();
		for (ApplicationReport appReport : appsReport) {
			String type = appReport.getApplicationType();
			if(type.equalsIgnoreCase("spark")){
				continue;
			}
			getStatusByAppId(appReport);
		}

	}
	private void getStatusByAppId(ApplicationReport app){
		String user = app.getUser();
		ApplicationId id = app.getApplicationId();
		String appId = app.getApplicationId().toString();
		System.out.println(appId);
		
	}
	private void killApplication(String applicationId) throws YarnException, IOException{
		ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
	    ApplicationReport  appReport = null;
	    try {
	      appReport = client.getApplicationReport(appId);
	    } catch (ApplicationNotFoundException e) {
	      sysout.println("Application with id '" + applicationId +
	          "' doesn't exist in RM.");
	      throw e;
	    }

	    if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
	        || appReport.getYarnApplicationState() == YarnApplicationState.KILLED
	        || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) {
	      sysout.println("Application " + applicationId + " has already finished ");
	    } else {
	      sysout.println("Killing application " + applicationId);
	      client.killApplication(appId);
	    }
	}

	private void getAppState() throws Exception {
		String[] args = { "-list" };
		ApplicationCLI.main(args);
	}

}

3、pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>

	<groupId>YarnManager</groupId>
	<artifactId>Yarn</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<packaging>jar</packaging>

	<name>Yarn</name>
	<url>http://maven.apache.org</url>

	<properties>
		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
	</properties>

	<dependencies>
		<dependency>
			<groupId>junit</groupId>
			<artifactId>junit</artifactId>
			<version>3.8.1</version>
			<scope>test</scope>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-mapreduce-client-core</artifactId>
			<version>2.3.0</version>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
			<version>2.3.0</version>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-hdfs</artifactId>
			<version>2.3.0</version>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-common</artifactId>
			<version>2.3.0</version>
		</dependency>
		<dependency>
			<groupId>jdk.tools</groupId>
			<artifactId>jdk.tools</artifactId>
			<version>1.6</version>
			<scope>system</scope>
			<systemPath>C:/Program Files/Java/jdk1.8.0_73/lib/tools.jar</systemPath>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-yarn-common</artifactId>
			<version>2.3.0</version>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-yarn-client</artifactId>
			<version>2.3.0</version>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
			<version>2.3.0</version>
		</dependency>
	</dependencies>
	<build>
		<plugins>
			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-jar-plugin</artifactId>
				<configuration>
					<archive>
						<manifest>
							<mainClass>YarnManager.Yarn.YarnAppManager</mainClass>
						</manifest>
					</archive>
				</configuration>
			</plugin>
		</plugins>
	</build>
</project>