Spark 打包 fat jar pom配置

最新推荐文章于 2022-08-02 19:36:23 发布

原创最新推荐文章于 2022-08-02 19:36:23 发布 · 992 阅读

0 ·

CC 4.0 BY-SA版权

Spark 同时被 2 个专栏收录

88 篇文章

订阅专栏

Spark

68 篇文章

订阅专栏

本文详细介绍了如何使用Maven配置pom.xml文件，以打包Spark应用程序为fat jar，包括依赖管理和构建过程，确保所有库都包含在jar中，以便于分布式运行。

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<groupId>myselfProject1</groupId>
	<artifactId>myselfProject1</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<name>myselfProject1</name>
	<description>myselfProject1</description>
	<properties>
		<org.springframework-version>4.0.6.RELEASE</org.springframework-version>
		<common-version>2.6.0</common-version>
		<zookeeper-version>3.4.0</zookeeper-version>
	</properties>
	<dependencies>
		<!-- spark -->
		<dependency>
			<groupId>org.apache.spark</groupId>
			<artifactId>spark-streaming_2.11</artifactId>
			<version>2.1.0</version>
		</dependency>

		<dependency>
			<groupId>org.apache.spark</groupId>
			<artifactId>spark-core_2.10</artifactId>
			<version>2.1.0</version>
		</dependency>

		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-client</artifactId>
			<version>2.2.0</version>
		</dependency>
		<dependency>
			<groupId>org.apache.spark</groupId>
			<artifactId>spark-streaming_2.10</artifactId>
			<version>2.1.0</version>
		</dependency>
		<dependency>
			<groupId>org.apache.spark</groupId>
			<artifactId>spark-streaming-kafka-0-8_2.11</artifactId>
			<version>2.1.0</version>
		</dependency>

	</dependencies>
	
	<!-- 指定把spark用到的jar包打进去 -->
	<build>
		<plugins>
			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-compiler-plugin</artifactId>
				<configuration>
					<source>1.7</source>
					<target>1.7</target>
					<encoding>UTF-8</encoding>
				</configuration>
			</plugin>

			
			<plugin>
				<artifactId>maven-assembly-plugin</artifactId>
				<configuration>
					<appendAssemblyId>false</appendAssemblyId>
					<descriptorRefs>
						<descriptorRef>jar-with-dependencies</descriptorRef>
					</descriptorRefs>
					<archive>
						<manifest>
							<!-- 此处指定main方法入口的class -->
							<mainClass>com.wzq.JavaDirectKafkaWordCount</mainClass>
						</manifest>
					</archive>
				</configuration>
				<executions>
					<execution>
						<id>make-assembly</id>
						<phase>package</phase>
						<goals>
							<goal>assembly</goal>
						</goals>
					</execution>
				</executions>
			</plugin>
		</plugins>
	</build>
</project>