package ads
import common.PortraitCommon.{ck_dim, ckdriver, ckpassword, ckurl, ckuser, dim_user_info_tmp}
import org.apache.
spark.sql.{DataFrame, Dataset, Row,
SparkSession}
import org.apache.
spark.sql.functions.col
import utils.{ClickhouseUtils,
SparkUtils}
object ads_car_brand_cut {
def main(args: Array[String]): Unit = {
// TODO: 构建
spark环境
val
spark:
SparkSession =
SparkUtils.get
Spark("ads_car_brand_cut")
// TODO: 取数据
val dwdreadData =
spark.read.format("jdbc")
.option("url", ckurl)
.option("user", ckuser)
.option("password", ckpassword)
.option("dbtable", "tieta_v2_dim.dim_user_info")
.option("driver", ckdriver)
.load()
// val mysqlurl = "jdbc:mysql://192.168.3.117:3309/lol"
// val mysqlusername = "root"
// val mysqlpassword = "123456"
// TODO: 过滤品牌
val filterDF: Dataset[Row] = dwdreadData.filter(col("car_brand_model") =!= "未知")
// TODO: 对车辆型号进行统计
val resDF: DataFrame = filterDF.groupBy("car_brand_model").count()
.withColumnRenamed("count", "brand_count")
resDF.show()
// // TODO: 将数据写入 ClickHouse
// ClickhouseUtils.writeClickHouse(resDF, "tieta_v2_ads", "ads_car_brand_cut")
// println("数据成功写入 ClickHouse")
// resDF.write
// .format("jdbc")
// .option("url", mysqlurl)
// .option("dbtable", "ads_car_brand_cut")
// .option("user", mysqlusername)
// .option("password", mysqlpassword)
// .option("driver", "com.mysql.cj.jdbc.Driver")
// .mode("append")
// .save()
//
// println("Data successfully written to mysql")
}
}我想在虚拟机的
spark运行这个文件，我的整个项目打
包的
jar包在、srv/untitled3
-1.0
-SNAPSHOT.
jar
<?xml version="1.0" encoding="UTF
-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema
-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven
-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.example</groupId>
<artifactId>untitled3</artifactId>
<version>1.0
-SNAPSHOT</version>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF
-8</project.build.sourceEncoding>
<!
-- 统一版本管理
-->
<
spark.version>3.1.1</
spark.version>
<
hive.version>2.3.7</
hive.version> <!
-- 与
Spark 3.1.1 兼容的
Hive 版本
-->
</properties>
<dependencies>
<!
-- Spark Core
-->
<dependency>
<groupId>org.apache.
spark</groupId>
<artifactId>
spark-core_2.12</artifactId>
<version>${
spark.version}</version>
</dependency>
<!
-- Spark SQL
-->
<dependency>
<groupId>org.apache.
spark</groupId>
<artifactId>
spark-sql_2.12</artifactId>
<version>${
spark.version}</version>
</dependency>
<!
-- Spark Hive Support
-->
<dependency>
<groupId>org.apache.
spark</groupId>
<artifactId>
spark-hive_2.12</artifactId>
<version>${
spark.version}</version>
</dependency>
<!
-- Spark ML
lib -->
<dependency>
<groupId>org.apache.
spark</groupId>
<artifactId>
spark-ml
lib_2.12</artifactId>
<version>${
spark.version}</version>
</dependency>
<!
-- Hadoop Client
-->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop
-client</artifactId>
<version>3.2.0</version>
</dependency>
<!
-- Hudi
Spark Bundle
-->
<dependency>
<groupId>org.apache.hudi</groupId>
<artifactId>hudi
-spark3.1
-bundle_2.12</artifactId>
<version>0.12.0</version>
</dependency>
<!
-- ClickHouse JDBC
-->
<dependency>
<groupId>ru.yandex.clickhouse</groupId>
<artifactId>clickhouse
-jdbc</artifactId>
<version>0.3.2</version>
</dependency>
<!
-- Jackson Databind
-->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson
-databind</artifactId>
<version>2.10.0</version>
</dependency>
<!
-- Jackson Core
-->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson
-core</artifactId>
<version>2.10.0</version>
</dependency>
<!
-- Akka Actor
-->
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka
-actor_2.12</artifactId>
<version>2.6.16</version>
</dependency>
<!
-- JSch
-->
<dependency>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
<version>0.1.51</version>
</dependency>
<!
-- Typesafe Config
-->
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>1.4.2</version>
</dependency>
<!
-- MySQL Connector
-->
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql
-connector
-java</artifactId>
<version>8.0.29</version>
</dependency>
<!
-- DOM4J
-->
<dependency>
<groupId>org.dom4j</groupId>
<artifactId>dom4j</artifactId>
<version>2.1.4</version>
</dependency>
</dependencies>
<build>
<plugins>
<!
-- Maven Shade Plugin 用于创建
包含所有依赖的fat
JAR -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven
-shade
-plugin</artifactId>
<version>3.2.4</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<!
-- 如果需要设置主类，可以在这里指定
-->
<!
-- <mainClass>your.main.Class</mainClass>
-->
</transformer>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META
-INF/*.SF</exclude>
<exclude>META
-INF/*.DSA</exclude>
<exclude>META
-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>这是我的pom文件，应该没问题吧
本文介绍了解决Hive启动时报错找不到spark-assembly相关Jar包的问题。由于Spark2之后，大JAR包被拆分成多个小JAR包，原有的spark-assembly-*.jar不再存在。文章提供了一个简单的修改方案，通过调整hive脚本来解决问题。
扫一扫
752
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
