概述:本文介绍Spark SQL操作parquet、hive及mysql的方法,并实现Hive和MySql两种不同数据源的连接查询
1、操作parquet
(1)编程实现
#启动spark-shell
./app/spark-2.4.2-bin-hadoop2.6/bin/spark-shell --master local[2] --jars /root/software/mysql-connector-java-5.1.47-bin.jar
#创建DataFrame
object ParqueApp {
def main(args: Array[String]): Unit = {
val spark = SparkSession.builder().appName("DataSetApp").master("local[2]").getOrCreate()
val userDF = spark.read.format("parquet").load("file:///root/app/spark-2.4.2-bin-hadoop2.6/examples/src/main/resources/users.parquet")
userDF.printSchema()
userDF.show()
userDF.select("name", "favorite_color").show()
//将查询结果写入json文件
userDF.select("name", "favorite_color").write.format("json").save("file:///root/app/tmp/jsonout")
//自定义分区数 默认200
spark.sqlContext.setConf("spark.sql.shuffle.partitions", "10")
//spark sq