sparkSQL 访问mysql数据

最新推荐文章于 2023-04-02 07:51:39 发布

原创最新推荐文章于 2023-04-02 07:51:39 发布 · 495 阅读

0 ·

CC 4.0 BY-SA版权

spark 专栏收录该内容

8 篇文章

订阅专栏

1、如果使用spark-shell 操作，需要修改spark-defaults.conf 配置
spark.executor.extraClassPath=/opt/software/hive/lib/mysql-connector-java-5.1.45-bin.jar
spark.driver.extraClassPath=/opt/software/hive/lib/mysql-connector-java-5.1.45-bin.jar

package com.hihi.learn.sparkSql
import java.util.Properties
import org.apache.spark.sql.{SaveMode, SparkSession}

object MySqlDemo {

  val url = "jdbc:mysql://hadoop001:3306"
  val table = "soctt.dept"
  val dBuser = "root"
  val dBpwd = "123456"
  val prop = new Properties()
  prop.setProperty("user", "root")
  prop.setProperty("password", "123456")
  case class DeptRecord(deptno: Int, dname: String, loc: String)

  def main(args: Array[String]): Unit = {

    val spark = SparkSession
      .builder()
      .appName("Mysql Demo.")
      .master("local")
      .getOrCreate()

    readFromMysql(spark)
    writeToMysql(spark)
    readFromMysql(spark)
  }

  // 从mysql读取数据
  def readFromMysql(spark: SparkSession): Unit = {
    val jdbcDF  = spark.read.format("jdbc")
      .option("url", url)
      .option("dbtable", table)
      .option("user", dBuser)
      .option("password", dBpwd).load()

    jdbcDF.show()
  }

  // 向mysql写数据
  def writeToMysql(spark: SparkSession): Unit = {
    val arr = Array(
      DeptRecord(50, "AA", "aa"),
      DeptRecord(60, "BB", "bb")
    )
    val jdbcDF = spark.createDataFrame(arr)
    jdbcDF.write.mode(SaveMode.Append).jdbc(url, table, prop)
  }
}