Spark源码解读(1)--Spark Sql使用JDBC写Write流程

最新推荐文章于 2025-06-10 14:19:14 发布

原创

最新推荐文章于 2025-06-10 14:19:14 发布 · 3.9k 阅读

7 ·

CC 4.0 BY-SA版权

第一步: RDD调用write save() 保存数据到数据库;

//保存数据
    dataFrame.write.format("jdbc")
      .mode(saveMode)
      .option("dbtable",dbtable)
      .option("url",ads_url)
      .option("user",ads_user)
      .option("password",ads_password)
      .save()

第二步: 进入save()方法,通过lookupDataSource判断数据源类型,调用不同方法,

def save(): Unit = {
   
   
    if (source.toLowerCase(Locale.ROOT) == DDLUtils.HIVE_PROVIDER) {
   
   
      throw new AnalysisException("Hive data source can only be used with tables, you can not " +
        "write files of Hive data source directly.")
    }

    assertNotBucketed("save")
    
    //Given a provider name, look up the data source class definition
    val cls = DataSource.lookupDataSource(source, df.sparkSession.sessionState.conf)
    
    if (classOf[DataSourceV2].isAssignableFrom(cls)) {
   
   
      val source = cls.newInstance().asInstanceOf[DataSourceV2]
      source match {
   
   
        case ws: WriteSupport =>
          val sessionOptions = DataSourceV2Utils.extractSessionConfigs(
            source,
            df.sparkSession.sessionState.conf)
          val options = sessionOptions ++ extraOptions

          val writer = ws.createWriter(
            UUID.randomUUID.toString, df.logicalPlan.output.toStructType, mode,
            new DataSourceOptions(options.asJava))

          if