Delta lake with Java--利用spark sql操作数据2

一个平凡的IT农民工

已于 2024-05-01 15:37:21 修改

阅读量963

点赞数 8

CC 4.0 BY-SA版权

于 2024-05-01 15:36:52 首次发布

本文链接：https://blog.youkuaiyun.com/kanganrui/article/details/138373870

上一篇文章尝试了建库，建表，插入数据，还差删除和更新，所以在这篇文章补充一下，代码很简单，具体如下：

import org.apache.spark.sql.SaveMode;
import org.apache.spark.sql.SparkSession;

public class DeltaLakeWithSparkSql2 {

    public static void main(String[] args) {
          SparkSession spark = SparkSession.builder()
                .master("local[*]")
                .appName("delta_lake")
                .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
                .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")
                .getOrCreate();
          String sourcePath="D:\\bookcode\\delta-lake-up-and-running-main\\data\\YellowTaxi\\";
          var df = spark.read().format("parquet").load(sourcePath);
          System.out.println("总数据量："+df.count());
          df.write().format("delta").mode(SaveMode.Overwrite).save("file:///D:\\\\bigdata\\\\detla-lake-with-java\\\\YellowTaxi");


          spark.sql("CREATE DATABA