SparkSQL 简单测试

最新推荐文章于 2023-11-22 17:54:26 发布

draven1122

最新推荐文章于 2023-11-22 17:54:26 发布

阅读量569

点赞数

分类专栏： Spark

本文链接：https://blog.youkuaiyun.com/draven1122/article/details/78425296

版权

Spark 专栏收录该内容

3 篇文章

订阅专栏

本文介绍了如何使用SparkSQL进行简单测试，详细步骤将在关联的配置文件文章中展示。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

其中注册表的方法都在另一篇记配置文件的文章中所示的代码里

/**
  * Created by Draven on 2017/11/1.
  *  draven1122@163.com
  */
object CassandraSqlExample extends App {
  val ss = getSparkSession

  import ss.implicits._

  ss.read.format(CASSANDRA_FORMAT).options(cassandraOptions("qiandw_test", "loan_program"))
    .load().filter($"funding_target" > 20000000).createOrReplaceTempView("loan_program")
  ss.sql("select * from loan_program").show()
}

/**
  * 从sql语句构建
  */
object CassandraSqlTextExample extends App {
  val ss = getSparkSession

  import ss.implicits._

  ss.read.format(CASSANDRA_FORMAT)
    .options(cassandraOptions("qiandw_test", "cash_account"))
    .load()
    .filter($"balance" > 0).createOrReplaceTempView("cash_account")
  ss.sql("select * from cash_account").orderBy("balance").show()
}

/**
  * 连表查询
  */
object JoinQueryExample extends App {
  val ss = getSparkSession

  import ss.implicits._

  ss.read.format(CASSANDRA_FORMAT)
    .options(cassandraOptions("qiandw_test", "cash_account"))
    .load()
    .createOrReplaceTempView("cash_account")

  ss.read.format(CASSANDRA_FORMAT)
    .options(cassandraOptions("qiandw_test", "cash_account_detail"))
    .load()
    .createOrReplaceTempView("cash_account_detail")

  ss.sql("select * from cash_account a left join cash_account_detail b on a.acct_id = b.acct ").show(5)
}

object DifferentDataBaseJoinQueryExample extends App {
  registerCassandraTables
  registerMySqlTables
  //  ss.read.format("jdbc")
  //    .options(Map("url" -> "jdbc:mysql://localhost:3306/huangjinxin",
  //      "driver" -> "com.mysql.jdbc.Driver",
  //      "dbtable" -> "t_user",
  //      "user" -> "root",
  //      "password" -> "root")).load().createOrReplaceTempView("t_user")
  //
  //  ss.read.format(CASSANDRA_FORMAT)
  //    .options(cassandraOptions("qiandw_test", "cash_account"))
  //    .load()
  //    .createOrReplaceTempView("cash_account")

  getSparkSession.sql("select * from cash_account a right join t_user b on a.state = b.age ").show()
}