197、Spark 2.0之Dataset开发详解-聚合函数：avg、sum、max、min、count、countDistinct

最新推荐文章于 2023-11-26 00:24:43 发布

原创最新推荐文章于 2023-11-26 00:24:43 发布 · 1k 阅读

0 ·

CC 4.0 BY-SA版权

Spark入坑专栏收录该内容

207 篇文章

订阅专栏

本文通过使用Spark SQL的聚合函数，演示了如何从员工和部门数据中计算平均工资、总工资、最高工资、最低工资以及不同部门的员工数量。代码示例展示了数据加载、连接操作、分组和聚合的基本步骤。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

代码

object AggregateFunction {

  case class Employee(name: String, age: Long, depId: Long, gender: String, salary: Long)

  case class Department(id: Long, name: String)

  def main(args: Array[String]): Unit = {
    val sparkSession = SparkSession
      .builder()
      .appName("AggregateFunction")
      .master("local")
      .getOrCreate()

    import sparkSession.implicits._
    import org.apache.spark.sql.functions._

    val employeePath = this.getClass.getClassLoader.getResource("employee.json").getPath
    val departmentPath = this.getClass.getClassLoader.getResource("department.json").getPath

    val employeeDF = sparkSession.read.json(employeePath)
    val departmentDF = sparkSession.read.json(departmentPath)

    val employeeDS = employeeDF.as[Employee]
    val departmentDS = departmentDF.as[Department]

    employeeDS
      .join(departmentDS, $"depId" === $"id")
      .groupBy(departmentDS("name"))
      .agg(avg(employeeDS("salary")), sum(employeeDS("salary")), max(employeeDS("salary")), min(employeeDS("salary")), count(employeeDS("name")), countDistinct(employeeDS("name")))
      .show()
  }
}