Spark数据排序实践-优快云博客

本文链接：https://blog.youkuaiyun.com/qq_31297573/article/details/108523370

输入数据格式

四列数据分别对应年月日温度

预期结果

按月份排序，再按温度排序

在这里插入图片描述

实现代码

package com.jackie.spark

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

class UDFSort(val first: Int, val second: Int) extends Ordered[UDFSort] with Serializable {
  override def compare(that: UDFSort): Int = {
    if (this.first - that.first != 0) {
      //第一个值不相等的时候，直接返回大小
      this.first - that.first //返回值
    }
    else {
      //第一个值相等的时候,比较第二个值
      this.second - that.second
    }
  }
}


object SecondSort {

  def main(args: Array[String]): Unit = {

    //创建SparkConf对象
    val config: SparkConf = new SparkConf().setMaster("local[*]").setAppName("SecondSort")
    //创建Spark上下文对象
    val sc = new SparkContext(config)

    //文件输入输出路径
    val inputPath = "D:\\IDEA2018\\Scala\\spark-study\\src\\test.txt"
    val outputPath = "D:\\IDEA2018\\Scala\\spark-study\\src\\test1.txt"

    //读取文件 将文件内容一行一行的读取出来
    val input: RDD[String] = sc.textFile(inputPath)

    val mapRDD: RDD[(UDFSort, (String, String))] = input.map(line => {
      val fields: Array[String] = line.split("\t")
      (new UDFSort(fields(1).toInt, fields(3).toInt), (fields(0) + "-" + fields(1), fields(3)))
    })
    val sortRDD: RDD[(String, String)] = mapRDD.sortByKey(true, 1).map(x => {
      (x._2._1, x._2._2)
    }).reduceByKey(_ + "\t" + _)

    sortRDD.foreach(println(_))

  }

}