自定义分区编号

Spark分区与排序实践

最新推荐文章于 2022-11-11 21:05:01 发布

原创最新推荐文章于 2022-11-11 21:05:01 发布 · 285 阅读

0 ·

CC 4.0 BY-SA版权

spark 专栏收录该内容

12 篇文章

订阅专栏

 val isLocal = args(0).toBoolean
    val conf = new SparkConf().setAppName(this.getClass.getCanonicalName)
    if (isLocal) {
      conf.setMaster("local[*]")
    }
    val sc = new SparkContext(conf)
    //指定以后从哪里读取数据创建RDD
    val lines = sc.textFile(args(1))
    val topN = args(2).toInt
    //对数据进行整理
    val reduced: RDD[((String, String), Int)] = lines.map(line => {
      val fields = line.split("/")
      val url = fields(2)
      val teacher = fields(3)
      val subject = url.split("[.]")(0)
      ((subject, teacher), 1)
    }).reduceByKey(_ + _)
    //计算学科的数量 distinct之后里面有多少种类的学科,就有多少,然后collect,写到磁盘上
    val subjects: Array[String] = reduced.map(_._1._1).distinct().collect()
    /*println(subjects.toBuffer)   //ArrayBuffer(javaee, php, bigdata)
    sc.stop()*/

    //初始化分区器 分区器创建好了
    val partitioner: SubjectPartitioner = new SubjectPartitioner(subjects)
    //创建好分区器之后,开始调用分区器
    //按照学科进行分区
    val partitioined: RDD[((String, String), Int)] = reduced.partitionBy(partitioner)

    //把每个分区的数据拿出来
    val result1: RDD[((String, String), Int)] = partitioined.mapPartitions(it => {
      it.toList.sortBy((-_._2)).take(topN).iterator //输入的是迭代器,返回的也是迭代器
    })
    println(result1.collect().toBuffer)
  sc.stop()
  }
}
//定义分区器,传入成员变量方便 以后传进去
class SubjectPartitioner(val subjects:Array[String]) extends Partitioner{
  //这里面一定要传进去传进去多少个分区,有多少个学科就有多少个分区
  //在柱构造器中定义分区规则,HashMap里面装的是学科名字和分区编号
  //柱构造器在new 的时候只会执行一次
  private val nameTonum: mutable.HashMap[String, Int] = new mutable.HashMap[String, Int]()
 var i = 0
  for(sub<-subjects){
    //这个是scala里面中 sub 当初key  i 当初v  put 到HashMap里面去
    nameTonum(sub)=i   //拿到一个,就把数据加进去
    i+=1   //
  }
  //这时候就设置好了(javaee,0),(php,1),(bigdate,2)
  override def numPartitions: Int = subjects.length//3

  //executor在Task中,shufflewriter之前会调用
  override def getPartition(key: Any): Int ={
    //asInstanceOf将引用指定为子类的引用
    val tuple: (String, String) = key.asInstanceOf[(String, String)]
    val subject: String = tuple._1
    nameTonum(subject) //根据学科获取相对应的编号

    //判断是否本地运行
    val isLocal: Boolean = args(0).toBoolean
    //加载配置文件
    val conf: SparkConf = new SparkConf().setAppName(this.getClass.getCanonicalName)
    //if语句执行判断
    if(isLocal){
      conf.setMaster("local[*]")
    }
    //创建连接
    val sc: SparkContext = new SparkContext(conf)
    //指定文件的路径,创建RDD
    val lines: RDD[String] = sc.textFile(args(1))
    //传入一个topN
    val topN: Int = args(2).toInt

    //对文件进行操作
    val reduced: RDD[((String, String), Int)] = lines.map(it => {
      val fields: Array[String] = it.split("/")
      val teacher: String = fields(3)
      val subjects: String = fields(2).split("[.]")(0)
      ((subjects, teacher), 1)
    }) reduceByKey (_ + _)
   /* val result: mutable.Buffer[((String, String), Int)] = reduced.collect().toBuffer
    println(result)*/

    //有通过subject进行分区
    //先找出subject的种类
    val subject: Array[String] = reduced.map(_._1._1).distinct().collect()

    //new一个定义了的构造器
    val partitioner: SubjectPartitioner2 = new SubjectPartitioner2(subject)

    //reduced调用分区器
    val partitioned: RDD[((String, String), Int)] = reduced.partitionBy(partitioner)
    //使用一个特殊的集合进行排序
    //TreeSet里面是一个Ordering的规则
    //然后修改排序规则,通过隐式转换,按照自己想要的规则进行排序
    //foreach把迭代器里面的数据遍历出来,也就是吧m每个分区里面的数据取出来
    val result2: RDD[((String, String), Int)] = partitioned.mapPartitions(t => {
      //定义一个隐式转换修改排序规则
      implicit val rules: Ordering[((String, String), Int)] = Ordering[Int].on[((String, String), Int)](t => -t._2)
      //new TreeSet 里面有自动排序规则 会自动找上面的隐式转换
      val sorted: mutable.TreeSet[((String, String), Int)] = new mutable.TreeSet[((String, String), Int)]()  
      sorted.foreach(t => {
        //将数据添加到TreeSet中
        sorted += t //相当于add
        if (sorted.size > topN) {
          sorted -= sorted.last
        }
      })
      sorted.iterator //输入的是迭代器,输出的也是迭代器
    })
    println(result2.collect().toBuffer)
  }
}
//自定义一个分区,然后以subject为分区规则,把subject传进去
class SubjectPartitioner2(var subject: Array[String]) extends Partitioner{
  //定义一个柱构造器,这个柱构造器每new 一次就执行一次
  private val nameToNum: mutable.HashMap[String, Int] = new mutable.HashMap[String, Int]()
  //通过遍历subject把学科,及其对应的编号传入HashMap中
  var i = 0
  for (sub<-subject){
    nameToNum(sub)=i  //这种形式在scala中就是把subject 0,1,2  put到map中
    i+=1
  }
  //Array(("javaee",0),("php",1),("bigdate",2))
  override def numPartitions: Int = subject.length  //求出数组的长度

  override def getPartition(key: Any): Int = {
    //因为传进来的是any类型,但是我们只需要(String,String) 元组
    //所以就需要强转
    val tuple: (String, String) = key.asInstanceOf[(String, String)]

    //获取subject
    val subject: String = tuple._1
    //取出subject对应的编号
    nameToNum(subject)

上传一种新的比较规则

val isLocal: Boolean = args(0).toBoolean
    //加载配置文件
    val conf: SparkConf = new SparkConf().setAppName(this.getClass.getCanonicalName)
    //if语句执行判断
    if(isLocal){
      conf.setMaster("local[*]")
    }
    //创建连接
    val sc: SparkContext = new SparkContext(conf)
    //指定文件的路径,创建RDD
    val lines: RDD[String] = sc.textFile(args(1))
    //传入一个topN
    val topN: Int = args(2).toInt

    //对文件进行操作
    val subjectAndOne: RDD[((String, String), Int)] = lines.map(it => {
      val fields: Array[String] = it.split("/")
      val teacher: String = fields(3)
      val subjects: String = fields(2).split("[.]")(0)
      ((subjects, teacher), 1)
    })
    val subjects: Array[String] = subjectAndOne.map(_._1._1).distinct().collect()

    //初始分化区
    val subPartitioned: SubjectPartitioner6 = new SubjectPartitioner6(subjects)


    //调用一个算子,可以重新分区,并且每一个分区内排序

    //制定一个比较规则,参与比较的字段必须是在key里面
    implicit val orderingRules: Ordering[((String, String), Int)] with Object = new Ordering[((String, String), Int)]{
      override def compare(x: ((String, String), Int), y: ((String, String), Int)): Int = {
        -(x._2 - y._2)
      }
    }
    //注意这个是按照元组的比较规则进行比较,先按第一个进行比较,如果第一个不相等,那就直接比较第一个字段,如果相等在比较第二个字段,一次比较下去
    //知道找到不相等的元组
    //如果数据比较简单就用这种排序方法 on里面调用的也是 compare方法
   // implicit val rules = Ordering[Int].on[((String, String), Int)](t=>t._2)
    //既可以shuffle又可以排序
    val result: RDD[((String, String), Int)] = subjectAndOne.repartitionAndSortWithinPartitions(subPartitioned)
    println(result.collect().toBuffer)

  }
}
//自定义一个分区,然后以subject为分区规则,把subject传进去
class SubjectPartitioner6(var subject: Array[String]) extends Partitioner{
  //定义一个柱构造器,这个柱构造器每new 一次就执行一次
  private val nameToNum: mutable.HashMap[String, Int] = new mutable.HashMap[String, Int]()
  //通过遍历subject把学科,及其对应的编号传入HashMap中
  var i = 0
  for (sub<-subject){
    nameToNum(sub)=i  //这种形式在scala中就是把subject 0,1,2  put到map中
    i+=1
  }
  //Array(("javaee",0),("php",1),("bigdate",2))
  override def numPartitions: Int = subject.length  //求出数组的长度

  override def getPartition(key: Any): Int = {
    //因为传进来的是any类型,但是我们只需要(String,String) 元组
    //所以就需要强转
    val tuple: (String, String) = key.asInstanceOf[(String, String)]

    //获取subject
    val subject: String = tuple._1
    //取出subject对应的编号
    nameToNum(subject)

shuffledRDD实现 repartitionAndSortWithPartition的功能

  def main(args: Array[String]): Unit = {
    //判断是否在本地运行
    val isLocal: Boolean = args(0).toBoolean
    //创建本地连接
    val conf: SparkConf = new SparkConf().setAppName(this.getClass.getCanonicalName)
    //判断为本地运行
    if(isLocal){
      conf.setMaster("local[*]")
    }
    //建立连接
    val sc: SparkContext = new SparkContext(conf)
    //指定传输路径,创建RDD
    val lines: RDD[String] = sc.textFile(args(1))  //把每一行的数据传到数组数组里面
    //topN,输出
    val topN: Int = args(2).toInt
    //对数据进行处理
    val reduced: RDD[((String, String), Int)] = lines.map(line => {
      val fields: Array[String] = line.split("/")
      val teacher: String = fields(3)
      val subject: String = fields(2).split("[.]")(0)
      ((subject, teacher), 1)
    }).reduceByKey(_ + _)
    //对数据进行去重和把数据写到磁盘里面
    val subjects: Array[String] = reduced.map(_._1._1).distinct().collect()
    //对reduced进行处理
    val maped: RDD[((String, String, Int), Null)] = reduced.map(t => {
      ((t._1._1, t._1._2, t._2), null)
    })

    //把数据放到自定义的分区器里面
    val partitioner: SubjectPartitioner7 = new SubjectPartitioner7(subjects)
    //要制定一个排序规则,new Ordering就要实现Ordering的方法
   implicit val rulesRodering: Ordering[(String, String, Int)] = new Ordering[(String, String, Int)] {
      override def compare(x: (String, String, Int), y: (String, String, Int)): Int = {
        -(x._3-y._3) //倒序
      }
    }

    //new ShullfeRDD实现reparitionAndSortWithPartition的功能
    //key是tuple3  value 是null   null,与null 合并的结果是null
    //不要忘记shuffledRDD里面一套传进去key ;类型,value:类型,combiner类型,他只是把数据放到一起,但不会排序
    val shuffled: ShuffledRDD[(String, String, Int), Null, Null] = new ShuffledRDD[(String, String, Int), Null, Null](maped, partitioner)
    //把数据和分区器放到一起执行
    println(shuffled.collect().toBuffer)


  }
}
//自定义一个分区器,按照谁分区,就把水传进来了
class SubjectPartitioner7(val subjects:Array[String]) extends Partitioner{
  //定义一个柱构造器,指定分区器得规则,每new 一次这里就执行一次
  //先创建一个HashMap,存放分区器机器分区编号,hashMap里面需要传入一个隐式转换
  private val rulesRodering: Ordering[(String, String, Int)]= Ordering[Int].on[(String, String, Int)](t=>t._3)
  private val rules: mutable.HashMap[String, Int] = new mutable.HashMap[String, Int]()
  var i = 0
  for (sub<-subjects){
    rules(sub)=i
    i+=1

  }

  override def numPartitions: Int = subjects.length

  override def getPartition(key: Any): Int = {
    val tuple: (String, String, Int) = key.asInstanceOf[(String, String, Int)]
    val subject: String = tuple._1
    rules(subject)