Spark——计算用户在基站停留时间并排序_spark算子基站时间段-优快云博客

通过Spark进行用户位置数据处理，分析用户活动模式，结合位置信息，实现用户行为的时间和空间特征提取。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

/home/hadoop/Data/logs/

user.log

15967170343,20180912082400,16030401EAFB68F1E3CDF819735E1C66,1
17606512176,20180912082500,16030401EAFB68F1E3CDF819735E1C66,1
15967170343,20180912170000,16030401EAFB68F1E3CDF819735E1C66,0
17606512176,20180912180000,16030401EAFB68F1E3CDF819735E1C66,0

user2.log

15967170343,20180912082480,9F36407EAD0629FC166F14DDE7970F68,1
17606512176,20180912082580,9F36407EAD0629FC166F14DDE7970F68,1
15967170343,20180912170000,9F36407EAD0629FC166F14DDE7970F68,0
17606512176,20180912180000,9F36407EAD0629FC166F14DDE7970F68,0

lac_info.log

16030401EAFB68F1E3CDF819735E1C66,116.303955,40.041935
9F36407EAD0629FC166F14DDE7970F68,116.296302,40.032296

/////\\\\\\\

////、、、、、、

package srt.spark
import org.apache.spark.{SparkConf, SparkContext}

object AdvUserLocation {
  def main(args: Array[String]): Unit = {

    val conf = new SparkConf().setAppName("AdvUserLocation").setMaster("local[2]")
    val sc = new SparkContext(conf)
    val rdd0 = sc.textFile("/home/hadoop/Data/logs/").map(line => {
      val fields = line.split(",")
      val eventType = fields(3)
      val time = fields(1)
      val timeLong = if (eventType == "1") -time.toLong else  time.toLong
      ((fields(0),fields(2)), timeLong)
    })
    println(rdd0.collect().toBuffer)

    val rdd1 = rdd0.reduceByKey(_+_).map(t =>{
      val mobile = t._1._1
      val lac = t._1._2
      val time = t._2
      (lac, (mobile, time))
    })
    println(rdd1.collect().toBuffer)

    val rdd2 = sc.textFile("/home/hadoop/Data/lac_info.log").map(line1 => {
      val f = line1.split(",")
      (f(0),(f(1),f(2)))
    })
    println(rdd2.collect().toBuffer)

    val rdd3 = rdd1.join(rdd2).map( t =>{
      val lac = t._1
      val mobile = t._2._1._1
      val time = t._2._1._2
      val x = t._2._2._1
      val y = t._2._2._2
      (mobile, lac, time, x, y)
    })
    println(rdd3.collect().toBuffer)

    val rdd4 = rdd3.groupBy(_._1)
    println(rdd4.collect().toBuffer)

    val rdd5 = rdd4.mapValues( t => {
      t.toList.sortBy(_._3).reverse.take(2)
    })
    println(rdd5.collect().toBuffer)

    sc.stop()
  }
}

////\\\\\\

ArrayBuffer(((15967170343,9F36407EAD0629FC166F14DDE7970F68),-20180912082480),
((17606512176,9F36407EAD0629FC166F14DDE7970F68),-20180912082580),
((15967170343,9F36407EAD0629FC166F14DDE7970F68),20180912170000), 
((17606512176,9F36407EAD0629FC166F14DDE7970F68),20180912180000), 
((15967170343,16030401EAFB68F1E3CDF819735E1C66),-20180912082400),
((17606512176,16030401EAFB68F1E3CDF819735E1C66),-20180912082500),
((15967170343,16030401EAFB68F1E3CDF819735E1C66),20180912170000), 
((17606512176,16030401EAFB68F1E3CDF819735E1C66),20180912180000))

/////\\\\\

ArrayBuffer((16030401EAFB68F1E3CDF819735E1C66,(15967170343,87600)), 
(16030401EAFB68F1E3CDF819735E1C66,(17606512176,97500)), 
(9F36407EAD0629FC166F14DDE7970F68,(15967170343,87520)),
 (9F36407EAD0629FC166F14DDE7970F68,(17606512176,97420)))

////\\\\\

ArrayBuffer((16030401EAFB68F1E3CDF819735E1C66,(116.303955,40.041935)),
 (9F36407EAD0629FC166F14DDE7970F68,(116.296302,40.032296)))

///\\\\

ArrayBuffer((15967170343,9F36407EAD0629FC166F14DDE7970F68,87520,116.296302,40.032296),
(17606512176,9F36407EAD0629FC166F14DDE7970F68,97420,116.296302,40.032296), 
(15967170343,16030401EAFB68F1E3CDF819735E1C66,87600,116.303955,40.041935), 
(17606512176,16030401EAFB68F1E3CDF819735E1C66,97500,116.303955,40.041935))

///\\\\

ArrayBuffer((15967170343,CompactBuffer((15967170343,9F36407EAD0629FC166F14DDE7970F68,87520,116.296302,40.032296), 
(15967170343,16030401EAFB68F1E3CDF819735E1C66,87600,116.303955,40.041935))), 
(17606512176,CompactBuffer((17606512176,9F36407EAD0629FC166F14DDE7970F68,97420,116.296302,40.032296), 
(17606512176,16030401EAFB68F1E3CDF819735E1C66,97500,116.303955,40.041935))))

////\\\\\

ArrayBuffer((15967170343,List((15967170343,16030401EAFB68F1E3CDF819735E1C66,87600,116.303955,40.041935),
 (15967170343,9F36407EAD0629FC166F14DDE7970F68,87520,116.296302,40.032296))), 
(17606512176,List((17606512176,16030401EAFB68F1E3CDF819735E1C66,97500,116.303955,40.041935), 
(17606512176,9F36407EAD0629FC166F14DDE7970F68,97420,116.296302,40.032296))))