大数据（8h）Spark页面单跳转化率

原理图

Python实现

user_behavior = """
剑圣_21:35:00_搜索页
剑圣_21:35:01_商品页
先知_21:35:01_搜索页
先知_21:35:04_商品页
巫妖_21:35:07_搜索页
先知_21:36:30_订单页
剑圣_21:38:22_订单页
巫妖_21:39:59_商品页
农民_21:40:01_搜索页
剑圣_21:40:52_付款页
""".strip().split()
user_behavior = [line.split('_') for line in user_behavior]
print(user_behavior)

# 页面点击合计
divisor = dict()
for _, _, page in user_behavior:
    divisor[page] = divisor.get(page, 0) + 1
print(divisor)

# 按用户分组
user_group = dict()
for name, time, page in user_behavior:
    if name not in user_group:
        user_group[name] = []
    user_group[name].append((time, page))
print(user_group)

# 按时间升序
for k, v in user_group.items():
    user_group[k] = sorted(v)
print(user_group)

# 页面跳转链条
page_chain = [[i[1] for i in v] for v in user_group.values()]
print(page_chain)

# 页面跳转（拉链）
skip = ['->'.join(s) for c in page_chain for s in zip(c[:-1], c[1:])]
print(skip)

# 页面跳转统计
dividend = dict()
for s in skip:
    dividend[s] = dividend.get(s, 0) + 1
print(dividend)

# 跳转率
rate = dict()
for k, v in dividend.items():
    rate[k + ' 跳转率'] = v / divisor.get(k.split('->')[0], -1)
print(rate)

Spark实现

import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.rdd.RDD

object Hello {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("a1").setMaster("local[*]")
    val sc = new SparkContext(conf)
    //数据读取
    val userBehavior =
      """|剑圣_21:35:00_搜索页
         |剑圣_21:35:01_商品页
         |先知_21:35:01_搜索页
         |先知_21:35:04_商品页
         |巫妖_21:35:07_搜索页
         |先知_21:36:30_订单页
         |剑圣_21:38:22_订单页
         |巫妖_21:39:59_商品页
         |农民_21:40:01_搜索页
         |剑圣_21:40:52_付款页
         |""".stripMargin.stripLineEnd.split("\n")
    val r0: RDD[UserBehavior] = sc.makeRDD(userBehavior, numSlices = 2).map(
      line => {
        val row = line.split("_")
        UserBehavior(row(0), row(1), row(2))
      }
    )
    //页面点击合计
    val m = r0.map(u => (u.page, 1)).reduceByKey(_ + _).collect.toMap
    println(m)
    //分组
    val r2: RDD[(String, Iterable[UserBehavior])] = r0.groupBy(_.name)
    //分组内数据处理
    val r3: RDD[(String, List[(String, Int)])] = r2.mapValues(b => {
      //分组内排序
      //val s = b.toList.sortBy(_.time)
      val s = b.toList.sortWith((left, right) => left.time < right.time)
      //只要页面
      val p = s.map(_.page)
      //拉链
      val z = p.zip(p.tail)
      //计数
      z.map(pp => (pp._1 + "->" + pp._2, 1))
    })
    //扁平化
    val r4: RDD[(String, Int)] = r3.map(_._2).flatMap(e => e)
    //合计
    val r5: RDD[(String, Int)] = r4.reduceByKey(_ + _)
    println(r5.collect.toList)
    //跳转率
    val r6: RDD[(String, Double)] = r5.map(
      t => (t._1, t._2.toDouble / m.getOrElse(t._1.split("->")(0), -1))
    )
    println(r6.collect.toList)
  }
}

//样例类：元数据
case class UserBehavior(name: String, time: String, page: String)

HIVE实现

https://yellow520.blog.youkuaiyun.com/article/details/116200798