spark构建图graphx

最新推荐文章于 2021-05-30 21:37:53 发布

ASD991936157

最新推荐文章于 2021-05-30 21:37:53 发布

阅读量1.5k

点赞数

CC 4.0 BY-SA版权

本文链接：https://blog.youkuaiyun.com/ASD991936157/article/details/75635656

本文介绍了一种基于雅可比相似度计算的方法来实现文本的语义归一化，并利用Apache Spark GraphX构建节点间的相似度图。通过计算文本中字符的频率分布并比较不同文本间的相似度，该方法能够有效地衡量文本之间的相似性。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

import org.apache.spark.graphx.{Edge, Graph}
import utility.Helpers

import scala.collection.mutable

/**
  * Created by fhqplzj on 2017/7/20.
  */
object SemanticNormalization {
  def word_count(s: String): mutable.Map[Char, Int] = {
    /**
      * word count的作用
      */
    val hashMap = new mutable.HashMap[Char, Int]().withDefaultValue(0)
    s.foreach {
      c =>
        hashMap(c) += 1
    }
    hashMap
  }

  def jaccard_similarity(s1: String, s2: String): Double = {
    /**
      * 广义雅可比相似度计算
      */
    val m1 = word_count(s1)
    val m2 = word_count(s2)
    val ks1 = m1.keySet
    val ks2 = m2.keySet
    val numerator = (ks1 & ks2).toSeq.map {
      x =>
        Math.min(m1(x), m2(x))
    }.sum
    val denominator = (ks1 | ks2).toSeq.map {
      x =>
        Math.max(m1(x), m2(x))
    }.sum
    1.0 * numerator / denominator
  }

  def main(args: Array[String]): Unit = {
    val sc = Helpers.getSc
    val vertices = sc.textFile("file:///tmp/graph.txt").map {
      line =>
        val parts = line.split("\t")
        (parts(0), parts(1))
    }.zipWithIndex().map(_.swap)
    val similarities = vertices.cartesian(vertices).flatMap {
      case ((id1, (content1, _)), (id2, (content2, _))) =>
        val sim = jaccard_similarity(content1, content2)
        if (id1 < id2) {
          Seq(Edge(id1, id2, sim), Edge(id2, id1, sim))
        } else if (id1 == id2) {
          Seq(Edge(id1, id2, sim))
        } else {
          None
        }
    }
    val graph = Graph.fromEdges(similarities, 0.0)
    /*构建好了图，接下来做自己想做的事，比如提取图的连通分量。*/
    println(graph.vertices.count())
  }
}