错误

Scala文档转换算法解析

最新推荐文章于 2024-12-30 19:34:23 发布

原创最新推荐文章于 2024-12-30 19:34:23 发布 · 335 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#scala

学习专栏收录该内容

45 篇文章

订阅专栏

本文介绍了一种使用Scala实现的文档转换算法，该算法通过哈希函数将输入文档中的项映射到特征向量中，并记录了每个特征的频率。此外，还探讨了如何从数据集中提取关键词及其对应的哈希值，以便于后续处理。



  def transform(document: Iterable[_]): Vector = {
   val keywordMap = scala.collection.mutable.Map(1 -> "name")
    val termFrequencies = mutable.HashMap.empty[Int, Double]
    val setTF = if (binary) (i: Int) => 1.0 else (i: Int) => termFrequencies.getOrElse(i, 0.0) + 1.0
    val hashFunc: Any => Int = getHashFunction
    document.foreach { term =>
      val i = nonNegativeMod(hashFunc(term), numFeatures)
      keywordMap += (i -> term.toString)
      termFrequencies.put(i, setTF(i))
    }
    println(keywordMap.size)
    Vectors.sparse(numFeatures, termFrequencies.toSeq)
  }

def tran(document: Iterable[_]): mutable.Map[Int,String] = {
    val termFrequencies = mutable.HashMap.empty[Int, String]
    val hashFunc: Any => Int = getHashFunction
    document.foreach { term =>
      val i = nonNegativeMod(hashFunc(term), numFeatures)
      termFrequencies.put(i, term.toString)
    }
    termFrequencies
  }

这个错。。。。。。。。。。

 def getdict[D <: Iterable[_]](dataset: RDD[D]): mutable.Map[Int,String] = {
    var keyMap = scala.collection.mutable.Map(1 -> "name")
    val a: RDD[mutable.Map[Int,String]]=dataset.map(this.tran)
     a.foreach{line=>
       keyMap=keyMap.++(line)
     }
    keyMap
  }

def getdict[D <: Iterable[_]](dataset: RDD[D]): mutable.Map[Int,String] = {
    val keyMap = mutable.HashMap.empty[Int, String]
    val a: RDD[mutable.Map[Int,String]]=dataset.map(this.tran)
     a.foreach{line=>line.foreach{x=>
       keyMap.put(x._1,x._2)
     }
       println(keyMap.size)
     }
    println(keyMap.size)
    keyMap
  }