scala写wordcount入门案例

最新推荐文章于 2022-08-04 20:31:30 发布

andycao1

最新推荐文章于 2022-08-04 20:31:30 发布

阅读量914

点赞数

CC 4.0 BY-SA版权

分类专栏： scala 文章标签： flume 大数据 big data

本文链接：https://blog.youkuaiyun.com/caoxueliangandy/article/details/121438684

scala 专栏收录该内容

7 篇文章

订阅专栏

本文介绍了两种使用Scala实现WordCount的方法。第一种通过将单词和计数转换为字符串列表，然后扁平化并分组计数。第二种方法直接从元组列表开始，通过分组和映射操作得到单词及其出现次数。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

复杂WordCount案例

1）方式一

object TestWordCount {

    def main(args: Array[String]): Unit = {

        // 第一种方式（不通用）
        val tupleList = List(("Hello Scala Spark World ", 4), ("Hello Scala Spark", 3), ("Hello Scala", 2), ("Hello", 1))

        val stringList: List[String] = tupleList.map(t=>(t._1 + " ") * t._2)

        //val words: List[String] = stringList.flatMap(s=>s.split(" "))
        val words: List[String] = stringList.flatMap(_.split(" "))

        //在map中，如果传进来什么就返回什么，不要用_省略
        val groupMap: Map[String, List[String]] = words.groupBy(word=>word)
        //val groupMap: Map[String, List[String]] = words.groupBy(_)

        // (word, list) => (word, count)
        val wordToCount: Map[String, Int] = groupMap.map(t=>(t._1, t._2.size))

        val wordCountList: List[(String, Int)] = wordToCount.toList.sortWith {
            (left, right) => {
                left._2 > right._2
            }
        }.take(3)

        //tupleList.map(t=>(t._1 + " ") * t._2).flatMap(_.split(" ")).groupBy(word=>word).map(t=>(t._1, t._2.size))
        println(wordCountList)
    }
}

2）方式二

object TestWordCount {

    def main(args: Array[String]): Unit = {

        val tuples = List(("Hello Scala Spark World", 4), ("Hello Scala Spark", 3), ("Hello Scala", 2), ("Hello", 1))

        // (Hello,4),(Scala,4),(Spark,4),(World,4)
        // (Hello,3),(Scala,3),(Spark,3)
        // (Hello,2),(Scala,2)
        // (Hello,1)
        val wordToCountList: List[(String, Int)] = tuples.flatMap {
            t => {
                val strings: Array[String] = t._1.split(" ")
                strings.map(word => (word, t._2))
            }
        }

        // Hello, List((Hello,4), (Hello,3), (Hello,2), (Hello,1))
        // Scala, List((Scala,4), (Scala,3), (Scala,2)
        // Spark, List((Spark,4), (Spark,3)
        // Word, List((Word,4))
        val wordToTupleMap: Map[String, List[(String, Int)]] = wordToCountList.groupBy(t=>t._1)

        val stringToInts: Map[String, List[Int]] = wordToTupleMap.mapValues {
            datas => datas.map(t => t._2)
        }
        stringToInts

        /*
        val wordToCountMap: Map[String, List[Int]] = wordToTupleMap.map {
            t => {
                (t._1, t._2.map(t1 => t1._2))
            }
        }

        val wordToTotalCountMap: Map[String, Int] = wordToCountMap.map(t=>(t._1, t._2.sum))
        println(wordToTotalCountMap)
        */
    }
}