java spark RDD 算子 WordCount

最新推荐文章于 2024-10-08 22:01:27 发布

原创最新推荐文章于 2024-10-08 22:01:27 发布 · 285 阅读

1 ·

CC 4.0 BY-SA版权

文章标签：

#spark #java算子 #map #flatmap #maptopair

spark 专栏收录该内容

21 篇文章

订阅专栏

本文介绍使用Apache Spark实现WordCount的全过程，从配置Spark环境到读取文本文件，再到使用flatMap、mapToPair和reduceByKey等函数进行单词计数，并展示最终的运行结果。

package examples;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.api.java.function.VoidFunction;
import scala.Tuple2;

import java.util.Arrays;
import java.util.Iterator;

/**
 * @Description
 * @Copyright credlink
 * @Author luzhen
 * @Create 2019/6/6 9:08
 */
public class wordCount3 {
    public static void main(String args[]) {
        countTest();
    }
    public static void countTest() {
        SparkConf conf = new SparkConf().setAppName("wordCount").setMaster("local[1]");
        JavaSparkContext jsc = new JavaSparkContext(conf);
        JavaRDD<String> lines = jsc.textFile("D:\\LICENSE");

        JavaRDD<String> words = lines.flatMap(
                new FlatMapFunction<String, String>() {
                    @Override
                    public Iterator<String> call(String s) throws Exception {
                        return Arrays.asList(s.split(" ")).iterator();
                    }
                }
        );
        JavaPairRDD<String, Integer> counts = words.mapToPair(
                new PairFunction<String, String, Integer>() {
                    @Override
                    public Tuple2<String, Integer> call(String s) throws Exception {
                        return new Tuple2<String, Integer>(s, 1);
                    }
                }
        );
        JavaPairRDD<String, Integer> results = counts.reduceByKey(
                new Function2<Integer, Integer, Integer>() {
                    @Override
                    public Integer call(Integer xx1, Integer xx2) throws Exception {
                        return xx1 + xx2;
                    }
                }
        );
        results.foreach(new VoidFunction<Tuple2<String, Integer>>() {
            @Override
            public void call(Tuple2<String, Integer> stringIntegerTuple2) throws Exception {
                System.out.println(stringIntegerTuple2);
            }
        });
    }
}

输出结果：

(additional,4)
(Unless,3)
(For,3)
(NON-INFRINGEMENT,,1)
(agree,1)
(reproduce,,1)
(offer,1)
(executed,1)
(event,1)
((or,3)
("Contributor",1)
(Grant,2)
(work.,1)
(include,3)
(content,1)
(nothing,1)
(MERCHANTABILITY,,1)
(add,2)
(through,1)
(However,,1)
(perform,,1)
(files;,1)
(result,1)
(been,2)
(goodwill,,1)
(herein,1)
(appropriateness,1)
(direct,,1)
(To,1)
(any,28)
(contract,,1)
(ANY,2)