Java Spark算子：distinct

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;

import java.util.Arrays;
import java.util.List;

/**
 * distinct（）算子
 * 去除RDD的重复元素。
 *
 */
public class DistinctDemo {

    public static void main(String[] args) {

        SparkConf conf = new SparkConf().setMaster("local").setAppName("spark");
        JavaSparkContext sc  = new JavaSparkContext(conf);
        List<String> list = Arrays.asList("a","b","c","a","b","c","d");
        JavaRDD<String> javaRDD = sc.parallelize(list);

        //distinct算子：去重
        JavaRDD<String> reduce = javaRDD.distinct();
        System.err.println(reduce.collect());
    }

}