Scala和Java代码实现的SprkAPI

scala+sparkAPI

package com.credi

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
  * Created by Administrator on 2019/2/7.
  */
object ScalaWordCountEasy {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()  //spark配置对象
    conf.setMaster("local")/*本地模式*/.setAppName("SWC")
    val sc = new SparkContext(conf)  //加载配置对象
    //以上spark程序必要步骤

    val lines = sc.textFile("./Words").flatMap(lines=>{
      lines.split(" ")
    }).map(word=>{
      new Tuple2(word , 1)  //单词 , 1
    }).reduceByKey(/*v1*/_+_/*v2*/).foreach(println)  //结果聚合后紧跟着直接遍历


    /*
    reduceByKey方法
    根据key是否相同 , 把元组分组
    相同key的value迭代聚合 , v1 1 + v2 1  => v1 2 + v2 1 => v1 3 + v2 1 => ....
    如果只有1条数据 ,不会进迭代聚合
     */

    sc.stop() //关闭
  }
}

java代码实现sparkAPI

package com.java;

import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.*;

import java.util.Arrays;
import java.util.List;

import org.codehaus.janino.Java;
import scala.Tuple2;
import scala.tools.cmd.gen.AnyVals;


/**
 * Created by Administrator on 2019/2/16.
 */
public class sprkAPI {
    public static void main(String[] args){
        SparkConf sparkConf = new SparkConf();
        sparkConf.setMaster("local").setAppName("wc");
        JavaSparkContext javaSparkContext = new JavaSparkContext(sparkConf);
        JavaRDD<String> javaRDD = javaSparkContext.textFile("./wc.txt");

        JavaRDD<String> words = javaRDD.flatMap(new FlatMapFunction<String, String>() {
            @Override
            public Iterable<String> call(String s) throws Exception {
                String[] split = s.split(" ");

                List<String> strings = Arrays.asList(split);
                return strings;
            }
        });

        PairFunction<String, String, Integer> f = new PairFunction<String, String, Integer>() {

            @Override
            public Tuple2<String, Integer> call(String s) throws Exception {
                return new Tuple2(s, 1);
            }
        };
        JavaPairRDD<String, Integer> wordMap = words.mapToPair(f);

        JavaPairRDD<String, Integer> rdd = wordMap.reduceByKey(new Function2<Integer, Integer, Integer>() {
            @Override
            public Integer call(Integer integer, Integer integer2) throws Exception {

                return integer + integer2;
            }
        });

        rdd.foreach(new VoidFunction<Tuple2<String, Integer>>() {
            @Override
            public void call(Tuple2<String, Integer> stringIntegerTuple2) throws Exception {
                System.out.println(stringIntegerTuple2._1+"  :  "+stringIntegerTuple2._2);
            }
        });
    }
}