spark sql在scala与java中的代码实现

在编写spark sql代码前，需要新建maven工程，将hadoop下的配置文件core-site.xml和hdfs-site.xml,以及hive中的hive-site.xml拷贝到工程的resource目录下，并在pom.xml中配置jar包信息。

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>begin</groupId>
    <artifactId>myspark</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <spark.version>2.4.3</spark.version>
        <scala.version>2.11.12</scala.version>
    </properties>
    <dependencies>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.11</artifactId>
            <version>2.4.3</version>
        </dependency>

        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-library</artifactId>
            <version>2.11.12</version>
        </dependency>

        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-reflect</artifactId>
            <version>2.11.12</version>
        </dependency>

        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-compiler</artifactId>
            <version>2.11.12</version>
        </dependency>
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.28</version>
        </dependency>


        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.47</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
        </dependency>

        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.11</artifactId>
            <version>2.4.3</version>
        </dependency>

        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-hive_2.11</artifactId>
            <version>2.4.3</version>
        </dependency>

    </dependencies>

</project>

scala实现

import org.apache.spark.sql.SparkSession

/**
  * 使用scala实现spark sql访问
  */
object SparkSqlDemoScala {
  def main(args: Array[String]):Unit= {
    val spark=SparkSession.builder().appName("SparkSql").master("local[*]").enableHiveSupport().getOrCreate()
    val rdd1=spark.sparkContext.textFile("/user/hadoop/data2/wc.txt")
    val rdd2=rdd1.flatMap(_.split(" "))
    //导入sparksession的隐式转换
    import spark.implicits._
    //将rdd转换成数据框
    val df=rdd2.toDF("word")
    //将数据框注册成临时视图
    df.createOrReplaceTempView("_doc")
    spark.sql("select word,count(*) from _doc group by word").show(1000,false)
  }
}

JAVA实现：

import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.Metadata;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;

import java.util.Arrays;
import java.util.Iterator;

/**
 * 使用java实现spark sql访问
 */
public class SparkSQLDemoJava {
    public static void main(String[] args) {
        SparkSession spark= SparkSession.builder().appName("sparkSQL").master("local").enableHiveSupport().getOrCreate();
        //创建javaSpark上下文
        JavaSparkContext sc=new JavaSparkContext(spark.sparkContext());
        //加载文件
        JavaRDD<String> rdd1=sc.textFile("/user/hadoop/data2/wc.txt");
        JavaRDD<String> rdd2=rdd1.flatMap(new FlatMapFunction<String,String>(){
            public Iterator<String> call(String s) throws Exception{
                return Arrays.asList(s.split(" ")).iterator();
            }
        });
        //将string 变换成 row
        JavaRDD<Row> rdd3=rdd2.map(new Function<String,Row>(){
           public Row call(String word) throws Exception{
               return RowFactory.create(word);
           }
        });
        //构造表结构
        StructField[] fields=new StructField[1];
        fields[0]=new StructField("word", DataTypes.StringType,true, Metadata.empty());
        //表结构类型
        StructType type=new StructType(fields);
        //将RDD转换成DataFrame
        Dataset<Row> df=spark.createDataFrame(rdd3,type);
        //注册临时视图
        df.createOrReplaceTempView("_doc");

        spark.sql("select word,count(*) from _doc group by word").show(1000,false);

    }
}