sparksql 动态设置schema将rdd转换成dataset/dataframe

最新推荐文章于 2024-03-31 01:03:30 发布

转载最新推荐文章于 2024-03-31 01:03:30 发布 · 221 阅读

0 ·

CC 4.0 BY-SA版权

原文链接：http://www.cnblogs.com/tele-share/p/10371158.html

文章标签：

#大数据 #java #scala

本文详细介绍了如何使用Spark SQL将RDD转换为DataFrame，并通过具体示例展示了数据读取、转换及SQL查询的全过程。从Java和Scala两种语言的角度出发，深入探讨了数据结构的创建、Schema定义以及DataFrame的创建方法。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

java

 1 public class DynamicDemo {
 2     private static SparkConf conf = new SparkConf().setAppName("dynamicdemo").setMaster("local");
 3     private static JavaSparkContext jsc = new JavaSparkContext(conf);
 4     private static SparkSession session = new SparkSession(jsc.sc());
 5 
 6     public static void main(String[] args) {
 7 
 8         // 创建rdd
 9         JavaRDD<String> rdd = jsc.textFile("./src/main/java/cn/tele/spark_sql/rdd2dataset/students.txt");
10 
11         // 创建Row的rdd
12         JavaRDD<Row> rowRdd = rdd.map(new Function<String, Row>() {
13 
14             private static final long serialVersionUID = 1L;
15 
16             @Override
17             public Row call(String v1) throws Exception {
18                 String[] fields = v1.split(",");
19                 return RowFactory.create(Integer.valueOf(fields[0]), fields[1], Integer.valueOf(fields[2]));
20             }
21         });
22 
23         // 创建schema
24         StructType schema = DataTypes
25                 .createStructType(Arrays.asList(DataTypes.createStructField("id", DataTypes.IntegerType, false),
26                         DataTypes.createStructField("name", DataTypes.StringType, false),
27                         DataTypes.createStructField("age", DataTypes.IntegerType, false)));
28 
29         // 转换
30         Dataset<Row> dataset = session.createDataFrame(rowRdd, schema);
31 
32         dataset.createOrReplaceTempView("students");
33 
34         Dataset<Row> result = session.sql("select * from students where age<=18");
35         result.show();
36 　　　　 session.stop();
37         jsc.close();
38     }
39 }

scala

 1 object DynamicDemo {
 2   def main(args: Array[String]): Unit = {
 3     val conf = new SparkConf().setAppName("reflectdemo").setMaster("local")
 4 
 5     val sc = new SparkContext(conf)
 6 
 7     val sqlContext = new SQLContext(sc)
 8 
 9     //创建rdd
10     val rdd = sc.textFile("./src/main/scala/cn/tele/spark_sql/rdd2dataframe/students.txt", 8)
11 
12     val rowRdd = rdd.map(lines => {
13       val arr = lines.split(",");
14       Row(arr(0).trim().toInt, arr(1), arr(2).trim().toInt)
15     })
16 
17     val schema = DataTypes.createStructType(Array(
18       /*    DataTypes.createStructField("id",DataTypes.IntegerType,false),
19           DataTypes.createStructField("name",DataTypes.StringType,false),
20           DataTypes.createStructField("age",DataTypes.IntegerType,false)*/
21       StructField("id", DataTypes.IntegerType, false),
22       StructField("name", DataTypes.StringType, false),
23       StructField("age", DataTypes.IntegerType, false)))
24 
25     //转换
26     val dataframe = sqlContext.createDataFrame(rowRdd, schema)
27 
28     dataframe.createOrReplaceTempView("students")
29 
30     val result = sqlContext.sql("select * from students where age<=18")
31     result.show()
32   }
33 }