spark定制之三：MySchemaRDD

最新推荐文章于 2025-02-17 21:13:16 发布

原创最新推荐文章于 2025-02-17 21:13:16 发布 · 1.1k 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#spark #schemardd

shell 专栏收录该内容

9 篇文章

订阅专栏

本文介绍了如何在Spark中创建一个自定义的MySchemaRDD类，该类扩展了SchemaRDD并提供了go()方法用于打印记录，以及saveto()方法用于将数据保存到指定的输出路径，支持HDFS和本地文件系统。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

org.apache.spark.repl.Main.interp.command("""
class MySchemaRDD(rdd:org.apache.spark.sql.SchemaRDD) extends java.io.Serializable {

    def go() = {
        var startstr = ""
        var endstr = RECORD_SEPERATOR
        val result = rdd.collect
        result.foreach( x =>
            print(x.mkString(startstr,FIELD_SEPERATOR,endstr))
          )
    }

    def saveto(output: String) = {
        import org.apache.hadoop.io.{NullWritable,Text}
        var startstr = ""
        var endstr = RECORD_SEPERATOR
        if(output.startsWith("hdfs:")) {
            val outputpath = AutoFileUtil.regularFile(output)
            FileUtil.deletePath(outputpath)
            rdd.map(x =>
                  (NullWritable.get(), new Text(x.mkString(FIELD_SEPERATOR)))
                ).saveAsHadoopFile[
                  org.apache.hadoop.mapred.TextOutputFormat[NullWritable, Text]
                ](outputpath)
        } else {
            val outputpath = FileUtil.regularFile(output)
            AutoFileUtil.deletePath(outputpath)
            val result = rdd.collect()
            val writer = new java.io.FileWriter(output)
            result.foreach(x =>
                writer.write(x.mkString(startstr,FIELD_SEPERATOR,endstr))
              )
            writer.close()
        }
    }
}
object MySchemaRDD {
    implicit def toMySchemaRDD(rdd:org.apache.spark.sql.SchemaRDD) = new MySchemaRDD(rdd)
}
""")