import scala.collection.mutable.ArrayBuffer
def dfOfPartition(input: Iterator[Int]) = {
var a = ArrayBuffer[Int]()
input.foreach { elem => a+=elem}
a.toDF
}
val list = 1 to 100
val rdd = sc.parallelize(list,10)
val result = rdd.mapPartitions(partition => Iterator(dfOfPartition(partition)))
result.collect().foreach(println)
def dfOfPartition(input: Iterator[Int]) = {
var a = ArrayBuffer[Int]()
input.foreach { elem => a+=elem}
a.toDF
}
val list = 1 to 100
val rdd = sc.parallelize(list,10)
val result = rdd.mapPartitions(partition => Iterator(dfOfPartition(partition)))
result.collect().foreach(println)
扫一扫
2410
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
