在使用sparkStreaming任务实时加载机器学习CTR模型并做实时预测时,调用transform方法抛出java.util.NoSuchElementException: next on empty iterator异常。
...
//加载模型
val model =PipelineModel.load("/user/hdfs/model/LRmodel")
dataStream.foreachRDD(rdd => {
val sqlContext = SQLContext.getOrCreate(rdd.sparkContext)
import sqlContext.implicits._
val dataFrame = rdd.toDF()
val prediction = model.transform(dataFrame)
...
})
任务运行后,抛出异常
ERROR JobScheduler JobScheduler: Error running job streaming job
java.util.NoSuchElementException: next on empty iterator
at scala.collection.Iterator$$anon$2.next(Iterator.scala:39)
at scala.collection.Iterator$$anon$2.next(Iterator.scala:37)
at scala.collection.IndexedSeqLike$Elements.next(IndexedSeqLike.scala:63)
at scala.collection.IterableLike$class.head(IterableLike.scala:107)
at scala.collection.mutable.ArrayOps$ofRef.scala$collection$IndexedSeqOptimized$$super$head(ArrayOps.scala:186)
at scala.collection.IndexedSeqOptimized$class.head(IndexedSeqOptimized.scala:126)
at scala.collection.mutable.ArrayOps$ofRef.head(ArrayOps.scala:186)
at org.apache.spark.sql.Dataset.head(Dataset.scala:2502)
at org.apache.spark.sql.Dataset.first(Dataset.scala:2509)
at org.apache.spark.ml.feature.VectorAssembler.first$lzycompute$1(VectorAssembler.scala:57)
at org.apache.spark.ml.feature.VectorAssembler