scala> // 导入必要的包
scala> import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.SparkSession
scala> import org.elasticsearch.spark.sql._
import org.elasticsearch.spark.sql._
scala>
scala> // 创建SparkSession实例
scala> val spark = SparkSession.builder()
spark: org.apache.spark.sql.SparkSession.Builder = org.apache.spark.sql.SparkSession$Builder@71e5cd05
scala> .appName("ElasticsearchReadExample")
res0: org.apache.spark.sql.SparkSession.Builder = org.apache.spark.sql.SparkSession$Builder@71e5cd05
scala> .getOrCreate()
res1: org.apache.spark.sql.SparkSession = org.apache.spark.sql.SparkSession@61267fa2
scala>
scala> // 查看spark变量的类型，确保是SparkSession
scala> println(spark.getClass)
class org.apache.spark.sql.SparkSession$Builder
scala>
scala> val defaultQuery: String = "?q=phone_no:5143217"
defaultQuery: String = ?q=phone_no:5143217
scala> val esTable = "mediamatch_usermsg"
esTable: String = mediamatch_usermsg
scala> val options = Map(
| ("es.nodes", "master"),
| ("es.port", "9200"),
| ("es.read.metadata", "false"),
| ("es.mapping.date.rich", "false"),
| ("es.net.http.auth.user", "elastic"),
| ("es.net.http.auth.pass", "i55on9YR90t+r8z8-OSpi"),
| ("es.nodes.wan.only", "true")
| )
options: scala.collection.immutable.Map[String,String] = Map(es.nodes.wan.only -> true, es.net.http.auth.user -> elastic, es.net.http.auth.pass -> i55on9YR90t+r8z8-OSpi, es.mapping.date.rich -> false, es.port -> 9200, es.read.metadata -> false, es.nodes -> master)
scala>
scala> val esDf = spark.esDF(esTable, defaultQuery, options)
<console>:30: error: value esDF is not a member of org.apache.spark.sql.SparkSession.Builder
val esDf = spark.esDF(esTable, defaultQuery, options)
^
scala> esDf.select("phone_no", "owner_name", "owner_code", "run_name", "run_time").show()
<console>:27: error: not found: value esDf
esDf.select("phone_no", "owner_name", "owner_code", "run_name", "run_time").show()
^
scala> val df = spark.read.format("org.elasticsearch.spark.sql")
<console>:27: error: value read is not a member of org.apache.spark.sql.SparkSession.Builder
val df = spark.read.format("org.elasticsearch.spark.sql")
^
scala> .options(options)
<console>:28: error: value options is not a member of scala.collection.immutable.Map[String,String]
options .options(options)
^
scala> .load(s"$esTable/$defaultQuery")
<console>:30: error: value load is not a member of scala.collection.immutable.Map[String,String]
options .load(s"$esTable/$defaultQuery")
^
scala>
scala> df.select("phone_no", "owner_name", "owner_code", "run_name", "run_time").show()
<console>:27: error: not found: value df
df.select("phone_no", "owner_name", "owner_code", "run_name", "run_time").show()
^
scala> import org.elasticsearch.spark.sql._ // 确保此行有效
import org.elasticsearch.spark.sql._
scala> implicit none => _root_.org.elasticsearch.spark.sql.EsSparkSQL.registerFunctions(spark.sqlContext)
<console>:1: error: expected start of definition
implicit none => _root_.org.elasticsearch.spark.sql.EsSparkSQL.registerFunctions(spark.sqlContext)
^
scala>
最新发布
本文介绍了作者开始尝试使用VSTS和DSL的经历，并分享了相关的下载链接、支持网站及视频教程等资源。此外还推荐了一些DSM工具和相关知识的学习网站。
扫一扫
253
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
