ES-Hadoop学习笔记-Spark交互

elasticsearch-hadoop提供ElasticSearch与Apache Spark的本地集成。从ElasticSearch读取的数据是以Spark里RDD的形式进行操作，同时Spark的RDD的内容也可以被转换为文档，保存进ElasticSearch中进行查询。下面是两个交互的简单实例：

相关的依赖jar包

elasticsearch-spark_2.10-2.3.2.jar 

elasticsearch-spark_2.11-2.3.2.jar

elasticsearch-spark-1.2_2.10-2.3.2.jar

elasticsearch-spark-1.2_2.11-2.3.2.jar

ElasticSearch -> Spark

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.elasticsearch.spark.rdd.EsSpark
import org.apache.spark.sql.SQLContext

object E2SExample {

  def main(args: Array[String]) {
    var conf = new SparkConf()
    conf.setAppName("S2EExample")
    conf.setMaster("spark://master:7077")
    conf.set("es.nodes", "master:9200")
    val sc = new SparkContext(conf)

    val rdd = EsSpark.esRDD(sc, "data/t1", "?q=王*")
    println("rdd count: " + rdd.count())

    rdd.collect().foreach(record => {
      print(record._1 + ":")
      for ((k, v) <- record._2) {
        print(k + ":" + v)
      }
      println()
    })

    val sqlContext = new SQLContext(sc)
    val df = sqlContext.read.format("org.elasticsearch.spark.sql").load("data/t1")
    df.printSchema()
    df.collect().foreach(println)

    sc.stop()
  }
}

Spark -> ElasticSearch

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.elasticsearch.spark.rdd.EsSpark

case class Job(jobName:String, jobUrl:String, companyName:String, salary:String)

object S2EExample {
  
  def main(args:Array[String]) {
    var conf = new SparkConf()
    conf.setAppName("S2EExample")
    conf.setMaster("spark://master:7077")
    conf.set("es.index.auto.create", "true")
    conf.set("es.nodes", "master:9200")
    val sc = new SparkContext(conf)
    
    val job1 = Job("C开发工程师","http://job.c.com","c公司","10000")
    val job2 = Job("C++开发工程师","http://job.c++.com","c++公司","10000")
    val job3 = Job("C#开发工程师","http://job.c#.com","c#公司","10000")
    val job4 = Job("Java开发工程师","http://job.java.com","java公司","10000")
    val job5 = Job("Scala开发工程师","http://job.scala.com","java公司","10000")
    
    val rdd = sc.makeRDD(Seq(job1, job2, job3, job4, job5))
    EsSpark.saveToEs(rdd, "data/job")
    
    sc.stop()
  }
  
}

$bin/spark-submit --class org.esspark.examples.S2EExample --master spark://master:7077 

--jars ../elasticsearch-hadoop-2.3.2/dist/elasticsearch-spark_2.10-2.3.2.jar 

../temp/esspark-0.0.1-SNAPSHOT.jar