Spark-ClickHouse-ES实时项目第三天-ES工具类编写测试

仓库位置

日志生成：https://github.com/SmallScorpion/gmall-mock.git
日志服务器：https://github.com/SmallScorpion/gmall-spark-ch-es-realtime.git

ElasticSearch和Kibana安装

https://blog.youkuaiyun.com/qq_40180229/article/details/106012997

新建模块导入依赖

<dependency>
    <groupId>io.searchbox</groupId>
    <artifactId>jest</artifactId>
    <version>5.3.3</version>

</dependency>

<dependency>
    <groupId>net.java.dev.jna</groupId>
    <artifactId>jna</artifactId>
    <version>4.5.2</version>
</dependency>

<dependency>
    <groupId>org.codehaus.janino</groupId>
    <artifactId>commons-compiler</artifactId>
    <version>2.7.8</version>
</dependency>

<!-- https://mvnrepository.com/artifact/org.elasticsearch/elasticsearch -->
<dependency>
    <groupId>org.elasticsearch</groupId>
    <artifactId>elasticsearch</artifactId>
    <version>2.4.6</version>
</dependency>

ES工具类编写

package com.warehouse.gmall.realtime.util

import java.{lang, util}

import io.searchbox.client.config.HttpClientConfig
import io.searchbox.client.{JestClient, JestClientFactory}
import io.searchbox.core.{Index, Search, SearchResult}
import org.elasticsearch.index.query.{BoolQueryBuilder, MatchQueryBuilder, TermQueryBuilder}
import org.elasticsearch.search.builder.SearchSourceBuilder
import org.elasticsearch.search.highlight.HighlightBuilder
import org.elasticsearch.search.sort.SortOrder

object MyEsUtil {

  // 连接池
  var factory: JestClientFactory = null

  // 获取jest
  def getClient:JestClient ={
    // 懒加载
    if(factory==null)build();
    factory.getObject

  }

  // 加载
  def  build(): Unit ={
    factory=new JestClientFactory
    factory.setHttpClientConfig(new HttpClientConfig.Builder("http://hadoop102:9200" )
      // 是否多线程
      .multiThreaded(true)
      // 最大连接数(并发数)
      .maxTotalConnection(20)
      // 连接超时时间
      .connTimeout(10000).readTimeout(10000).build())
  }

  // 插入
  def addDoc(): Unit ={

    val jest: JestClient = getClient

    // Builder(可转化为json对象)
    val index: Index = new Index.Builder(Movie( "01", "tmp", "test" )).index("movie_test").`type`("_doc").id("01").build()

    val message: String = jest.execute( index ).getErrorMessage

    if(message != null){
      println(message)
    }


    jest.close()

  }

  def queryDoc(): Unit ={

    val jest: JestClient = getClient

    val str: String =
      """
        |{
        |  "query": {
        |    "bool": {
        |      "must": [
        |        {
        |          "match": {
        |            "name": "operation"
        |          }
        |        }
        |      ],
        |      "filter": {
        |        "term": {
        |          "actorList.name.keyword": "zhang han yu"
        |        }
        |      }
        |    }
        |  },
        |  "from": 0,
        |  "size": 20,
        |  "sort": [
        |    {
        |      "doubanScore": {
        |        "order": "desc"
        |      }
        |    }
        |  ],
        |  "highlight": {
        |    "fields": {
        |      "name": {}
        |    }
        |  }
        |
        |}
        |""".stripMargin


    // 使用对象创建方式，适合增改查询结构
    val boolQueryBuilder = new BoolQueryBuilder()
      .must( new MatchQueryBuilder( "name", "operation" ) )
      .filter( new TermQueryBuilder( "actorList.name.keyword", "zhang han yu" ) )

    val searchSourceBuilder = new SearchSourceBuilder()
      .query(boolQueryBuilder)
      .from( 0).size(20)
      .sort( "doubanScore", SortOrder.DESC )
      .highlight( new HighlightBuilder().field( "name" ) )

    val query2: String = searchSourceBuilder.toString

    val search = new Search.Builder(str).addIndex("movie_index").addType("movie").build()

    val result: SearchResult = jest.execute( search )

    // 结果封装使用需要用util
    val hits: util.List[SearchResult#Hit[util.Map[String, Any], Void]] = result.getHits( classOf[ util.Map[String, Any] ] )
    import scala.collection.JavaConversions._
    for ( hit <- hits ){
      println(hit.source.mkString(","))
    }

    val list: List[lang.Double] = hits.map( _.score ).toList
    println(hits.mkString("\n"))
    jest.close()

  }


  def main(args: Array[String]): Unit = {
    queryDoc()
  }


  case class Movie ( id: String,  movie_name: String, name: String )

}