ElasticSearch自定义pinyin和ik分词库

---

1 语料库映射OpenAPI

环境准备:

1. 先下载ik分词和pinyin分词，并放到esplugins相应目录中

请求kibana:GET /_cat/plugins?v&s=component&h=name,component,version,description

结果

name            component   version description
WIN-A5KARTU1A65 analysis-ik 7.10.1  IK Analyzer for Elasticsearch
WPhvS8c analysis-pinyin    7.10.1   Pinyin Analysis for Elasticsearch

2. 定义ik分词后的pinyin分词器，即定义一个自定义分词器ik_pinyin_analyzer

PUT test_index
{
  "settings":{
    "number_of_shards":"1",
    "index.refresh_interval":"15s",
    "index":{
      "analysis":{
        "analyzer":{
           "ik_pinyin_analyzer":{
            "type":"custom",
            "tokenizer":"ik_smart",
            "filter":"pinyin_filter"
          }
        },
        "filter":{
          "pinyin_filter":{
            "type":"pinyin",
            "keep_first_letter": false
          }
        }
      }
    }
  }
}

下面的目的就是用API实现这种效果

这里tokenizer使用ik分词，分词之后将分词结果通过pinyin再filter一次，这样就可以了。

测试一下

POST test_index/_analyze
{
  "analyzer": "ik_pinyin_analyzer",
  "text":"测试"
}

结果

{
  "tokens": [
    {
      "token": "ce",
      "start_offset": 0,
      "end_offset": 2,
      "type": "CN_WORD",
      "position": 0
    },
    {
      "token": "shi",
      "start_offset": 0,
      "end_offset": 2,
      "type": "CN_WORD",
      "position": 1
    }
  ]
}

这样，当我们建立index的mapping的时候，就可以像使用ik_smart分词器一样使用ik_pinyin_analyzer

比如lawbasis字段的mapping可以是这样的

PUT test_index/_mapping/test_type
{
  "properties": {
    "lawbasis":{
      "type": "text",
      "analyzer": "ik_smart",
      "search_analyzer": "ik_smart",
      "fields": {
        "my_pinyin":{
          "type":"text",
          "analyzer": "ik_pinyin_analyzer",
          "search_analyzer": "ik_pinyin_analyzer"
        }
      }
    }
  }
}

其中field满足以不同的目的以不同的方式为相同的字段编制索引，也就是说lawbasis这个field会以中文ik_smart分词以及分词后的pinyin分词来编制索引，并支持中文和拼音搜索。

4. 测试一下

加入两条数据

POST test_index/test_type
{
  "lawbasis":"测试一下"
}
POST test_index/test_type
{
  "lawbasis":"测试东西"
}

使用拼音搜索

GET test_index/test_type/_search
{
  "query":{
    "match": {
      "lawbasis.my_pinyin": "ceshi"
    }
  }
}

可以看到有两条结果

1.1 定义索引（映射）接口

package com.oldlu.service;
import com.oldlu.commons.pojo.CommonEntity;
import org.elasticsearch.rest.RestStatus;
import java.util.List;
import java.util.Map;
/**
* @Class: ElasticsearchIndexService
* @Package com.oldlu.service
* @Description: 索引操作接口
* @Company: oldlu
*/
public interface ElasticsearchIndexService {
  //新增索引+映射
  boolean addIndexAndMapping(CommonEntity commonEntity) throws Exception;
}

1.2 定义索引（映射）实现

/**
* @Class: ElasticsearchIndexServiceImpl
* @Package com.oldlu.service.impl
* @Description: 索引操作实现类
* @Company: oldlu
*/
@Service("ElasticsearchIndexServiceImpl")
public class ElasticsearchIndexServiceImpl implements ElasticsearchIndexService
{
  @Resource
  private RestHighLevelClient client;
  private static final int START_OFFSET = 0;
  private static final int MAX_COUNT = 5;
  /*
  * @Description: 新增索引+setting+映射+自定义分词器pinyin
  * setting可以为空（自定义分词器pinyin在setting中）
  * 映射可以为空
  * @Method: addIndexAndMapping
  * @Param: [commonEntity]
  * @Update:
  * @since: 1.0.0
  * @Return: boolean
  *
  */
 public boolean addIndexAndMapping(CommonEntity commonEntity) throws
Exception {
    //设置setting的map
    Map<String, Object> settingMap = new HashMap<String, Object>();
    //创建索引请求
    CreateIndexRequest request = new
CreateIndexRequest(commonEntity.getIndexName());
    //获取前端参数
    Map<String, Object> map = commonEntity.getMap();
    //循环外层的settings和mapping
    for (Map.Entry<String, Object> entry : map.entrySet()) {
      if ("settings".equals(entry.getKey())) {
        if (entry.getValue() instanceof Map && ((Map)
entry.getValue()).size() > 0) {
          request.settings((Map<String, Object>) entry.getValue());
       }
     }
      if ("mapping".equals(entry.getKey())) {
        if (entry.getValue() instanceof Map && ((Map)
entry.getValue()).size() > 0) {
          request.mapping((Map<String, Object>) entry.getValue());
       }
     }
   }
    //创建索引操作客户端
    IndicesClient indices = client.indices();
    //创建响应对象
    CreateIndexResponse response = indices.create(request,
RequestOptions.DEFAULT);
    //得到响应结果
    return response.isAcknowledged();
 }
 
}

1.3 新增控制器

package com.oldlu.controller;
import com.oldlu.commons.enums.ResultEnum;
import com.oldlu.commons.enums.TipsEnum;
import com.oldlu.commons.pojo.CommonEntity;
import com.oldlu.commons.result.ResponseData;
import com.oldlu.service.ElasticsearchIndexService;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
/**
* @Class: ElasticsearchIndexController
* @Package com.oldlu.controller
* @Description: 索引操作控制器
* @Company: oldlu
*/
@RestController
@RequestMapping("v1/indices")
public class ElasticsearchIndexController {
  private static final Logger logger = LoggerFactory
     .getLogger(ElasticsearchIndexController.class);
  @Autowired
  ElasticsearchIndexService elasticsearchIndexService;
  /*
  * @Description: 新增索引、映射
  * @Method: addIndex
  * @Param: [commonEntity]
  * @Update:
  * @since: 1.0.0
  * @Return: com.oldlu.commons.result.ResponseData
  *
  */
  @PostMapping(value = "/add")
  public ResponseData addIndexAndMapping(@RequestBody CommonEntity
commonEntity) {
    //构造返回数据
    ResponseData rData = new ResponseData();
    if (StringUtils.isEmpty(commonEntity.getIndexName())) {
      rData.setResultEnum(ResultEnum.PARAM_ISNULL);
      return rData;
   }
    //增加索引是否成功
    boolean isSuccess = false;
    try {
      //通过高阶API调用增加索引方法
      isSuccess =
elasticsearchIndexService.addIndexAndMapping(commonEntity );
      //构建返回信息通过类型推断自动装箱（多个参数取交集）
      rData.setResultEnum(isSuccess, ResultEnum.SUCCESS, 1);
      //日志记录
      logger.info(TipsEnum.CREATE_INDEX_SUCCESS.getMessage());
   } catch (Exception e) {
      //日志记录
      logger.error(TipsEnum.CREATE_INDEX_FAIL.getMessage(), e);
      //构建错误返回信息
      rData.setResultEnum(ResultEnum.ERROR);
   }
    return rData;
 }
}

1.4 开始新增映射

http://172.17.0.225:8888/v1/indices/add
或者
http://127.0.0.1:8888/v1/indices/add

参数
自定义分词器ik_pinyin_analyzer（ik和pinyin组合分词器）

tips 在创建映射前，需要安装拼音插件

{
 "indexName": "product_completion_index",
 "map": {
   "settings": {
     "number_of_shards": 1,
     "number_of_replicas": 2,
     "analysis": {
       "analyzer": {
         "ik_pinyin_analyzer": {
           "type": "custom",
           "tokenizer": "ik_smart",
           "filter": "pinyin_filter"
         }
       },
       "filter": {
         "pinyin_filter": {
           "type": "pinyin",
           "keep_first_letter": true,
           "keep_separate_first_letter": false,
           "keep_full_pinyin": true,
           "keep_original": true,
           "limit_first_letter_length": 16,
           "lowercase": true,
           "remove_duplicated_term": true
         }
       }
     }
   },
   "mapping": {
     "properties": {
       "name": {
         "type": "keyword"
       },
       "searchkey": {
         "type": "completion",
         "analyzer": "ik_pinyin_analyzer"
       }
     }
   }
 }
}

settings下面的为索引的设置信息，动态设置参数，遵循DSL写法
mapping下为映射的字段信息，动态设置参数，遵循DSL写法

返回

{
 "code": "200",
 "desc": "操作成功！",
 "data": true
}

2 语料库文档OpenAPI

2.1 定义批量新增文档接口

package com.oldlu.service;
import com.oldlu.commons.pojo.CommonEntity;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
import java.util.List;
import java.util.Map;
/**
* @Class: ElasticsearchDocumentService
* @Package com.oldlu.service
* @Description: 文档操作接口
* @Company: 
*/
public interface ElasticsearchDocumentService {
  //批量新增文档
  public RestStatus bulkAddDoc(CommonEntity commonEntity) throws Exception;
}

2.2 定义批量新增文档实现

/*
  * @Description: 批量新增文档,可自动创建索引、自动创建映射
  * @Method: bulkAddDoc
  * @Param: [indexName, map]
  * @Update:
  * @since: 1.0.0
  * @Return: org.elasticsearch.rest.RestStatus
  *
  */
 @Override
 public RestStatus bulkAddDoc(CommonEntity commonEntity) throws Exception {
   //通过索引构建批量请求对象
   BulkRequest bulkRequest = new BulkRequest(commonEntity.getIndexName());
   //循环前台list文档数据
   for (int i = 0; i < commonEntity.getList().size(); i++) {
     bulkRequest.add(new IndexRequest().source(XContentType.JSON,
SearchTools.mapToObjectGroup(commonEntity.getList().get(i))));
   }
   //执行批量新增
   BulkResponse bulkResponse = client.bulk(bulkRequest,
RequestOptions.DEFAULT);
   return bulkResponse.status();
 }

官方文档介绍https://www.elastic.co/guide/en/elasticsearch/client/java-rest/7.4/java-rest-high-document-bulk.html

如上图，需要定义成箭头中的形式
所以上面SearchTools.mapToObjectGroup将map转成了数组

2.3 定义批量新增文档控制器

/*
  * @Description: 批量新增文档,可自动创建索引、自动创建映射
  * @Method: bulkAddDoc
  * @Param: [indexName, map]
  * @Update:
  * @since: 1.0.0
  * @Return: org.elasticsearch.rest.RestStatus
  *
  */
  @PostMapping(value = "/batch")
  public ResponseData bulkAddDoc(@RequestBody CommonEntity commonEntity) {
    //构造返回数据
    ResponseData rData = new ResponseData();
    if (StringUtils.isEmpty(commonEntity.getIndexName()) ||
CollectionUtils.isEmpty(commonEntity.getList())) {
      rData.setResultEnum(ResultEnum.PARAM_ISNULL);
      return rData;
 }
    //批量新增操作返回结果
    RestStatus result = null;
    try {
      //通过高阶API调用批量新增操作方法
      result = elasticsearchDocumentService.bulkAddDoc(commonEntity);
      //通过类型推断自动装箱（多个参数取交集）
      rData.setResultEnum(result, ResultEnum.SUCCESS, null);
      //日志记录
      logger.info(TipsEnum.BATCH_CREATE_DOC_SUCCESS.getMessage());
   } catch (Exception e) {
      //日志记录
      logger.info(TipsEnum.BATCH_CREATE_DOC_FAIL.getMessage(), e);
      //构建错误返回信息
      rData.setResultEnum(ResultEnum.ERROR);
   }
    return rData;
 }

2.4 开始批量新增调用

http://172.17.0.225:8888/v1/docs/batch
或者
http://127.0.0.1:8888/v1/docs/batch

参数
定义23个suggest词库（定义了两个小米手机，验证是否去重）

{
 "indexName": "product_completion_index",
 "list": [
   {
     "searchkey": "小米手机",
     "name": "小米(MI)"
   },
   {
     "searchkey": "小米10",
     "name": "小米(MI)"
   },
   {
     "searchkey": "小米电视",
     "name": "小米(MI)"
   },
   {
     "searchkey": "小米路由器",
     "name": "小米(MI)"
   },
   {
     "searchkey": "小米9",
     "name": "小米(MI)"
   },
   {
     "searchkey": "小米手机",
"name": "小米(MI)"
   },
   {
     "searchkey": "小米耳环",
     "name": "小米(MI)"
   },
   {
     "searchkey": "小米8",
     "name": "小米(MI)"
   },
   {
     "searchkey": "小米10Pro",
     "name": "小米(MI)"
   },
   {
     "searchkey": "小米笔记本",
     "name": "小米(MI)"
   },
   {
     "searchkey": "小米摄像头",
     "name": "小米(MI)"
   },
   {
     "searchkey": "小米电饭煲",
     "name": "小米(MI)"
   },
   {
     "searchkey": "小米充电宝",
     "name": "小米(MI)"
   },
   {
     "searchkey": "adidas男鞋",
     "name": "adidas男鞋"
   },
   {
     "searchkey": "adidas女鞋",
     "name": "adidas女鞋"
   },
   {
     "searchkey": "adidas外套",
     "name": "adidas外套"
   },
   {
     "searchkey": "adidas裤子",
     "name": "adidas裤子"
   },
   {
     "searchkey": "adidas官方旗舰店",
     "name": "adidas官方旗舰店"
   },
   {
     "searchkey": "阿迪达斯袜子",
     "name": "阿迪达斯袜子"
   },
   
    {
     "searchkey": "阿迪达斯外套",
     "name": "阿迪达斯外套"
   },
   {
     "searchkey": "阿迪达斯运动鞋",
     "name": "阿迪达斯运动鞋"
   },
   {
     "searchkey": "耐克外套",
     "name": "耐克外套"
   },
   {
     "searchkey": "耐克运动鞋",
     "name": "耐克运动鞋"
   }
 ]
}

返回

{
 "code": "200",
 "desc": "操作成功！",
 "data": "OK"
}

查看GET product_completion_index/_search