elasticsearch-全量

最新推荐文章于 2022-05-05 11:23:45 发布
最新推荐文章于 2022-05-05 11:23:45 发布
阅读量252 收藏
点赞数
分类专栏： 学习 文章标签： java elasticsearch
原文链接：https://www.cnblogs.com/LQBlog/p/10566845.html
版权

maven依赖

官方客户端 https://www.elastic.co/guide/en/elasticsearch/client/java-rest/6.4/index.html

<dependency>
                <groupId>org.elasticsearch</groupId>
                <artifactId>elasticsearch</artifactId>
                <version>6.5.0</version>
                <optional>true</optional>
            </dependency>
            <dependency>
                <groupId>org.elasticsearch.client</groupId>
                <artifactId>elasticsearch-rest-high-level-client</artifactId>
                <version>6.5.0</version>
                <exclusions>
                    <exclusion>
                        <artifactId>commons-codec</artifactId>
                        <groupId>commons-codec</groupId>
                    </exclusion>
                    <!--此处要排除掉自带的，这个自带的版本低，会报错-->
                    <exclusion>
                        <artifactId>elasticsearch</artifactId>
                        <groupId>org.elasticsearch</groupId>
                    </exclusion>
                </exclusions>
            </dependency>

Util类

配置类

package com.crb.ocms.product.domain.config;

import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;

/**
 * @Project crb-product-domain
 * @PackageName com.crb.ocms.product.admin.config
 * @ClassName ESConfiguration
 * @Author liqiang
 * @Date 2019/3/6 5:52 PM
 * @Description 用于加载es的相关配置
 */
@ConfigurationProperties(prefix = "esconfig")
@Data
public class ESConfiguration {
    /**
     * index别名
     */
    private String aliasName;
    
    /**
     * 索引名字
     */
    private String indexName;
    
    
    /**
     * 全量索引的最大处理线程大小
     */
    private int threadSize;
    
    /**
     * 全量索引并行执行每个线程每次执行数据大小
     */
    private int treadDataSize;
    
    /**
     * typename
     */
    private String typeName;
    
    /**
     * ESURL
     */
    private String esUrl;

    /**
     * es ip
      */
    private String host;

    /**
     * es端口
     */
    private Integer port;

    /***
     *   用于标识是否正在处理的rediskey
     */
    private String redissKey;

    /**
     *mappingJson
     */
    private String mappingJson;
}

util工具类

package com.crb.ocms.product.service.impl;

import com.crb.ocms.product.domain.config.ESConfiguration;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpHost;
import org.apache.http.entity.ContentType;
import org.apache.http.nio.entity.NStringEntity;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.support.replication.ReplicationResponse;
import org.elasticsearch.client.*;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.reindex.BulkByScrollResponse;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.search.suggest.SuggestBuilder;
import org.elasticsearch.search.suggest.SuggestBuilders;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.util.*;
import java.util.regex.Pattern;


/**
 * @Project crb-product-service
 * @PackageName com.crb.ocms.demo.service.test.impl
 * @ClassName ESHLRestUtil
 * @Author liqiang
 * @Date 2019/2/25 13:45
 * @Description es工具类
 */
@Slf4j
@Component
@EnableConfigurationProperties(ESConfiguration.class)
public class ESHLRestUtil {
    private RestHighLevelClient client = null;

    public RestHighLevelClient getClient() {
        return client;
    }

    public void setClient(RestHighLevelClient client) {
        this.client = client;
    }
    ESConfiguration esConfiguration;

    @Autowired
    public ESHLRestUtil(ESConfiguration configuration) {
        client = new SimpleRestHighLevelClient(RestClient.builder(new HttpHost(configuration.getHost(), configuration.getPort(), "http")));
        this.esConfiguration=configuration;
    }
    public SimpleRestHighLevelClient getSimpleClient(){
        return (SimpleRestHighLevelClient)client;
    }


    /**
     *根据indexname获得index名字 支持通配符匹配
     * @param indexName
     * @return
     */
    public   String [] getIndexNames(String indexName)  {

        GetIndexResponse getIndexResponse= null;
        try {
            getIndexResponse = getIndexInfo(indexName);
        } catch (IOException e) {
            e.printStackTrace();
        }
        return getIndexResponse!=null?getIndexResponse.getIndices():null;
    }

    /**
     * 根据index名字获得index信息
     * @param indexName
     * @return
     */
    public GetIndexResponse  getIndexInfo(String indexName) throws IOException {
      //GetIndexRequest request = new GetIndexRequest().indices(indexName);
//        try {
//            GetIndexResponse getIndexResponse =client.indices().get(request, RequestOptions.DEFAULT);
//            return  getIndexResponse;
//        } catch (IOException e) {
//            e.printStackTrace();
//        }
        //master_timeout
        /**
         * 因为现在是api使用6.5 线上是6.24 master_timeout   使用以下方式 替换掉
         */
        GetIndexRequest request = new GetIndexRequest().indices(indexName);
        String[] indices = request.indices() == null ? Strings.EMPTY_ARRAY : request.indices();
        String endpoint =indexName;
        Request httpequest = new Request("GET", endpoint);
        httpequest.addParameter("ignore_unavailable", "false");
        httpequest.addParameter("expand_wildcards", "open");
        httpequest.addParameter("allow_no_indices", "true");
        org.apache.http.HttpEntity entity = new NStringEntity("", ContentType.APPLICATION_JSON);
        httpequest.setEntity(entity);
        Response httpResponse= getClient().getLowLevelClient().performRequest(httpequest);
        GetIndexResponse getIndexResponse=getSimpleClient().parseGetIndexRespons(httpResponse);
        return  getIndexResponse;
    }

    /**
     * 获得最大index名字
     * @param names indexName_number 格式
     * @return
     */
    public  Integer getMaxIndexNumber(String []names) {
        if(names==null||names.length<=0){
            return null;
        }
        List<Integer> indexNumber=new ArrayList<Integer>();
        for (String name:
                names) {
            String[] arrs=name.split("_");
            if(arrs.length<=1){
                continue;
            }
            indexNumber.add(Integer.valueOf(arrs[1]));
        }
        indexNumber.sort(new Comparator<Integer>() {
            @Override
            public int compare(Integer o1, Integer o2) {
                 return o2 - o1;
            }
        });
        return CollectionUtils.isEmpty(indexNumber)?null:indexNumber.get(0);
    }
    /**
     * 验证索引是否存在
     *
     * @param index 索引名称
     * @return
     * @throws Exception
     */
    public boolean indexExists(String index) throws Exception {
        GetIndexRequest request = new GetIndexRequest();
        request.indices(index);
        boolean exists = client.indices().exists(request, RequestOptions.DEFAULT);
        return exists;
    }

    /**
     * 创建index
     *
     * @param index
     * @param indexType
     * @param properties 结构: {name:{type:text}} {age:{type:integer}}
     * @return
     * @throws Exception
     */
    public boolean indexCreate(String index, String indexType,
                               Map properties) throws Exception {

        if (indexExists(index)) {
            return true;
        }
        CreateIndexRequest request = new CreateIndexRequest(index);
        request.settings(Settings.builder().put("index.number_of_shards", 3)
                .put("index.number_of_replicas", 2));
        Map jsonMap = new HashMap<>();
        Map mapping = new HashMap<>();
        mapping.put("properties", properties);
        jsonMap.put(indexType, mapping);
        request.mapping(indexType, jsonMap);

        CreateIndexResponse createIndexResponse = client.indices().create(
                request,RequestOptions.DEFAULT);
        boolean acknowledged = createIndexResponse.isAcknowledged();
        return acknowledged;
    }

    /**
     * 删除指定索引
     * @param indexName
     * @return
     * @throws IOException
     */
    public boolean deleteIndex(String indexName) throws IOException {
        DeleteIndexRequest request = new DeleteIndexRequest(indexName);
        AcknowledgedResponse deleteIndexResponse = client.indices().delete(request, RequestOptions.DEFAULT);
        return deleteIndexResponse.isAcknowledged();
    }
    /**
     * 创建索引
     * @param index       索引名字
     * @param settiongs settiongs
     * @return
     * @throws Exception
     * @author lqiang
     */
    public boolean indexCreate(String index, String settiongs
    ) throws Exception {

        if (indexExists(index)) {
            return true;
        }
        CreateIndexRequest request = new CreateIndexRequest(index);
//        request.settings(Settings.builder().put("index.number_of_shards", 3)
//                .put("index.number_of_replicas", 2));
        request.source(settiongs, XContentType.JSON);

        CreateIndexResponse createIndexResponse = client.indices().create(
                request,RequestOptions.DEFAULT);
        boolean acknowledged = createIndexResponse.isAcknowledged();
        return acknowledged;
    }

    /**
     * 创建更新文档
     *
     * @param index
     * @param indexType
     * @param documentId
     * @param josonStr
     * @return
     * @throws Exception
     */
    public boolean documentCreate(String index, String indexType,
                                  String documentId, String josonStr) throws Exception {
        IndexRequest request = new IndexRequest(index, indexType, documentId);

        request.source(josonStr, XContentType.JSON);
        IndexResponse indexResponse = client.index(request, RequestOptions.DEFAULT);

        if (indexResponse.getResult() == DocWriteResponse.Result.CREATED
                || indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
            return true;
        }
        ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo();
        if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
            return true;
        }
        if (shardInfo.getFailed() > 0) {
            for (ReplicationResponse.ShardInfo.Failure failure : shardInfo
                    .getFailures()) {
                throw new Exception(failure.reason());
            }
        }
        return false;
    }

    /**
     * 创建更新文档
     *
     * @param index
     * @param indexType
     * @param documentId
     * @param map
     * @return
     * @throws Exception
     */
    public boolean documentCreate(String index, String indexType,
                                  String documentId, Map map) throws Exception {
        IndexRequest request = new IndexRequest(index, indexType, documentId);

        request.source(map);
        IndexResponse indexResponse = client.index(request,RequestOptions.DEFAULT);

        if (indexResponse.getResult() == DocWriteResponse.Result.CREATED
                || indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
            return true;
        }
        ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo();
        if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
            return true;
        }
        if (shardInfo.getFailed() > 0) {
            for (ReplicationResponse.ShardInfo.Failure failure : shardInfo
                    .getFailures()) {
                throw new Exception(failure.reason());
            }
        }
        return false;
    }


    /**
     * 批量创建更新文档
     *
     * @param index
     * @param indexType
     * @param list      建索引时传入_id作为docuemntId
     * @return
     * @throws Exception
     */
    public boolean documentCreateBulk(String index, String indexType,
                                      List<Map<String, Object>> list) throws Exception {

        if (list.size() > 0) {

            BulkRequest bulkRequest = new BulkRequest();

            for (Map map : list) {
                IndexRequest indexRequest;

                Object idObj = map.get("_id");
                if (idObj != null && StringUtils.isNotBlank(idObj.toString())) {
                    map.remove("_id");
                    String documentId = idObj.toString();
                    indexRequest = new IndexRequest(index, indexType, documentId);
                } else {
                    indexRequest = new IndexRequest(index, indexType);
                }
                indexRequest.source(map);
                bulkRequest.add(indexRequest);

            }

            BulkResponse bulkResponse = client.bulk(bulkRequest,RequestOptions.DEFAULT);

            if (bulkResponse.hasFailures()) {
                System.out.println("索引异常信息：" + bulkResponse.buildFailureMessage());
                return false;
            }
        }

        return true;

    }

    /**
     * 创建更新文档
     *
     * @param index
     * @param indexType
     * @param documentId
     * @param routing
     * @param map
     * @return
     * @throws Exception
     */
    public boolean documentCreate(String index, String indexType,
                                  String documentId, String routing, Map map) throws Exception {
        IndexRequest request = new IndexRequest(index, indexType, documentId);

        request.routing(routing);
        request.source(map);
        IndexResponse indexResponse = client.index(request,RequestOptions.DEFAULT);

        if (indexResponse.getResult() == DocWriteResponse.Result.CREATED
                || indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
            return true;
        }
        ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo();
        if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
            return true;
        }
        if (shardInfo.getFailed() > 0) {
            for (ReplicationResponse.ShardInfo.Failure failure : shardInfo
                    .getFailures()) {
                throw new Exception(failure.reason());
            }
        }
        return false;
    }

    /**
     * 创建索引
     *
     * @param index
     * @param indexType
     * @param josonStr
     * @return
     * @throws Exception
     */
    public String documentCreate(String index, String indexType, String josonStr)
            throws Exception {
        IndexRequest request = new IndexRequest(index, indexType);

        request.source(josonStr, XContentType.JSON);
        IndexResponse indexResponse = client.index(request,RequestOptions.DEFAULT);

        String id = indexResponse.getId();
        if (indexResponse.getResult() == DocWriteResponse.Result.CREATED
                || indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
            return id;
        }
        ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo();
        if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
            return id;
        }
        if (shardInfo.getFailed() > 0) {
            for (ReplicationResponse.ShardInfo.Failure failure : shardInfo
                    .getFailures()) {
                throw new Exception(failure.reason());
            }
        }
        return null;
    }

    /**
     * 创建索引
     *
     * @param index
     * @param indexType
     * @param map
     * @return
     * @throws Exception
     */
    public String documentCreate(String index, String indexType,
                                 Map map) throws Exception {
        IndexRequest request = new IndexRequest(index, indexType);

        request.source(map);
        IndexResponse indexResponse = client.index(request,RequestOptions.DEFAULT);

        String id = indexResponse.getId();
        if (indexResponse.getResult() == DocWriteResponse.Result.CREATED
                || indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
            return id;
        }
        ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo();
        if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
            return id;
        }
        if (shardInfo.getFailed() > 0) {
            for (ReplicationResponse.ShardInfo.Failure failure : shardInfo
                    .getFailures()) {
                throw new Exception(failure.reason());
            }
        }
        return null;
    }

    public boolean documentDelete(String index, String indexType,
                                  String documentId) throws Exception {
        DeleteRequest request = new DeleteRequest(index, indexType, documentId);
        DeleteResponse deleteResponse = client.delete(request,RequestOptions.DEFAULT);
        if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) {
            return true;
        }
        ReplicationResponse.ShardInfo shardInfo = deleteResponse.getShardInfo();
        if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
            return true;
        }
        if (shardInfo.getFailed() > 0) {
            for (ReplicationResponse.ShardInfo.Failure failure : shardInfo
                    .getFailures()) {
                throw new Exception(failure.reason());
            }
        }
        return false;
    }

    /**
     * 为指定index指定别名
     * @param indexName index名字
     * @param aliasesName 别名名字
     * @return
     */
    public boolean aliases(String indexName,String aliasesName){
            IndicesAliasesRequest request=new IndicesAliasesRequest();
            IndicesAliasesRequest.AliasActions aliasAction =
                    new IndicesAliasesRequest.AliasActions(IndicesAliasesRequest.AliasActions.Type.ADD)
                            .index(indexName)
                            .alias(aliasesName);
            request.addAliasAction(aliasAction);
            try {
                AcknowledgedResponse indicesAliasesResponse =
                        getClient().indices().updateAliases(request, RequestOptions.DEFAULT);
                return indicesAliasesResponse.isAcknowledged();
            } catch (IOException e) {
                log.info("指定别名失败");
                return  false;
            }
    }

    /**
     * 指定别名 并删除oldIndex的别名
     * @param oldindex
     * @param newIndex
     * @param aliasesName
     * @return
     */
    public boolean aliases(String oldindex, String newIndex,String aliasesName) {
        IndicesAliasesRequest request=new IndicesAliasesRequest();
        //旧的删除
        IndicesAliasesRequest.AliasActions aliasAction =
                new IndicesAliasesRequest.AliasActions(IndicesAliasesRequest.AliasActions.Type.REMOVE)
                        .index(oldindex)
                        .alias(aliasesName);
        request.addAliasAction(aliasAction);
        //新的绑定
        aliasAction =
                new IndicesAliasesRequest.AliasActions(IndicesAliasesRequest.AliasActions.Type.ADD)
                        .index(newIndex)
                        .alias(aliasesName);
        request.addAliasAction(aliasAction);
        try {
            AcknowledgedResponse indicesAliasesResponse =
                    getClient().indices().updateAliases(request, RequestOptions.DEFAULT);
            return indicesAliasesResponse.isAcknowledged();
        } catch (IOException e) {
            return  false;
        }
    }
    /**
     * 根据id获得文档信息
     * @param index indexName
     * @param type typeName
     * @param id id
     * @return
     * @throws IOException
     */
    public GetResponse getDoucmentById(String index, String type, String id) throws IOException {
        GetRequest getRequest = new GetRequest(
                index,//索引
                type,//类型
                id);//文档ID
        GetResponse getResponse = client.get(getRequest,RequestOptions.DEFAULT);
        return getResponse;
    }
    /**
     * 简单的单表根据条件进行查询
     *
     * @param index      index
     * @param type       type
     * @param parameters 参数
     * @return
     */
    public List<String> queryByMatch(String index, String type, Map<String, String> parameters) {
        List<String> jsons=new ArrayList<String>();
        try {
            SearchRequest searchRequest=new SearchRequest();
            BoolQueryBuilder booleanQueryBuilder= QueryBuilders.boolQuery();
            SearchSourceBuilder searchSourceBuilder=new SearchSourceBuilder();
            for (String key :
                        parameters.keySet()) {
                    booleanQueryBuilder.must().add(QueryBuilders.termQuery(key,parameters.get(key)));
            }
            searchSourceBuilder.query(booleanQueryBuilder);
            searchRequest.source(searchSourceBuilder);

            SearchResponse response = client.search(searchRequest,RequestOptions.DEFAULT);
            if(response.getHits().getHits()!=null){
                for (SearchHit searchHit:
                response.getHits().getHits() ) {
                    jsons.add(searchHit.getSourceAsString());
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        return jsons;
    }
    /**
     * 只包含字母
     *
     * @return 验证成功返回true，验证失败返回false
     */
    public static boolean checkLetter(String cardNum) {
        String regex = "^[A-Za-z]+$";
        return Pattern.matches(regex, cardNum);
    }

    /**
     * 验证中文
     *
     * @param chinese 中文字符
     * @return 验证成功返回true，验证失败返回false
     */
    public static boolean checkChinese(String chinese) {
        String regex = "^[\u4E00-\u9FA5]+$";
        return Pattern.matches(regex, chinese);
    }


    /**
     * Description:提示词，支持中文、拼音、首字母等
     * <p>
     * 1、检测搜索词是中文还是拼音
     * 2、若是中文，直接按照name字段提示
     * 3、若是拼音（拼音+汉字），先按照name.keyword_pinyin获取，若是无结果按照首字母name.keyword_first_py获取
     * @param index
     * @param type
     * @param field 提示字段名字
     * @param text 文本
     * @return
     * @author liqiang
     */
    public Set<String> getSuggestWord(String index, String type, String field, String text) throws IOException {
        String postField=field;
        if (checkLetter(text)) {
            postField = field + ".keyword_pinyin";
        } else if (checkChinese(text)) {
            postField = field;
        } else {
            postField = field + ".keyword_pinyin";
        }
        Set<String> suggestTexts= postSuggestWord(index,type,postField,text);
        if(org.springframework.util.CollectionUtils.isEmpty(suggestTexts)){
            return postSuggestWord(index,type,field+".keyword_first_py",text);
        }
        return suggestTexts;
    }
    /**
     * Description:提示词，支持中文、拼音、首字母等
     * @param index
     * @param type
     * @param field 提示字段名字
     * @param text 文本
     * @return
     * @author liqiang
     */
    private Set<String> postSuggestWord(String index, String type, String field, String text) throws IOException {

        // 1、创建search请求
        SearchRequest searchRequest = new SearchRequest(index);
        searchRequest.types(type);
        // 2、用SearchSourceBuilder来构造查询请求体 ,请仔细查看它的方法，构造各种查询的方法都在这。
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        sourceBuilder.fetchSource(false);//不返回_source数据
        sourceBuilder.size(0);//忽略hits
        //做查询建议
        //词项建议
        SuggestionBuilder termSuggestionBuilder =
                SuggestBuilders.completionSuggestion(field).text(text);
        SuggestBuilder suggestBuilder = new SuggestBuilder();
        suggestBuilder.addSuggestion("suggest_productName", termSuggestionBuilder);
        sourceBuilder.suggest(suggestBuilder);
        searchRequest.source(sourceBuilder);
        Set<String> suggestTextList = new HashSet<String>();
        //3、发送请求
        SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
        //4、处理响应
        //搜索结果状态信息
        if (RestStatus.OK.equals(searchResponse.status())) {
            // 获取建议结果
            Suggest suggest = searchResponse.getSuggest();
            CompletionSuggestion termSuggestion = suggest.getSuggestion("suggest_productName");
            for (CompletionSuggestion.Entry entry : termSuggestion.getEntries()) {
                for (CompletionSuggestion.Entry.Option option : entry) {
                    suggestTextList.add(option.getText().string());
                }
            }
        }
        return suggestTextList;
    }
    /**
     * @Project demo
     * @PackageName com.crb.ocms.product.service.impl
     * @ClassName SimpleRestHighLevelClient
     * @Author qiang.li
     * @Date 2019/4/2 10:45 AM
     * @Description es版本降级导致的高级api会生成部分属性 低版本不能识别  将解析结果的方法暴露在外面
     */
    public class SimpleRestHighLevelClient extends RestHighLevelClient {
        public SimpleRestHighLevelClient(RestClientBuilder restClientBuilder) {
            super(restClientBuilder);
        }

        /**
         * 父类解析的解析响应内容是受保护的 所以定义一个继承
         * 解决版本不一致的 自定义请求 解析
         * @param response
         * @param entityParser
         * @param <Req>
         * @param <Resp>
         * @return
         * @throws IOException
         */
        public <Req extends Validatable, Resp> Resp parseEntity(Response response,CheckedFunction<XContentParser, Resp, IOException> entityParser)throws IOException {

            return this.parseEntity(response.getEntity(),entityParser);
        }

        /**
         * 解析获得idnex的响应
         * @param response
         * @return
         * @throws IOException
         */
        public GetIndexResponse parseGetIndexRespons(Response response) throws IOException {
            return  parseEntity(response,GetIndexResponse::fromXContent);
        }

        /**
         * 解析查询响应
         * @param response
         * @return
         * @throws IOException
         */
        public SearchResponse parseSearchResponse(Response response) throws IOException {
            return parseEntity(response,SearchResponse::fromXContent);
        }

        /**
         * 解析索引迁移响应
         * @param response
         * @return
         * @throws IOException
         */
        public BulkByScrollResponse parseBulkByScrollResponse(Response response) throws IOException {
            return  parseEntity(response,BulkByScrollResponse::fromXContent);
        }



    }

}

封装的处理器

工作原理
将指定条件的数据迁移到新索引,然后再新索引上面进行导入（多线程并行导入）,导入完毕删除老索引和别名 然后为新索引绑定别名 实现不停机更新

Elasticsearch和MySQL数据同步（logstash-input-jdbc）全量增量方式同步近千万数据
Lifelong learning
04-10 7452
同步方案： 同步读写：最为简单的方式在将数据写到mysql时，同时将数据写到ES，实现数据的双写。 异步双写(MQ方式)：MQ的性能基本比mysql高出一个数量级，所以性能可以得到显著的提高。 定时器程序：相关表中增加一个字段为timestamp的字段，任何crud操作都会导致该字段的时间发生变化，原来程序中的CURD操作不做任何变化；增加一个定时器程序，让该程序按一定的时间周期扫描指定的表，把该时间段内发生变化的数据提取出来；逐条写入到ES中。 binlog日志：读取mysql的binlog日志，获取指
[ElasticSearch2.x]Java API 之 索引管理
SmartSi
10-11 9726
ElasticSearch为了便于处理索引管理（Indices administration）请求，提供了 org.elasticsearch.client.IndicesAdminClient接口。通过如下代码从 Client 对象中获得这个接口的实现： IndicesAdminClient indicesAdminClient = client.admin().indices();
Elasticsearch加载大量数据
优快云_for_learn的博客
10-10 448
import os import sys from elasticsearch import Elasticsearch from elasticsearch import helpers def load_ques_data(es, index_v, fpw): '''es: Elasticsearch对象， index_v: 索引名 fpw: 文件写入对象 ''' query={"query" : {"match_all" : {}}} scanResp= he
java全量数据加载到内存中_Elasticsearch全量数据增量遍历实现原理
weixin_35035699的博客
02-25 473
0、需求针对凤凰网财经版块的新闻数据和评论数据， 1个索引存储采集凤凰网财经版块的新闻数据；1个索引存储相关的财经数据评论结果。统计：1)某条新闻的评论数的多少？2)某条评论属于哪条新闻？3)当前已采集数据的所有评论、评论数汇总，按照评论数逆序排序，以便于图形化展示。1、问题分解1.1 数据如何存储，方案选型？方案一：类似需求，1个索引ifeng_index存储新闻数据；1个索引ifeng_com...
elasticsearch的归并以及重索引
siwangxiaoxuesheng的博客
04-26 2479
ES版本 7.9.1 归并索引 kibana操作 #查询 GET /_cat/segments/kys_recommend?v #归并操作 POST kys_recommend/_forcemerge?max_num_segments=1 java代码 @Autowired public RestHighLevelClient restHighLevelClient; private void forMergeIndex(String indexName) { Forc
分布式搜索Elasticsearch——创建客户端（三）
村长Korbin
01-07 8315
4. org.elasticsearch.client.IndicesAdminClient接口 IndicesAdminClient提供了针对索引的action的operation的管理进行操作的接口，它有以下方法： 1) exists(IndicesExistsRequest request)、exists(IndicesExistsRequest request, ActionLis
ElasticSearch 索引相关操作
huangshanchun的专栏
04-09 1497
主要介绍基于ElasticSearch 5.0 客户端模式开发，索引相关操作代码实现~
最新版windows elasticsearch-7.13.0-windows-x86_64.zip
05-25
10. **备份与恢复**：定期备份Elasticsearch的数据非常重要，可以使用`Snapshot and Restore`功能进行全量或增量备份。恢复时，只需将备份还原到新的或已有的集群。 理解并掌握以上知识点，将有助于你在Windows环境...
【ElasticSearch】进阶电商检索实战必备：分词-同义词-联想词-增量同步-全量同步---持续更新)
ykx____h的博客
01-12 1619
【ElasticSearch】进阶电商检索实战必备：进阶电商检索实战必备：分词-同义词-联想词-增量同步-全量同步-持续更新】ElasticSearch电商检索实战必备---持续更新1、ES中的几个重要概念：集群、节点、文档、索引、分片及插件ik分词器2、商品检索模糊匹配、过滤、排序、分页、高亮及聚合分析3、es中商品数据的增删改查，canal增量同步4、nested 内嵌属性的保存，使用script进行增删改查操作5、搜索联想词suggestor的使用，汉语拼音插件的安装6、同义词分析检索，同义词插件的安
elasticsearch-jdbc-2.2.0.0-dist
04-18
5. **数据同步**：Elasticsearch-JDBC会持续监控数据库变化，实现增量或全量数据同步，具体取决于配置。 在实际应用中，Elasticsearch-JDBC常用于大数据分析场景，例如日志分析、实时报表和搜索引擎索引构建。由于...
Elasticsearch6.5.4全量包
01-10
本资源中包含Elasticsearc6.5.4 kibana-6.5.4 logstash-6.5.4 Jest-6.x, ik6.5.4 curl-7.63.0
Elasticsearch整合--springboot整合high-level-client及添加依赖时可能出现的问题
weixin_54982854的博客
05-05 672
Elasticsearch整合--springboot整合high-level-client及添加依赖时可能出现的问题
元数据平台解决方案
asfjgvajfghaklsbf的博客
08-17 489
一 遇到的问题 数据平台中元数据的管理一般有离线数据元数据 实时数据元数据，本次主要解决了离线的元数据。也就是hive的metastore信息。 但是metastore数据存储在mysql中，而且数据之间关联性比较强比如存储数据库的dbs，存储表的tbls，存储字段信息的colums_v2等等。想要获取到一个字段需要关联大量的表。 造成了查询速度过慢，而且对于hive的元数据库也会有比较大的压力。...
ElasticSearch7.X版本实现中文、英文、数字搜索
有糖哥
05-02 2280
EleasticSearch7.X版本实现中文、英文、数字搜索引入依赖客户端连接一些通用的api一些常量和配置类初始化索引库等ES配置信息最后别忘了封装查询的Bean信息 由于基金业务需要接入基金搜索功能，但是网上各种7.X版本的帖子又很少，这里就以自己线上代码，总结了下es相关客户端java代码实现搜索功能。 引入依赖 <dependency> <groupId>org.elasticsearch.client</groupId> <artifactId
Java 操作 ElasticSearch 的更多查询
liu1shi的博客
02-27 519
@Test public void t15() throws IOException { DeleteByQueryRequest request = new DeleteByQueryRequest(index); request.types(type); // 删除查询到到数据 request.setQuery(QueryBuilders.rangeQuery("fee").lt(5)); BulkByS...
[Elasticsearch] java.lang.ClassNotFoundException org.elasticsearch.common.CheckedConsumer异常
u012433915的专栏
04-06 1898
java.lang.ClassNotFoundException: org.elasticsearch.common.CheckedConsumer异常 解决方案：添加相关的依赖 <dependency> <groupId>org.elasticsearch</groupId> <artifactId>elasticsearch</artifactId> &...
ElasticSearch 7.5.1+ 指定分词器查询一次完整搜索debug过程 (2020-10)
qfzhangwei的专栏
10-09 1724
// 调试时学习源码一个途径（you agree？） org.elasticsearch.client.node.NodeClient#executeLocally(org.elasticsearch.action.ActionType, Request, org.elasticsearch.action.ActionListener) org.elasticsearch.client.node.NodeClient#transportAction --- value transportAction
java.lang.ClassNotFoundException: org.elasticsearch.common.transport.InetSocketTransportAddress
热门推荐
大痴小乙的博客
08-09 1万+
在做Elasticsearch 版本升级的时候，报出了 java.lang.ClassNotFoundException: org.elasticsearch.common.transport.InetSocketTransportAddress 这个错误，在此记录一下解决方法 开发环境 IntelliJ IDEA 2018.1.6 maven3.3.3 SpringBoot 2.0.4...
java.lang.NoClassDefFoundError: org/elasticsearch/common/xcontent/DeprecationHandler
悟已往之不谏，知来者之可追
10-21 7957
java.lang.NoClassDefFoundError: org/elasticsearch/common/xcontent/DeprecationHandler 报找不到类，这个类应该 7.3.2里面的，我pom依赖也没问题 编译阶段没问题，到运行时候，有问题，所以到打好的lib包中查看： 发现 这个es还是老的包。 从pom中查看 有问题。 将es依赖重新放到项目的根级父类后： 并在子模块中，将pom依赖中的版本号干掉，让它自动继承 再次打包，查看lib 全部ok，重启服务，正常，问
elasticsearch 全量快照
最新发布
03-05
Elasticsearch全量快照是一种备份和恢复数据的机制，它可以将整个Elasticsearch集群的数据进行快速且高效的备份。全量快照包含了索引的所有数据和元数据，可以用于在灾难恢复、数据迁移或者测试环境的数据复制等场景...
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值