1.工具类
@Component
@Slf4j
public class ElasticsearchUtil {
@Autowired
private TransportClient transportClient;
private static TransportClient client;
/**
* @PostContruct是spring框架的注解 spring容器初始化的时候执行该方法
*/
@PostConstruct
public void init() {
setClient(transportClient);
}
private static void setClient(TransportClient transportClient) {
ElasticsearchUtil.client = transportClient;
}
/**
* 数据添加，正定ID
*
* @param source 要增加的数据
* @param index 索引，类似数据库
* @param type 类型，类似表
* @return
*/
public static String addData(Map<String, Object> source, String index, String type) {
IndexResponse response = client.prepareIndex(index, type).setSource(source).get();
return response.getId();
}
public static Map<String, Object> getEsData(String index, SearchSourceBuilder searchSourceBuilder, int from, int size, Boolean flg) {
SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index);
searchRequestBuilder.setSource(searchSourceBuilder);
searchRequestBuilder.setTypes(index);
searchRequestBuilder.setFrom(from);
searchRequestBuilder.setSize(size);
SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
long totalHits = searchResponse.getHits().totalHits;
SearchHit[] hits = searchResponse.getHits().getHits();
Map<String, Object> result = new LinkedHashMap<>();
List<Object> ls = new ArrayList<>();
if (hits != null) {
for (SearchHit hit : hits) {
Map<String, Object> source = hit.getSource();
if (flg) {
Map<String, Object> data = new LinkedHashMap<>();
source.forEach((k, v) -> {
if ("content".equals(k)) {
return;
}
data.put(k, v);
});
MarketVo mk = new MarketVo();
mk.setId(hit.getId());
mk.setSource(data);
ls.add(mk);
} else {
HashMap<String, Object> map = new HashMap<>();
map.put("id", hit.getId());
map.putAll(source);
ls.add(map);
}
}
}
result.put("total", totalHits);
result.put("data", ls);
return result;
}
public static SearchSourceBuilder builderMarketQuery(Map<String, String> parms) {
// 构建搜索 DSL语句
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
String keyword = parms.get("keyword");
String order = parms.get("order");
SortOrder sortorder;
if ("desc".equalsIgnoreCase(order)) {
sortorder = SortOrder.DESC;
} else if (("asc").equalsIgnoreCase(order)) {
sortorder = SortOrder.ASC;
} else {
sortorder = SortOrder.DESC;
}
if (StringUtils.isNotEmpty(keyword)) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
//分词匹配
// String[] fielsnames = {"content", "title", "subtitle", "author", "source", "type", "abstract", "label"};
// boolQueryBuilder.must(QueryBuilders.multiMatchQuery(keyword, fielsnames));
//短语匹配，间距1000
boolQueryBuilder.should(QueryBuilders.matchPhraseQuery("content",keyword).slop(1000).boost(3.0F))
.should(QueryBuilders.matchPhraseQuery("title",keyword).slop(1000).boost(3.0F))
.should(QueryBuilders.matchPhraseQuery("subtitle",keyword).slop(1000))
.should(QueryBuilders.matchPhraseQuery("author",keyword).slop(1000))
.should(QueryBuilders.matchPhraseQuery("source",keyword).slop(1000))
.should(QueryBuilders.matchPhraseQuery("type",keyword).slop(1000))
.should(QueryBuilders.matchPhraseQuery("abstract",keyword).slop(1000))
.should(QueryBuilders.matchPhraseQuery("label",keyword).slop(1000));
searchSourceBuilder.query(boolQueryBuilder).sort("update_date", sortorder).sort(SortBuilders.scoreSort().order(SortOrder.DESC));
} else {
searchSourceBuilder.query(QueryBuilders.matchAllQuery()).sort("update_date", sortorder).sort(SortBuilders.scoreSort().order(SortOrder.DESC));
}
return searchSourceBuilder;
}
/**
* 通过ID删除数据
*
* @param index 索引，类似数据库
* @param type 类型，类似表
* @param id 数据ID
*/
public static void deleteDataById(String index, String type, String id) {
DeleteResponse response = client.prepareDelete(index, type, id).execute().actionGet();
log.info("deleteDataById response status:{},id:{}", response.status().getStatus(), response.getId());
}
/**
* 通过ID 更新数据
*
* @param index 索引，类似数据库
* @param type 类型，类似表
* @return
*/
public static void updateDataById(MarketVo marketVo, String index, String type) {
String id = marketVo.getId();
if (id == null) {
log.error("ES 动态市场数据修改失败: _id为 nulll");
} else {
Map<String, Object> source = marketVo.getSource();
LocalDateTime dt = LocalDateTime.now();
DateTimeFormatter frt = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
String format = frt.format(dt);
source.put("update_date", format);
UpdateRequest updateRequest = new UpdateRequest();
updateRequest.index(index).type(type).id(id).doc(source);
client.update(updateRequest);
}
}
/**
* 通过ID 更新数据
*
* @param index 索引，类似数据库
* @param type 类型，类似表
* @return
*/
public static void updateDataById(Map<String, String> marketMap, String index, String type) {
String id = marketMap.get("id");
if (id == null) {
log.error("ES 动态市场数据修改失败: _id为 nulll");
} else {
Map<String, String> source = new HashMap<>();
for (Map.Entry<String, String> entries : marketMap.entrySet()) {
if (!entries.getKey().equals("id")) {
source.put(entries.getKey(), entries.getValue());
}
}
LocalDateTime dt = LocalDateTime.now();
DateTimeFormatter frt = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
String format = frt.format(dt);
source.put("update_date", format);
UpdateRequest updateRequest = new UpdateRequest();
updateRequest.index(index).type(type).id(id).doc(source);
client.update(updateRequest);
}
}
/**
* 通过ID获取数据
*
* @param index 索引，类似数据库
* @param type 类型，类似表
* @param id 数据ID
* @param fields 需要显示的字段，逗号分隔（缺省为全部字段）
* @return
*/
public static MarketVo searchDataById(String index, String type, String id, String fields) {
GetRequestBuilder getRequestBuilder = client.prepareGet(index, type, id);
if (StringUtils.isNotEmpty(fields)) {
getRequestBuilder.setFetchSource(fields.split(","), null);
}
GetResponse getResponse = getRequestBuilder.execute().actionGet();
MarketVo mk = new MarketVo();
if (getResponse != null) {
mk.setId(getResponse.getId());
mk.setSource(getResponse.getSource());
}
return mk;
}
}
2.聚合查询DSL构建，以及结果解析
/**
* 构建dsl语句,条件加聚合
* filter不打分,must存在 should只影响打分,不需要满足，具体去了解es相关语法
*/
private static SearchSourceBuilder getSearchSourceBuilder(String indicatrix, Map<String, Object> params) {
// 构建搜索对象
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
String startDate = params.get("startDate") == null ? "" : (String) params.get("startDate");
String endDate = params.get("endDate") == null ? "" : (String) params.get("endDate");
String countryCode = params.get("countryCode") == null ? "" : (String) params.get("countryCode");
String productCode = params.get("productCode") == null ? "" : (String) params.get("productCode");
String group = (String) params.get("group");
//聚合不需要查看具体数据，只需要聚合值
searchSourceBuilder.size(0);
// bool 构件查询
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.filter(QueryBuilders.termsQuery("productCode", productCode.split(",")));
boolQueryBuilder.filter(QueryBuilders.rangeQuery("orderDate").gte(startDate).lte(endDate));
//聚合 构建
TermsAggregationBuilder agg = AggregationBuilders.terms("customerCardCode").field("customerCardCode").size(10000);
//组装 es查询,查询条件以及聚合分组条件
searchSourceBuilder.query(boolQueryBuilder).aggregation(agg);
return searchSourceBuilder;
}
/**
* 根据查询结果的结构进行解析。了解返回值结构，很容易明白
* */
public static List<LinkedHashMap<String, Object>> getAggregation(QueryBuilder query, TermsAggregationBuilder aggregation, String index, String indicatrix, String group) {
List<LinkedHashMap<String, Object>> lst = new ArrayList<>();
// 构建搜索对象
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.size(0);
//组装 es查询
searchSourceBuilder.query(query).aggregation(aggregation);
log.info("Es "+group+" DSL: "+searchSourceBuilder.toString());
SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index);
searchRequestBuilder.setSource(searchSourceBuilder);
searchRequestBuilder.setTypes("ex_pickup_link_detail");
SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
List<Aggregation> aggregations = getAggregations(searchResponse.getAggregations());
aggregations.stream().forEach(x -> {
//获取一级桶
List<Terms.Bucket> xbuckets = getBuckets((StringTerms) x);
xbuckets.stream().forEach(y -> {
LinkedHashMap<String, Object> mp = new LinkedHashMap<>();
mp.put(key, y.getKey() == null ? "" : y.getKey());
mp.put(indicatrix, y.getDocCount());
List<Aggregation> aggregations1 = getAggregations(y.getAggregations());
aggregations1.stream().forEach(z->{
//获取二级桶
List<Terms.Bucket> zbuckets = getBuckets((StringTerms)z);
zbuckets.stream().forEach(h->{
if("Y".equalsIgnoreCase(h.getKey()!=null?(String) h.getKey():"")){
long handledQuantity = h.getDocCount();
long handlingQuantity = (long)mp.get(indicatrix);
mp.put("xxx1",handledQuantity);
mp.put("xxx2",handlingQuantity-handledQuantity);
}
});
});
if(mp.get("xxx1")==null){
mp.put("xxx2",0L);
mp.put("xxx3",y.getDocCount());
}
lst.add(mp);
});
});
return lst;
}
//获取桶层
private static List<Terms.Bucket> getBuckets(StringTerms x) {
if (x.getBuckets() == null) {
List<Terms.Bucket> ls = new ArrayList<>();
return ls;
}
return x.getBuckets();
}
//获取聚合字段层
private static List<Aggregation> getAggregations(Aggregations aggregations) {
return aggregations.asList();
}
- RestClient(方式)
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>7.6.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>7.6.1</version>
</dependency>
public static RestHighLevelClient restClient() {
///final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
// credentialsProvider.setCredentials(AuthScope.ANY,
// new UsernamePasswordCredentials(userName, password));
List<org.apache.http.HttpHost> httpHosts = new ArrayList<>();
httpHosts.add(new org.apache.http.HttpHost("10.202.116.35",9200));
org.apache.http.HttpHost[] httpHostArr = httpHosts.toArray(new HttpHost[httpHosts.size()]);
RestClientBuilder builder = RestClient.builder(httpHostArr);
//.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
RestHighLevelClient client = new RestHighLevelClient(builder);
return client;
}
//
public Map<String, Object> getAllConfigs1() {
Map<String, Object> allConfigs = new HashMap<String, Object>();
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.should(QueryBuilders.termQuery("xxxx","vvvvv"));
// 存活时间，当索引数据量特别大时，出现超时可能性大，此值适当调大
Scroll scroll = new Scroll(TimeValue.timeValueMinutes(10L));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(boolQuery);
searchSourceBuilder.size(ES_RES_SIZE);
SearchRequest searchRequest = new SearchRequest()
// ES7已经去掉type，查询时加type
.indices(EsClientHelper.AIR_FLIGHT_ZONE_INDEX_NAME)
.scroll(scroll)
.source(searchSourceBuilder);
SearchResponse searchResponse = null;
try {
searchResponse = restClient().search(searchRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits();
// for (SearchHit searchHit : searchHits) {
// System.out.println(searchHit.getSourceAsString());
// }
//遍历搜索命中的数据，直到没有数据
while (searchHits != null && searchHits.length > 0) {
if (searchHits != null && searchHits.length > 0) {
//数据处理 scroll全量查询
for (SearchHit hit : searchResponse.getHits().getHits()) {
JSONObject object = JSONObject.parseObject(hit.getSourceAsString());
AirFlightZoneConfig airFlightZoneConfig = new AirFlightZoneConfig();
airFlightZoneConfig.setId(object.getString("id"));
String sendZoneCode = object.getString("send_zone_code");
airFlightZoneConfig.setSendZoneCode(sendZoneCode);
airFlightZoneConfig.setLineCode(object.getString("line_code"));
airFlightZoneConfig.setCvyName(object.getString("cvy_name"));
airFlightZoneConfig.setConveyType(object.getString("trans_capacity_type"));
airFlightZoneConfig.setWorkDays(object.getString("work_days"));
airFlightZoneConfig.setVersiondt(object.getString("versiondt"));
allConfigs.put(sendZoneCode, airFlightZoneConfig);
}
}
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
try {
searchResponse = restClient().scroll(scrollRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
}
//clean scroll
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
ClearScrollResponse clearScrollResponse = null;
try {
clearScrollResponse = restClient().clearScroll(clearScrollRequest,RequestOptions.DEFAULT);
} catch (IOException e) {
}
return allConfigs;
}
扫一扫
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
