业务场景
1、RestClientBuilder初始化（同时支持单机与集群） 2、发送ES查询请求公共方法封装（支持sql、kql、代理访问、集群访问、鉴权支持） 3、判断ES索引是否存在（/_cat/indices/${indexName}） 4、判断ES索引别名是否存在 （/_cat/aliases/${indexName}） 5、判断ES索引指定字段/属性是否存在（这里字段支持多级，如：logObj.id） 6、判断ES索引指定字段/属性的类型（字段支持多级） 7、阻塞线程直至索引就绪（为了应对跨日时索引名短时间可能不存在的问题） 8、创建索引别名（可用于支持sql查询，索引名中有特殊字符不能用作表名，可通过创建别名来解决） 9、索引别名创建结果解析（ 判断acknowledged） 10、KQL查询ES（Kibana语法查询ElasticSearch） 11、SQL查询ES（标准SQL语法查询ElasticSearch） 12、Java在本地通过代理访问ES（可用于解决网络不能直接的问题） 13、Java 客户端访问ES集群（同时支持单机与集群） 14、Java ES客户端鉴权（安全需求）
软件环境
说明：当前例子中用的7，理论上8也通用
Kibana查询效果
KQL查询ES
SQL查询ES
下面讲java代码实现
Java类方法详解
1、RestClientBuilder初始化
同时支持单机与集群
/**
* RestClientBuilder 初始化
*
* @param host 同时支持单机与集群
* 单机：host和port各司其职
* 集群时：port参数无效，host中包含IP和PORT，多个实例用逗号分隔
*
* eg：
* 10.***.6.247
* 或： host:10.***.6.247:9200,10.***.6.34:9200,10.***.6.120:9200
* @param port
* @return
*/
private RestClientBuilder buildClient(String host, Integer port){
RestClientBuilder restClientBuilder = null;
if(host.indexOf(",")==-1)
{
// 单机 host:10.***.6.247, 只有单机会使用port参数
restClientBuilder = RestClient.builder(new HttpHost( host, port, "http" ) );
}
else
{
// 集群 host:10.***.6.247:9200,10.***.6.34:9200,10.***.6.120:9200,10.***.6.9:9200,10.***.6.183:9200
String[] hostArr = host.split("\\,");
HttpHost[] httpHosts = new HttpHost[hostArr.length];
for( int i=0; i<hostArr.length; i++ )
{
String[] addrs = hostArr[i].split("\\:");
HttpHost httpHost = new HttpHost( addrs[0], Integer.valueOf(addrs[1]), "http" );
httpHosts[i] = httpHost;
}
restClientBuilder = RestClient.builder( httpHosts );
}
return restClientBuilder;
}2、发送ES查询请求公共方法
- SQL支持
- KQL支持
- 支持代理访问支持
- 鉴权支持
/**
* 发送ES 查询请求
*
* @param host
* @param port
* @param username
* @param password
* @param method
* @param endpoint ES接口
* eg：
* 1、创建别名： "/_aliases"
* 2、判断索引是否存在： "/_cat/indices/myIndexName"
* 2、判断索引别名是否存在： "/_cat/aliases/indexName"
*
* @param jsonEntity 查询语句
* eg：
* 1、为索引创建别名（可用于支持sql查询，如果使用sql查询时原索引名中有特殊字符不能用作表名，可通过创建别名来解决）
* String kqlJson = "{\"actions\" : [{ \"add\" : { \"index\" : \""+idxName+"\",\"alias\" : \""+idxAliases+"\" } }]} ";
* String kqlJson= "";
* 2、String jsonEntity = "{\"query\": \"" + sqlQuery2 + "\", " +
* "\"params\": ["+cStart+","+cEnd+"]," +
* " \"fetch_size\": 65536 }";
*
* 实际业务场景举例：
* POST _sql?format=txt
* {
* "query": "SELECT tags.svc_code, sum(iif(tags.response_code.keyword='0000',1,0)) as success_count, count(metric) as total
* FROM order_service_****
* where create_time between '2025-02-27T11:00:00+0800' and '2025-02-27T13:59:00+0800'
* group by tags.svc_code having count(metric)>=50
* order by 3 desc",
* "fetch_size": 65536
* }
*
* @return
* @throws IOException
*/
public String request(String host, Integer port,
String username, String password,
String method, String endpoint, String jsonEntity) throws IOException {
RestClientBuilder restClientBuilder = buildClient( host, port );
if(!StringUtils.isEmpty(username)){
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
restClientBuilder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
//线程设置
httpClientBuilder.setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(2).build());
/* ******
* 能直连ES的不需要；如果本地不能直连ES的则加上，IP根据实际调整
*
* */
if("dev".equals(profile) && host.indexOf("10.***.137")==0 ) {
httpClientBuilder.setProxy(
new HttpHost("10.***.248.54", 8443, "http") //设置代理服务
);
}else if("dev".equals(profile) && host.indexOf("10.***.6")==0 ){
httpClientBuilder.setProxy(
new HttpHost( "192.***.66.30", 8443,"http") //设置代理服务
);
}
return httpClientBuilder;
}
});
}
RestClient restClient = restClientBuilder.build();
Request request = new Request(method, endpoint );
request.setJsonEntity( jsonEntity );
Response response = restClient.performRequest(request);
HttpEntity entity=response.getEntity();
restClient.close();
entity = new BufferedHttpEntity(entity);
return EntityUtils.toString(entity);
}3、判断ES索引是否存在
/**
* 判断索引名是否存在
*
* @param host
* @param port
* @param username
* @param password
* @param indexName 索引名
* @return
* @throws IOException
*/
public boolean isExistsIndex( String host, Integer port,
String username, String password,
String indexName ) throws IOException {
RestClientBuilder restClientBuilder = buildClient( host, port );
if(!StringUtils.isEmpty(username)){
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
restClientBuilder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
//线程设置
httpClientBuilder.setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(2).build());
return httpClientBuilder;
}
});
}
RestClient restClient = restClientBuilder.build();
Request request = new Request("GET", "/_cat/indices/"+indexName );
try{
Response response = restClient.performRequest(request);
HttpEntity entity=response.getEntity();
entity = new BufferedHttpEntity(entity);
if (!StringUtils.hasLength(EntityUtils.toString(entity))) {
System.out.println("Index exists.");
return true;
} else {
System.out.println("Index does not exist.");
return false;
}
}catch (Exception e){
//如果不存在会报404的错误，返回false创建别名
return false;
}finally {
restClient.close();
}
}4、判断ES索引别名是否存在
/**
* 判断索引别名是否存在
*
* @param host
* @param port
* @param username
* @param password
* @param indexName
* @return
* @throws IOException
*/
public boolean isExistsAliases( String host, Integer port,
String username, String password,
String indexName ) throws IOException {
RestClientBuilder restClientBuilder = buildClient( host, port );
if(!StringUtils.isEmpty(username)){
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
restClientBuilder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
//线程设置
httpClientBuilder.setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(2).build());
return httpClientBuilder;
}
});
}
RestClient restClient = restClientBuilder.build();
Request request = new Request("GET", "/_cat/aliases/"+indexName );
Response response = restClient.performRequest(request);
HttpEntity entity=response.getEntity();
restClient.close();
entity = new BufferedHttpEntity(entity);
if (!StringUtils.isEmpty(EntityUtils.toString(entity))) {
System.out.println("Index alias exists.");
return true;
} else {
System.out.println("Index alias does not exist.");
return false;
}
}5、获取ES索引指定字段/属性是否存在
这里字段支持多级，如：logObj.id
/**
* 判断索引 某个字段/属性是否存在
* 说明： 这里字段支持多级，如：logObj.id
*
* @param host
* @param port
* @param username
* @param password
* @param indexName
* @param property eg：id、 logObj.id
* @return
* @throws IOException
*/
public boolean isExistsProperty( String host, Integer port,
String username, String password,
String indexName, String property ) throws IOException {
RestClientBuilder restClientBuilder = buildClient( host, port );
if(!StringUtils.isEmpty(username)){
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
restClientBuilder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
//线程设置
httpClientBuilder.setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(2).build());
return httpClientBuilder;
}
});
}
RestClient restClient = restClientBuilder.build();
Request request = new Request("GET", "/"+indexName+"/_mapping" );
Response response = restClient.performRequest(request);
HttpEntity entity=response.getEntity();
restClient.close();
entity = new BufferedHttpEntity(entity);
JSONObject obj = JSONObject.parseObject(EntityUtils.toString(entity));
JSONObject properties = obj.getJSONObject( obj.keySet().iterator().next() )
.getJSONObject("mappings" )
.getJSONObject("properties" );
String[] arr = property.split("\\.");
for( int i=0; i<arr.length; i++ ){
if(i==arr.length-1){
}else{
if(properties.containsKey( arr[i] )){
properties = properties.getJSONObject(arr[i]).getJSONObject("properties" );
}else{
return false;
}
}
}
boolean bool = properties.containsKey( arr[arr.length-1] );
log.info("property:{} , isExist:{}", property, bool );
return bool;
}6、获取ES索引指定字段/属性的类型
同一个索引的同一字段，不同时间的数据类型可能不一样，从而影响sql语句的写法（sql语法不一样），所以个别场景要做判断
/**
* 判断索引某个字段/属性的类型
* 说明： 这里字段支持多级，如：logObj.id
*
* @param host
* @param port
* @param username
* @param password
* @param indexName
* @param property eg：id、 logObj.id
* @return
* @throws IOException
*/
public String getIndexPropertyType( String host, Integer port,
String username, String password,
String indexName, String property ) throws IOException {
RestClientBuilder restClientBuilder = buildClient( host, port );
if(!StringUtils.isEmpty(username)){
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
restClientBuilder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@O
最低0.47元/天 解锁文章
扫一扫
3492
到【灌水乐园】发言
