Elasticsearch API操作01

基本操作

集群健康度检查

GET http://172.16.18.114:9200/_cluster/health

判断索引是否存在

HEAD http://172.16.18.114:9200/logstash-2016.01.07

创建索引

PUT http://172.16.18.116:9200/test
{
    "settings" : {
        "index" : {
            "number_of_shards" : 3,
            "number_of_replicas" : 2
        }
    }
}

查看索引映射信息

GET http://172.16.18.116:9200/cdr-2015-08-30/_mappings

修改索引映射

PUT http://172.16.18.116:9200/logstash-2015.11.17/heartbeat/_mappings
{
        "_ttl": {
            "enabled": true,
            "default": "3d"
        },
        "properties": {
            "message": {
                "index": "no",
                "type": "string"
            },
            "@timestamp": {
                "format": "dateOptionalTime",
                "type": "date"
            },
            "host": {
                "index": "no",
                "type": "string"
            }
        },
        "_all": {
            "enabled": false
        }
}

查看索引属性

GET http://172.16.18.114:9200/logstash-2015.12.30/_settings

设置索引属性

PUT http://172.16.18.116:9200/.marvel-kibana/_settings
{
        "index" : {
            "number_of_replicas" : 0
        }
}

查看索引信息

GET http://172.16.18.116:9200/cdr-2015-08-30/_stats

关闭索引(使用时再_open)

POST http://172.16.18.114:9200/logstash-2015.11*/_close

列出所有节点简要状态信息

GET http://172.16.18.116:9200/_nodes/stats/indices/search

查看节点配置情况

GET http://172.16.18.116:9200/_nodes/elasticsearch_114/settings

列出节点存储信息

GET http://172.16.18.114:9200/_stats/store

查看节点插件

GET http://172.16.18.114:9200/_nodes/elasticsearch_114/plugins?pretty=true

列出节点详细信息

GET http://172.16.18.116:9200/_stats

查询特定字段

POST http://172.16.18.116:9200/my_index/_search?fields=_all,_source,full_name
{
      "query": {
        "match": {
          "_all": "John Smith"
        }
      },
      "highlight": {
        "fields": {
          "_all": {},
          "full_name": {}
        }
      }
}

查询索引下数据

GET http://172.16.18.116:9200/kibana-int/_search

查看某一索引/类型下文档总数

GET http://172.16.18.116:9200/cdr-2015-10-21/u2_gcdr/_count

单词搜索

POST http://172.16.18.116:9200/jfyindex/_search
{
      "query" : {
        "match" : {
          "_all": "rock climbing"
        }
      }
}

简单搜索

GET http://172.16.18.116:9200/jfy*/_search

短语搜索

GET http://172.16.18.116:9200/jfyindex/_search?q="rock climbing"

短语搜索

POST http://172.16.18.116:9200/jfyindex/_search?q="rock climbing"
{
      "query" : {
        "match_phrase" : {
          "_all": "rock climbing"
        }
      }
}

搜索清单号码（全词匹配）

POST http://172.16.18.116:9200/cdr*/_search
{
      "query" : {
        "match" : {
          "msisdn": "85267944762"
        }
      }
}

分析查询文本

POST http://172.16.18.116:9200/logstash-2015.08.01/_analyze
O_Str[9]=20150630-111344

Explan Query

POST http://172.16.18.116:9200//_validate/query?explain 
    {
      "query": {
        "query_string": {
          "query": "abc"
        }
      }
    }

定义模板

PUT http://172.16.18.116:9200/_template/cdr
{
        "order": 0,
        "template": "cdr*",
        "settings": {
            "index.refresh_interval": "30m",
            "index.number_of_replicas": "1",
            "index.number_of_shards": "1",
            "index.translog.flush_threshold_ops": "100000"
        },
        "mappings": {
            "_default_": {
                "dynamic_templates": [
                    {
                        "string_fields": {
                            "mapping": {
                                "index": "not_analyzed",
                                "omit_norms": true,
                                "type": "string"
                            },
                            "match": "*",
                            "match_mapping_type": "string"
                        }
                    }
                ],
                "_all": {
                    "enabled": true
                },
                "date_detection": false
            },
            "hjscdr": {
                "properties": {
                    "starttime": {
                        "index": "not_analyzed",
                        "type": "string"
                    },
                    "msisdn": {
                        "index": "not_analyzed",
                        "type": "string"
                    },
                    "rectype": {
                        "index": "no",
                        "type": "string"
                    },
                    "pdp_address": {
                        "index": "not_analyzed",
                        "type": "string"
                    },
                    "roaming_charge": {
                        "type": "long"
                    },
                    "unit": {
                        "type": "long"
                    },
                    "totallink": {
                        "type": "long"
                    },
                    "@timestamp": {
                        "type": "long"
                    },
                    "startdate": {
                        "index": "not_analyzed",
                        "type": "string"
                    },
                    "charging_item": {
                        "index": "not_analyzed",
                        "type": "string"
                    },
                    "sgsn": {
                        "index": "not_analyzed",
                        "type": "string"
                    },
                    "imsi": {
                        "index": "not_analyzed",
                        "type": "string"
                    },
                    "apn": {
                        "index": "not_analyzed",
                        "type": "string"
                    },
                    "uplink": {
                        "type": "long"
                    },
                    "ggsn": {
                        "index": "not_analyzed",
                        "type": "string"
                    },
                    "downlink": {
                        "type": "long"
                    }
                }
            }
        }
}

logstash模板示例

PUT http://172.16.18.116:9200/_template/logstash
{
        "order": 99,
        "template": "logstash-*",
        "settings": {        
          "index.refresh_interval": "5s",
          "index.number_of_replicas": "1",
          "index.number_of_shards": "1",
          "index.translog.flush_threshold_ops": "5000"          
        },
        "mappings": {
            "_default_": {
              "_all": {
                "enabled": true,
                "omit_norms": true
              },
              "dynamic_templates": [
                {
                  "message_field": {
                    "match": "message",
                    "match_mapping_type": "string",
                    "mapping": {
                      "type": "string",
                      "index": "analyzed",
                      "omit_norms": true
                    }
                  }
                },
                {
                  "string_fields": {
                    "match": "*",
                    "match_mapping_type": "string",
                    "mapping": {
                      "type": "string",
                      "index": "analyzed",
                      "omit_norms": true,
                      "fields": {
                        "raw": {
                          "type": "string",
                          "index": "not_analyzed",
                          "ignore_above": 256
                        }
                      }
                    }
                  }
                }
              ],
              "properties": {
                "@version": {
                  "type": "string",
                  "index": "not_analyzed"
                },
                "geoip": {
                  "type": "object",
                  "dynamic": true,
                  "properties": {
                    "location": {
                      "type": "geo_point"
                    }
                  }
                }
              }
            }
          }
      }
}      

进阶操作

刷新数据到磁盘

POST http://172.16.18.116:9200/_flush

设置索引刷新频率

PUT http://172.16.18.116:9200/cdr*/_settings
    {
      "index.translog.flush_threshold_ops": 5000,
      "index.refresh_interval": "5s"
    }

查看node segment的memory

GET http://172.16.18.179:9200/_cat/nodes?v=&h=name,port,sm

查看fielddata占用内存情况(查询时es会把fielddata信息load进内存)

GET http://172.16.18.114:9200/_stats/fielddata

aggs max(求某个索引类型一段时间内某一字段最大值)

POST http://172.16.18.116:9200/logstash-2015.11.17/heartbeat/_search
    {  
      "query": {
        "range": {
          "@timestamp": {
            "from": 1547330883236,
            "to": 1547330883236
          }
        }
      },
      "size": 0,
      "aggs": {
        "max_time": {
          "max": {
            "field": "@timestamp"
          }
        }
      }
    }

监控FieldData

GET http://172.16.18.114:9200//_nodes/stats/indices/fielddata?fields=*

cat fielddata

https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-fielddata.html

GET http://172.16.18.179:9200/_cat/fielddata?v

多列聚合操作

实现如mysql的group by功能：
select yearmon(createtime) yearmon,method,status,count(*) from test group by yearmon,method,status

POST http://172.16.18.116:9200/test/_search
    {
        "size": 0,
        "aggs" : {
            "day_total" : {
                "date_histogram":{
                    "field": "CreateTime",
                    "interval": "day",
                    "format": "yyyyMMdd"
                },
                "aggs": {
                    "method_total":{
                        "terms": {"field": "method"},
                        "aggs":{                    
                            "status_total":{
                                "terms": {"field": "status"}
                            }
                        }                      
                    }      
                }
            }
        }
    }

设置文档_ttl值

PUT http://172.16.18.116:9200/_template/template_logstash
    {
      "template" : "logstash*",
      "settings" : {
          "number_of_shards" : 1,
          "number_of_replicas": 1      
      },
      "mappings" : {
        "hostapd.log":{
          "_ttl" : { "enabled" : true, "default" : "30d" }
        },
        "hostapd1.log":{
          "_ttl" : { "enabled" : true, "default" : "30d" }
        }    
      }
    }

Indices shard stores(2.0以上)

GET http://172.16.18.114:9200//logstash-2016.01.07/_shard_stores

节点jvm信息

GET http://172.16.18.114:9200/_nodes/*/stats/jvm

暂停集群的shard自动均衡

PUT http://172.16.18.116:9200/_cluster/settings
    {
        "transient" : {
            "cluster.routing.allocation.enable" : "none"
        }
    }

恢复集群的shard自动均衡

PUT http://172.16.18.179:9200/_cluster/settings
    {
        "transient" : {
            "cluster.routing.allocation.enable" : "all"
        }
    }

重启节点

POST http://172.16.18.116:9200/_cluster/nodes/elasticsearch_114/_shutdown

查看索引Segments信息

GET http://172.16.18.116:9200//logstash-2016.01.07/_segments

scan-and-scroll

POST http://172.16.18.116:9200/_search/scroll?scroll=1m
c2NhbjsxOzU3NzIzOmZpVXBnR0VDUl9lRFF3ejBJUjBkOVE7MTt0b3RhbF9oaXRzOjE5OTc2ODs=

Limiting Memory Usage(fielddata-size)

https://www.elastic.co/guide/en/elasticsearch/guide/current/_limiting_memory_usage.html#fielddata-size

POST http://172.16.18.116:9200//_cluster/settings
    {
        "persistent" : {
            "indices.fielddata.break.limit" : "80%",
            "indices.fielddata.cache.size" : "60%"
        }
    }

清理cache

POST http://172.16.18.114:9200//_cache/clear

优化索引

POST http://172.16.18.114:9200/_optimize?max_num_segments=1

nodes/hot_threads

GET http://172.16.18.114:9200/_nodes/hot_threads

刷新synced(1.7以上才支持)

    PUT http://172.16.18.116:9200/_flush/synced
    {
        "transient" : {
            "cluster.routing.allocation.enable" : "none"
        }
    }

段合并节流控制

PUT http://172.16.18.116:9200//_cluster/settings
    {
        "persistent" : {
            "indices.fielddata.break.limit" : "80%",
            "indices.fielddata.cache.size" : "60%"
        }
    }

查看segment的memory

GET http://172.16.18.179:9200/_cat/segments?v

深翻页(scan-and-scroll)

    POST http://172.16.18.116:9200/test_index/_search?search_type=scan&scroll=1m
    {
      "query": { 
        "match_all": {}
      }, 
      "size": 1000 
    }

集群性能优化

PUT http://172.16.18.116:9200//_cluster/settings
    {
        "persistent" : {
            "indices.store.throttle.type" : "none",
            "indices.store.throttle.max_bytes_per_sec" : "100mb",
            "indices.fielddata.break.limit" : "80%",
            "indices.fielddata.cache.size" : "60%"        
        }
    }

searchguard-auth

PUT http://172.16.18.114:9201/searchguard/ac/ac
    {"acl": [
        {
            "__Comment__": "Default is to execute all filters",
            "filters_bypass": [],
            "filters_execute": ["actionrequestfilter.deny"]
        },
        {
            "__Comment__": "This means that every requestor (regardless of the requestors hostname and username) which has the root role can do anything",
            "roles": ["root"],
            "filters_bypass": ["*"],
            "filters_execute": []
        },
        {
            "__Comment__": "172.16.18.171 can do anything",
            "hosts": ["172.16.18.171"],
            "filters_bypass": ["*"],
            "filters_execute": []
        },
        {
            "__Comment__": "172.16.18.114 readonly",
            "hosts": ["172.16.18.114"],
            "filters_bypass": ["actionrequestfilter.deny"],
            "filters_execute": ["actionrequestfilter.readonly"]
        },
        {
            "__Comment__": "172.16.18.114, index: *kibana*, can do anything",
            "hosts": ["172.16.18.114"],
            "indices": ["*kibana*"],
            "filters_bypass": ["*"],
            "filters_execute": []
        },
        {
            "__Comment__": "This means that for the user spock on index popstuff only the actionrequestfilter.readonly will be executed, no other",
            "users": ["user"],
            "indices": ["cdr*"],
            "filters_bypass": ["actionrequestfilter.deny"],
            "filters_execute": ["actionrequestfilter.readonly"]
        }
    ]}