从es中批量导出数据

import json
import time
import requests


class exportEsData():
    size = 10000
    def __init__(self, url,index,type):
        self.url = url+"/"+index+"/"+type+"/_search"
        self.index = index
        self.type = type
    def exportData(self):
        print("export data begin...")
        begin = time.time()
        msg = requests.get(self.url).text
        print(msg)
        obj = json.loads(msg)
        num = obj["hits"]["total"]
        start = 0
        end =  num/self.size+1
        while(start<end):
            msg = requests.get(self.url+"?from="+str(start*self.size)+"&size="+str(self.size)).text
            self.writeFile(msg)
            start=start+1
        print("export data end!!!\n\t total consuming time:"+str(time.time()-begin)+"s")
    def writeFile(self,msg):
        obj = json.loads(msg)
        print(obj)
        vals = obj["hits"]["hits"]
        try:
            f = open(self.index+"_"+self.type+".json","a")
            for val in vals:
                a = json.dumps(val["_source"],ensure_ascii=False)
                f.write(a+"\n")
        finally:
            f.flush()
            f.close()

if __name__ == '__main__':
    exportEsData(host,index,type).exportData()

需要注意的是当es数据量大于1w时会报错，解决方法详见上一篇博文