es 定时备份 删除 与数据恢复

1.创建快照仓库

$ curl -u elastic:123456 -XPOST '192.168.43.104:9200/_snapshot/hsbc_backup' -H  'Content-Type: application/json' -d '{ "type": "fs",  "settings": { "location": "/opt/elasticsearch/elasticsearch-6.8.0/repo", "compress": true, "chunk_size": "1g", "max_snapshot_bytes_per_sec": "50m", "max_restore_bytes_per_sec": "50m"}}'

2.备份 并且保留60个备份记录

#!/bin/bash

## define vars
es_url="http://192.168.43.104:9200"
es_bak_repo="hsbc_backup"
es_user="elastic"
es_pass="123456 "
es_bak_date=`date +'%Y-%m-%d-%H'`

echo -e "\n =========== $es_bak_date ============"

## bak
curl -u ${es_user}:${es_pass} -XPUT "${es_url}/_snapshot/${es_bak_repo}/es_bak_${es_bak_date}?wait_for_completion=true"
echo ""

##sleep
sleep 10

##del 最后的60: 定时任务 每4小时备份一次 一天备份6次  数据保留10天
es_del_snapshots=`curl -u ${es_user}:${es_pass} -s -XGET "${es_url}/_snapshot/${es_bak_repo}/_all" | python -m json.tool| python -c 'import json; import sys; input = sys.stdin.read(); result = [x["snapshot"] for x in json.loads(input)["snapshots"]]; print("\n".join(result[:-60]));'`
for es_del_snapshot in ${es_del_snapshots}; do
    echo "===========$es_del_snapshot del"
    curl -u ${es_user}:${es_pass} -s -XDELETE "${es_url}/_snapshot/${es_bak_repo}/${es_del_snapshot}"
    echo ""
done

3.创建定时任务

$ crontab -e
0 */4 * ** /bin/bash /opt/elasticsearch/scripts/es_crontab.sh >> /appvol/logs/crontab/es_bak/crontab.log 2>&1

4.将备份的数据恢复
在原来的机器(192.168.43.104)操作
(1)备份数据

# 查看索引
$ curl -s -u elastic:123456 192.168.43.104:9200/_cat/indices?v |grep -v index| awk '{print $3}'
![查看索引](https://img-blog.csdnimg.cn/202008131431023.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N1bl94dWVnYW5n,size_16,color_FFFFFF,t_70#pic_center)

# 数据备份, 会将所有索引都备份
$ curl -u elastic:123456 -XPUT "http://192.168.43.104:9200/_snapshot/hsbc_backup/es_bak_20200811?wait_for_completion=true"

# 查看备份文件包含的索引
$ curl -s -u elastic:123456 -XGET http://192.168.43.104:9200/_snapshot/hsbc_backup/es_bak_20200811
![备份文件包含的索引](https://img-blog.csdnimg.cn/20200813143145165.png#pic_center)

(2)将备份文件打包拷贝到192.168.43.105, 然后在192.168.43.105上解压到es指定的repo.path下

在新的机器(192.168.43.105)操作

# 查看存在的索引
![查看已经存在的索引](https://img-blog.csdnimg.cn/20200813143227123.png#pic_center)


# 执行shell 进行数据恢复
$ sh es_restore.sh

# 再次查看索引
![数据恢复后查看索引](https://img-blog.csdnimg.cn/20200813143249413.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N1bl94dWVnYW5n,size_16,color_FFFFFF,t_70#pic_center)



##############  shell 脚本
#!/bin/bash

set -e
## define vars
es_url="http://192.168.43.105:9200"
es_bak_repo="hsbc_backup"
es_restore_snapshots="es_bak_20200811"
es_user="elastic"
es_pass="123456"

##(1)获取备份文件中存在的索引
es_get_snapshots_index=`curl -u ${es_user}:${es_pass} -s -XGET "${es_url}/_snapshot/${es_bak_repo}/${es_restore_snapshots}" | python -m json.tool| python -c 'import json; import sys; input = sys.stdin.read(); result=json.loads(input); indices = result["snapshots"][0]["indices"]; sys.stdout.write("  ".join([i for i in indices]));'`

##(2)将备份文件中存在的索引删除  然后恢复
for index in ${es_get_snapshots_index}; do
    if [ ${index} == ".security-6" ]; then # 如果索引为 .security-6 则不能删除
        continue
    fi
    # 判断索引是否存在
    res=`curl -s -u ${es_user}:${es_pass} -XGET "${es_url}/${index}"| grep "error" |grep 404|awk '{print $1}'`
    if [ -z ${res} ] ; then   #如果存在则删除索引
        curl -s -u ${es_user}:${es_pass} -XDELETE ${es_url}/${index}
        echo -e "\ndelete index data : ${index} "
    fi

    #数据恢复
    echo -e "data restore : ${index}\n"
    curl -s -u ${es_user}:${es_pass} -XPOST "${es_url}/_snapshot/${es_bak_repo}/${es_restore_snapshots}/_restore" -H 'Content-Type: application/json' -d '{"indices": "'${index}'"}'
done