Elasticsearch Data Migration

最新推荐文章于 2025-05-27 15:32:53 发布

最新推荐文章于 2025-05-27 15:32:53 发布 · 279 阅读

文章标签：

#Elasticsearch 数据迁移

Elasticsearch 专栏收录该内容

8 篇文章

订阅专栏

本文介绍了如何利用JavaAPI进行Elasticsearch的数据迁移，包括获取客户端连接、迁移方法实现以及测试过程。重点在于通过合理设置参数，确保在迁移大量数据时集群性能稳定，同时优化索引速度。

本文介绍ES的数据迁移方案：

由于ES更新速度比较快，很大程度上，我们需要更新版本、插件、甚至更新分词器，单纯的upgrade很有可能不能满足业务需求，更坏的情况下，可能需要重建索引。本文从Java API 的角度来介绍ES的数据迁移（或数据重新索引）。基于以下逻辑实现，个人已测试过2亿数据的迁移，可以放心使用。

1. 获取clientl连接。本文选择transportClient。

public class ClientUtil
{

    static Settings defaultSettings = ImmutableSettings.settingsBuilder().put("client.transport.sniff", false).put("client.transport.ping_timeout","10s").build(); //如果你的集群node数量是稳定的，那么最好关闭sniff。 同时， 将ping时间设置高于默认5s, 很大程序上可以解决No Node available exception.
 
    // 创建私有对象
    private static TransportClient targetClient;
    
    private static TransportClient sourceClient;
 
    static {
        try {
            Class<?> clazz = Class.forName(TransportClient.class.getName());
            Constructor<?> constructor = clazz.getDeclaredConstructor(Settings.class);
            constructor.setAccessible(true);
		    Settings finalSettings = ImmutableSettings.settingsBuilder()
		                .put(defaultSettings)
		                .build();
		    targetClient = (TransportClient) constructor.newInstance(finalSettings);
		    targetClient.addTransportAddress(new InetSocketTransportAddress("192.168.1.100", 9300))
		    			.addTransportAddress(new InetSocketTransportAddress("192.168.1.101", 9300));
		    sourceClient = (TransportClient) constructor.newInstance(finalSettings);
		    sourceClient.addTransportAddress(new InetSocketTransportAddress("192.168.1.110", 9300))
		                .addTransportAddress(new InetSocketTransportAddress("192.168.1.111", 9300));
        } catch (Exception e) {
            e.printStackTrace();
        } 
    }
 
    // 取得源实例
    public static synchronized Client getSourceTransportClient() {
        return sourceClient;
    }
    
    // 取得目标实例
    public static synchronized Client getTargetTransportClient() {
        return targetClient;
    }

}

以上代码用于获取源cluster和目标cluster的client.

2.迁移主方法：

private void doMigrate(Client sourceclient, Client targetclient, String sourceIndexName, String targetIndexName, String indexDocType, int pageSize)
	{
		int total = 0;
		SearchResponse searchResponse = sourceclient.prepareSearch(sourceIndexName).setSearchType(SearchType.SCAN)
				.setQuery(matchAllQuery()).setSize(pageSize).setScroll(TimeValue.timeValueSeconds(20)).execute()
				.actionGet(); //scroll 的time不能太大， 以免对集群造成负载
		boolean exists = targetclient.admin().indices().prepareExists(targetIndexName).execute().actionGet().isExists();
		if (!exists)
			targetclient
					.admin()
					.indices()
					.prepareCreate(targetIndexName)
					.setSettings(
							settingsBuilder().put("index.number_of_replicas", 0).put("index.refresh_interval", "-1"))
					.execute().actionGet(); //设置replica为0， 不refresh， 为了提高索引速度。
		try
		{
			Thread.sleep(200);
		} catch (InterruptedException e)
		{
			e.printStackTrace();
		}
		BulkProcessor bulkProcessor = BulkProcessor.builder(targetclient, new BulkProcessor.Listener()
		{

			@Override
			public void beforeBulk(long executionId, BulkRequest request)
			{

			}

			@Override
			public void afterBulk(long executionId, BulkRequest request, BulkResponse response)
			{
				if (response.hasFailures())
				{
					throw new RuntimeException("BulkResponse show failures: " + response.buildFailureMessage());
				}
			}

			@Override
			public void afterBulk(long executionId, BulkRequest request, Throwable failure)
			{
				throw new RuntimeException("Caught exception in bulk: " + request + ", failure: " + failure, failure);
			}
		}).setConcurrentRequests(10).build(); //设置线程数量， 大小可以根据自己机器调配。

		while (true)
		{
			searchResponse = sourceclient.prepareSearchScroll(searchResponse.getScrollId())
					.setScroll(TimeValue.timeValueSeconds(20)).execute().actionGet();
			for (SearchHit hit : searchResponse.getHits())
			{

				IndexRequestBuilder indexRequestBuilder = targetclient.prepareIndex(targetIndexName, indexDocType);
				indexRequestBuilder.setSource(hit.getSource());
				indexRequestBuilder.setId(hit.getId());
				indexRequestBuilder.setOpType(IndexRequest.OpType.INDEX);
				bulkProcessor.add(indexRequestBuilder.request());
				total++;
			}
			System.out.println("Already migrated : " + total + " records!");
			if (searchResponse.getHits().hits().length == 0)
			{
				break;
			}
		}
		try
		{
			Thread.sleep(10000);//Sleep 10s waiting the cluster.
		} catch (InterruptedException e)
		{
			e.printStackTrace();
		}
		bulkProcessor.close();
		targetclient
		.admin()
		.indices().prepareUpdateSettings(targetIndexName).setSettings(
				settingsBuilder().put("index.number_of_replicas", 1).put("index.refresh_interval", "1s"))
		.execute().actionGet();
	}

3.测试：

public static void main(String[] args) throws ElasticSearchException, IOException, InterruptedException
	{
		int pageSize = 40; //分页大小， 不能过大， 太大影响集群性能， 可能引起no node 异常。
		Client sourceclient = ClientUtil.getSourceTransportClient();
		Client targetclient = ClientUtil.getTargetTransportClient();
	        //调用doMigrate方法。 doMigrate(sourceclient, targetclient, "test", "testnew", "test", pageSize);}