Elasticsearch 滚动索引详解
Elasticsearch 滚动索引（Rolling Indices）是一种重要的索引管理策略，主要用于处理时间序列数据、日志数据和不断增长的数据集。通过滚动索引机制，可以实现数据的自动分区、生命周期管理和性能优化。
1. 滚动索引概念
什么是滚动索引
滚动索引是一种按时间或其他维度自动创建新索引的策略，当当前索引达到预设条件（如时间、大小、文档数量）时，系统会自动创建新的索引来接收数据，而旧的索引则进入不同的生命周期阶段。
滚动索引的优势
2. 滚动索引架构设计
整体架构
索引命名策略
3. 滚动策略配置
基于时间的滚动
# 基于时间的滚动索引配置
PUT _index_template/time_based_template
{
"index_patterns": ["logs-*"],
"template": {
"settings": {
"number_of_shards": 3,
"number_of_replicas": 1,
"index.lifecycle.name": "logs_policy",
"index.lifecycle.rollover_alias": "logs"
},
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"message": {
"type": "text"
},
"level": {
"type": "keyword"
}
}
}
},
"data_stream": {}
}
# 索引生命周期策略
PUT _ilm/policy/logs_policy
{
"policy": {
"phases": {
"hot": {
"actions": {
"rollover": {
"max_age": "1d",
"max_size": "50GB",
"max_docs": 100000000
}
}
},
"warm": {
"min_age": "7d",
"actions": {
"shrink": {
"number_of_shards": 1
},
"forcemerge": {
"max_num_segments": 1
}
}
},
"cold": {
"min_age": "30d",
"actions": {
"searchable_snapshot": {
"snapshot_repository": "my_repository"
}
}
},
"delete": {
"min_age": "90d",
"actions": {
"delete": {}
}
}
}
}
}
基于大小的滚动
// Java API 实现基于大小的滚动
public class SizeBasedRolloverManager {
private final ElasticsearchClient client;
private final String indexPattern;
private final long maxSizeBytes;
private final int maxDocs;
public SizeBasedRolloverManager(ElasticsearchClient client, String indexPattern,
long maxSizeBytes, int maxDocs) {
this.client = client;
this.indexPattern = indexPattern;
this.maxSizeBytes = maxSizeBytes;
this.maxDocs = maxDocs;
}
/**
* 检查是否需要滚动
*/
public boolean shouldRollover(String currentIndex) throws IOException {
// 获取索引统计信息
IndicesStatsRequest statsRequest = new IndicesStatsRequest()
.indices(currentIndex);
IndicesStatsResponse statsResponse = client.indices()
.stats(statsRequest, RequestOptions.DEFAULT);
IndexStats indexStats = statsResponse.getIndex(currentIndex);
long totalSize = indexStats.getTotal().getStore().getSizeInBytes();
long docCount = indexStats.getTotal().getDocs().getCount();
log.info("Index {} - Size: {} bytes, Docs: {}", currentIndex, totalSize, docCount);
// 检查滚动条件
return totalSize >= maxSizeBytes || docCount >= maxDocs;
}
/**
* 执行索引滚动
*/
public String performRollover(String aliasName, String newIndexName) throws IOException {
// 创建新索引
CreateIndexRequest createRequest = new CreateIndexRequest(newIndexName);
// 应用索引模板
applyIndexTemplate(createRequest);
CreateIndexResponse createResponse = client.indices()
.create(createRequest, RequestOptions.DEFAULT);
if (createResponse.isAcknowledged()) {
// 更新别名
updateAlias(aliasName, newIndexName);
log.info("Successfully rolled over to new index: {}", newIndexName);
return newIndexName;
} else {
throw new IOException("Failed to create new index: " + newIndexName);
}
}
/**
* 监控索引大小并自动滚动
*/
public void monitorAndAutoRollover(String aliasName) throws IOException {
// 获取当前写入索引
String currentIndex = getCurrentWriteIndex(aliasName);
if (shouldRollover(currentIndex)) {
String newIndexName = generateNewIndexName();
performRollover(aliasName, newIndexName);
}
}
private String getCurrentWriteIndex(String aliasName) throws IOException {
GetAliasesRequest aliasesRequest = new GetAliasesRequest(aliasName);
GetAliasesResponse aliasesResponse = client.indices()
.getAlias(aliasesRequest, RequestOptions.DEFAULT);
// 返回别名的第一个索引（假设只有一个写入索引）
return aliasesResponse.getAliases().keySet().iterator().next();
}
private void updateAlias(String aliasName, String newIndexName) throws IOException {
IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
// 移除旧索引的别名
String oldIndex = getCurrentWriteIndex(aliasName);
aliasesRequest.addAliasAction(
IndicesAliasesRequest.AliasActions.remove()
.index(oldIndex)
.alias(aliasName)
);
// 添加新索引的别名
aliasesRequest.addAliasAction(
IndicesAliasesRequest.AliasActions.add()
.index(newIndexName)
.alias(aliasName)
.writeIndex(true)
);
client.indices().updateAliases(aliasesRequest, RequestOptions.DEFAULT);
}
private String generateNewIndexName() {
// 生成基于时间戳的索引名
return indexPattern + "-" + LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy.MM.dd-HH.mm.ss"));
}
private void applyIndexTemplate(CreateIndexRequest request) {
// 应用索引模板设置
request.settings(Settings.builder()
.put("number_of_shards", 3)
.put("number_of_replicas", 1)
.put("index.lifecycle.name", "size_based_policy")
.build());
}
}
4. 索引生命周期管理 (ILM)
ILM 策略配置
# 完整的 ILM 策略示例
PUT _ilm/policy/comprehensive_policy
{
"policy": {
"phases": {
"hot": {
"min_age": "0ms",
"actions": {
"rollover": {
"max_age": "1d",
"max_size": "50GB",
"max_docs": 100000000
},
"set_priority": {
"priority": 100
},
"unfollow": {}
}
},
"warm": {
"min_age": "7d",
"actions": {
"allocate": {
"number_of_replicas": 0,
"require": {
"data": "warm"
}
},
"forcemerge": {
"max_num_segments": 1
},
"shrink": {
"number_of_shards": 1
},
"set_priority": {
"priority": 50
}
}
},
"cold": {
"min_age": "30d",
"actions": {
"allocate": {
"require": {
"data": "cold"
}
},
"freeze": {},
"set_priority": {
"priority": 0
}
}
},
"frozen": {
"min_age": "90d",
"actions": {
"searchable_snapshot": {
"snapshot_repository": "frozen_repository"
}
}
},
"delete": {
"min_age": "365d",
"actions": {
"delete": {
"delete_searchable_snapshot": true
}
}
}
}
}
}
ILM 监控和管理
// ILM 监控和管理工具
public class ILMMonitor {
private final ElasticsearchClient client;
public ILMMonitor(ElasticsearchClient client) {
this.client = client;
}
/**
* 获取索引生命周期状态
*/
public IndexLifecycleMetadata getIndexLifecycleStatus(String index) throws IOException {
ExplainLifecycleRequest request = new ExplainLifecycleRequest()
.indices(index);
ExplainLifecycleResponse response = client.indexLifecycle()
.explainLifecycle(request, RequestOptions.DEFAULT);
return response.getIndexLifecycleMetadata(index);
}
/**
* 监控 ILM 执行状态
*/
public ILMExecutionStatus monitorILMExecution() throws IOException {
// 获取所有受 ILM 管理的索引
GetLifecycleRequest lifecycleRequest = new GetLifecycleRequest();
GetLifecycleResponse lifecycleResponse = client.indexLifecycle()
.getLifecyclePolicy(lifecycleRequest, RequestOptions.DEFAULT);
Map<String, LifecyclePolicyMetadata> policies = lifecycleResponse.getPolicies();
ILMExecutionStatus status = new ILMExecutionStatus();
for (Map.Entry<String, LifecyclePolicyMetadata> entry : policies.entrySet()) {
String policyName = entry.getKey();
LifecyclePolicyMetadata policyMetadata = entry.getValue();
// 获取使用该策略的索引
String[] indices = getIndicesUsingPolicy(policyName);
for (String index : indices) {
IndexLifecycleMetadata indexMetadata = getIndexLifecycleStatus(index);
// 分析索引状态
analyzeIndexStatus(index, indexMetadata, status);
}
}
return status;
}
/**
* 手动触发索引滚动
*/
public boolean manualRollover(String aliasName) throws IOException {
RolloverRequest rolloverRequest = new RolloverRequest(aliasName, null);
rolloverRequest.addMaxIndexAgeCondition(new TimeValue(1, TimeUnit.MILLISECONDS)); // 立即滚动
RolloverResponse rolloverResponse = client.indices()
.rollover(rolloverRequest, RequestOptions.DEFAULT);
boolean rolledOver = rolloverResponse.isRolledOver();
String oldIndex = rolloverResponse.getOldIndex();
String newIndex = rolloverResponse.getNewIndex();
log.info("Manual rollover completed - Old: {}, New: {}, Success: {}",
oldIndex, newIndex, rolledOver);
return rolledOver;
}
/**
* 修复卡住的索引
*/
public boolean fixStuckIndex(String index) throws IOException {
IndexLifecycleMetadata metadata = getIndexLifecycleStatus(index);
if (metadata == null) {
log.warn("Index {} is not managed by ILM", index);
return false;
}
String currentPhase = metadata.getPhase();
String currentAction = metadata.getAction();
String currentStep = metadata.getStep();
log.info("Index {} is in phase: {}, action: {}, step: {}",
index, currentPhase, currentAction, currentStep);
// 如果索引卡在错误步骤，尝试重试
if ("ERROR".equals(currentStep)) {
retryFailedStep(index);
return true;
}
return false;
}
/**
* 获取 ILM 统计信息
*/
public ILMStatistics getILMStatistics() throws IOException {
ILMExecutionStatus status = monitorILMExecution();
// 统计各种状态的索引数量
long totalIndices = 0;
long hotIndices = 0;
long warmIndices = 0;
long coldIndices = 0;
long errorIndices = 0;
for (IndexStatus indexStatus : status.getIndexStatuses()) {
totalIndices++;
switch (indexStatus.getPhase()) {
case "hot":
hotIndices++;
break;
case "warm":
warmIndices++;
break;
case "cold":
coldIndices++;
break;
default:
if ("ERROR".equals(indexStatus.getStep())) {
errorIndices++;
}
}
}
return new ILMStatistics(totalIndices, hotIndices, warmIndices, coldIndices, errorIndices);
}
private void retryFailedStep(String index) throws IOException {
RetryLifecyclePolicyRequest retryRequest = new RetryLifecyclePolicyRequest(index);
AcknowledgedResponse retryResponse = client.indexLifecycle()
.retryLifecyclePolicy(retryRequest, RequestOptions.DEFAULT);
if (retryResponse.isAcknowledged()) {
log.info("Successfully retried failed step for index: {}", index);
} else {
log.warn("Failed to retry step for index: {}", index);
}
}
private String[] getIndicesUsingPolicy(String policyName) throws IOException {
// 获取所有索引并筛选使用指定策略的索引
GetIndexRequest getIndexRequest = new GetIndexRequest("*");
GetIndexResponse getIndexResponse = client.indices()
.get(getIndexRequest, RequestOptions.DEFAULT);
return Arrays.stream(getIndexResponse.getIndices())
.filter(index -> {
try {
IndexLifecycleMetadata metadata = getIndexLifecycleStatus(index);
return metadata != null && policyName.equals(metadata.getPolicyName());
} catch (IOException e) {
return false;
}
})
.toArray(String[]::new);
}
private void analyzeIndexStatus(String index, IndexLifecycleMetadata metadata, ILMExecutionStatus status) {
String phase = metadata.getPhase();
String action = metadata.getAction();
String step = metadata.getStep();
IndexStatus indexStatus = new IndexStatus(index, phase, action, step);
status.addIndexStatus(indexStatus);
// 检查是否需要告警
if ("ERROR".equals(step)) {
log.error("Index {} is in ERROR state - Phase: {}, Action: {}", index, phase, action);
status.addErrorIndex(index);
}
// 检查执行时间
long ageMillis = System.currentTimeMillis() - metadata.getPhaseTime();
if (ageMillis > TimeUnit.DAYS.toMillis(1)) {
log.warn("Index {} has been in phase {} for more than 1 day", index, phase);
status.addStuckIndex(index);
}
}
}
5. 监控和告警
滚动索引监控
// 滚动索引监控系统
@Component
@Slf4j
public class RollingIndexMonitor {
private final ElasticsearchClient client;
private final MeterRegistry meterRegistry;
private final Counter rolloverCounter;
private final Timer rolloverTimer;
private final Gauge activeIndexSizeGauge;
private final Gauge activeIndexDocsGauge;
public RollingIndexMonitor(ElasticsearchClient client, MeterRegistry meterRegistry) {
this.client = client;
this.meterRegistry = meterRegistry;
// 初始化指标
this.rolloverCounter = Counter.builder("elasticsearch.rollover.count")
.description("Number of index rollovers")
.register(meterRegistry);
this.rolloverTimer = Timer.builder("elasticsearch.rollover.duration")
.description("Time taken for index rollover")
.register(meterRegistry);
this.activeIndexSizeGauge = Gauge.builder("elasticsearch.active.index.size.bytes")
.description("Size of the active index in bytes")
.register(meterRegistry, this, RollingIndexMonitor::getActiveIndexSize);
this.activeIndexDocsGauge = Gauge.builder("elasticsearch.active.index.docs.count")
.description("Number of documents in the active index")
.register(meterRegistry, this, RollingIndexMonitor::getActiveIndexDocCount);
}
/**
* 监控滚动索引状态
*/
@Scheduled(fixedDelay = 300000) // 每5分钟检查一次
public void monitorRollingIndices() {
try {
log.info("Starting rolling index monitoring...");
// 获取所有数据流
GetDataStreamRequest dataStreamRequest = new GetDataStreamRequest("*");
GetDataStreamResponse dataStreamResponse = client.indices()
.getDataStream(dataStreamRequest, RequestOptions.DEFAULT);
for (DataStream dataStream : dataStreamResponse.getDataStreams()) {
monitorDataStream(dataStream);
}
log.info("Rolling index monitoring completed");
} catch (IOException e) {
log.error("Failed to monitor rolling indices", e);
meterRegistry.counter("elasticsearch.monitoring.errors").increment();
}
}
/**
* 监控特定数据流
*/
private void monitorDataStream(DataStream dataStream) {
String dataStreamName = dataStream.getName();
String writeIndex = dataStream.getWriteIndex();
try {
// 获取写入索引的统计信息
IndicesStatsRequest statsRequest = new IndicesStatsRequest()
.indices(writeIndex);
IndicesStatsResponse statsResponse = client.indices()
.stats(statsRequest, RequestOptions.DEFAULT);
IndexStats indexStats = statsResponse.getIndex(writeIndex);
long indexSize = indexStats.getTotal().getStore().getSizeInBytes();
long docCount = indexStats.getTotal().getDocs().getCount();
log.info("DataStream: {}, WriteIndex: {}, Size: {} bytes, Docs: {}",
dataStreamName, writeIndex, indexSize, docCount);
// 检查是否需要告警
checkAlertConditions(dataStreamName, writeIndex, indexSize, docCount);
// 更新指标
updateMetrics(dataStreamName, indexSize, docCount);
} catch (IOException e) {
log.error("Failed to monitor data stream: {}", dataStreamName, e);
}
}
/**
* 检查告警条件
*/
private void checkAlertConditions(String dataStreamName, String writeIndex,
long indexSize, long docCount) {
// 检查索引大小
long maxSizeThreshold = 50L * 1024 * 1024 * 1024; // 50GB
if (indexSize > maxSizeThreshold) {
sendAlert("WARNING", String.format(
"Data stream %s write index %s size (%d bytes) exceeds threshold (%d bytes)",
dataStreamName, writeIndex, indexSize, maxSizeThreshold));
}
// 检查文档数量
long maxDocsThreshold = 100_000_000L; // 1亿文档
if (docCount > maxDocsThreshold) {
sendAlert("WARNING", String.format(
"Data stream %s write index %s document count (%d) exceeds threshold (%d)",
dataStreamName, writeIndex, docCount, maxDocsThreshold));
}
// 检查索引年龄
try {
long indexAge = getIndexAge(writeIndex);
long maxAgeThreshold = TimeUnit.DAYS.toMillis(1); // 1天
if (indexAge > maxAgeThreshold) {
sendAlert("INFO", String.format(
"Data stream %s write index %s age (%d ms) exceeds threshold (%d ms)",
dataStreamName, writeIndex, indexAge, maxAgeThreshold));
}
} catch (IOException e) {
log.error("Failed to get index age for: {}", writeIndex, e);
}
}
/**
* 获取活跃索引大小
*/
private double getActiveIndexSize() {
try {
// 获取当前写入索引
String writeIndex = getCurrentWriteIndex();
if (writeIndex != null) {
IndicesStatsRequest statsRequest = new IndicesStatsRequest()
.indices(writeIndex);
IndicesStatsResponse statsResponse = client.indices()
.stats(statsRequest, RequestOptions.DEFAULT);
return statsResponse.getIndex(writeIndex)
.getTotal().getStore().getSizeInBytes();
}
} catch (IOException e) {
log.error("Failed to get active index size", e);
}
return 0.0;
}
/**
* 获取活跃索引文档数量
*/
private double getActiveIndexDocCount() {
try {
String writeIndex = getCurrentWriteIndex();
if (writeIndex != null) {
IndicesStatsRequest statsRequest = new IndicesStatsRequest()
.indices(writeIndex);
IndicesStatsResponse statsResponse = client.indices()
.stats(statsRequest, RequestOptions.DEFAULT);
return statsResponse.getIndex(writeIndex)
.getTotal().getDocs().getCount();
}
} catch (IOException e) {
log.error("Failed to get active index doc count", e);
}
return 0.0;
}
/**
* 获取当前写入索引
*/
private String getCurrentWriteIndex() throws IOException {
// 这里假设监控主要的日志数据流
String[] dataStreams = {"logs-*", "metrics-*"};
for (String pattern : dataStreams) {
GetDataStreamRequest request = new GetDataStreamRequest(pattern);
GetDataStreamResponse response = client.indices()
.getDataStream(request, RequestOptions.DEFAULT);
if (!response.getDataStreams().isEmpty()) {
return response.getDataStreams().get(0).getWriteIndex();
}
}
return null;
}
/**
* 获取索引年龄
*/
private long getIndexAge(String index) throws IOException {
GetIndexRequest request = new GetIndexRequest(index);
GetIndexResponse response = client.indices()
.get(request, RequestOptions.DEFAULT);
Settings settings = response.getSettings().get(index);
String creationDate = settings.get("index.creation_date");
if (creationDate != null) {
long creationTime = Long.parseLong(creationDate);
return System.currentTimeMillis() - creationTime;
}
return 0;
}
/**
* 发送告警
*/
private void sendAlert(String severity, String message) {
log.warn("ALERT [{}]: {}", severity, message);
// 可以集成邮件、短信、钉钉等告警系统
// 这里只是记录日志，实际应用中需要集成具体的告警服务
// 记录告警指标
meterRegistry.counter("elasticsearch.alerts", "severity", severity).increment();
// 发送告警通知（示例）
if ("CRITICAL".equals(severity)) {
// 发送紧急告警
sendCriticalAlert(message);
} else if ("WARNING".equals(severity)) {
// 发送警告告警
sendWarningAlert(message);
}
}
/**
* 更新指标
*/
private void updateMetrics(String dataStreamName, long indexSize, long docCount) {
// 更新数据流特定的指标
meterRegistry.gauge("elasticsearch.datastream.size.bytes",
Tags.of("datastream", dataStreamName), indexSize);
meterRegistry.gauge("elasticsearch.datastream.docs.count",
Tags.of("datastream", dataStreamName), docCount);
}
/**
* 生成监控报告
*/
public String generateMonitoringReport() {
StringBuilder report = new StringBuilder();
report.append("=== Elasticsearch Rolling Index Monitoring Report ===\n");
report.append("Generated at: ").append(LocalDateTime.now()).append("\n\n");
// 获取指标数据
double rolloverCount = rolloverCounter.count();
double avgRolloverTime = rolloverTimer.mean(TimeUnit.MILLISECONDS);
report.append("Rollover Statistics:\n");
report.append("- Total Rollovers: ").append(rolloverCount).append("\n");
report.append("- Average Rollover Time: ").append(String.format("%.2f", avgRolloverTime)).append(" ms\n");
report.append("- Active Index Size: ").append(getActiveIndexSize()).append(" bytes\n");
report.append("- Active Index Docs: ").append(getActiveIndexDocCount()).append("\n");
return report.toString();
}
private void sendCriticalAlert(String message) {
// 实现紧急告警逻辑
log.error("CRITICAL ALERT: {}", message);
}
private void sendWarningAlert(String message) {
// 实现警告告警逻辑
log.warn("WARNING ALERT: {}", message);
}
}
6. 最佳实践和性能优化
滚动索引最佳实践
# 最佳实践配置示例
PUT _index_template/best_practice_template
{
"index_patterns": ["application-logs-*"],
"template": {
"settings": {
"number_of_shards": 3,
"number_of_replicas": 1,
"index.lifecycle.name": "optimized_policy",
"index.lifecycle.rollover_alias": "application-logs",
# 性能优化设置
"index.refresh_interval": "30s",
"index.translog.durability": "async",
"index.translog.sync_interval": "30s",
# 存储优化
"index.codec": "best_compression",
"index.routing.allocation.total_shards_per_node": 2,
# 查询优化
"index.max_result_window": 50000,
"index.max_inner_result_window": 100,
"index.search.idle.after": "30s"
},
"mappings": {
"properties": {
"@timestamp": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"message": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"level": {
"type": "keyword"
},
"application": {
"type": "keyword"
},
"host": {
"type": "keyword"
}
}
}
},
"composed_of": ["logs-mappings", "logs-settings"],
"priority": 500
}
性能优化策略
// 滚动索引性能优化器
public class RollingIndexOptimizer {
private final ElasticsearchClient client;
public RollingIndexOptimizer(ElasticsearchClient client) {
this.client = client;
}
/**
* 优化滚动索引性能
*/
public void optimizeRollingIndex(String indexPattern) throws IOException {
log.info("Optimizing rolling index: {}", indexPattern);
// 1. 分析当前索引性能
IndexPerformanceAnalysis analysis = analyzeIndexPerformance(indexPattern);
// 2. 基于分析结果进行优化
if (analysis.isQueryPerformancePoor()) {
optimizeQueryPerformance(indexPattern);
}
if (analysis.isWritePerformancePoor()) {
optimizeWritePerformance(indexPattern);
}
if (analysis.isStorageEfficiencyPoor()) {
optimizeStorageEfficiency(indexPattern);
}
// 3. 应用优化建议
applyOptimizationRecommendations(indexPattern, analysis);
}
/**
* 分析索引性能
*/
private IndexPerformanceAnalysis analyzeIndexPerformance(String indexPattern) throws IOException {
IndexPerformanceAnalysis analysis = new IndexPerformanceAnalysis();
// 获取索引统计信息
IndicesStatsRequest statsRequest = new IndicesStatsRequest()
.indices(indexPattern);
IndicesStatsResponse statsResponse = client.indices()
.stats(statsRequest, RequestOptions.DEFAULT);
for (String index : statsResponse.getIndices().keySet()) {
IndexStats indexStats = statsResponse.getIndex(index);
// 分析查询性能
SearchStats searchStats = indexStats.getTotal().getSearch();
long totalQueryTime = searchStats.getTotal().getQueryTimeInMillis();
long totalQueryCount = searchStats.getTotal().getQueryCount();
if (totalQueryCount > 0) {
double avgQueryTime = (double) totalQueryTime / totalQueryCount;
analysis.addQueryMetric(index, avgQueryTime, totalQueryCount);
}
// 分析写入性能
IndexingStats indexingStats = indexStats.getTotal().getIndexing();
long totalIndexTime = indexingStats.getTotal().getIndexTimeInMillis();
long totalIndexCount = indexingStats.getTotal().getIndexCount();
if (totalIndexCount > 0) {
double avgIndexTime = (double) totalIndexTime / totalIndexCount;
analysis.addIndexingMetric(index, avgIndexTime, totalIndexCount);
}
// 分析存储效率
long storeSize = indexStats.getTotal().getStore().getSizeInBytes();
long docCount = indexStats.getTotal().getDocs().getCount();
if (docCount > 0) {
double bytesPerDoc = (double) storeSize / docCount;
analysis.addStorageMetric(index, bytesPerDoc, storeSize);
}
}
return analysis;
}
/**
* 优化查询性能
*/
private void optimizeQueryPerformance(String indexPattern) throws IOException {
log.info("Optimizing query performance for: {}", indexPattern);
// 1. 优化分片数量
optimizeShardCount(indexPattern);
// 2. 优化副本策略
optimizeReplicaStrategy(indexPattern);
// 3. 优化缓存设置
optimizeCacheSettings(indexPattern);
// 4. 优化映射设置
optimizeMappingSettings(indexPattern);
}
/**
* 优化写入性能
*/
private void optimizeWritePerformance(String indexPattern) throws IOException {
log.info("Optimizing write performance for: {}", indexPattern);
// 1. 优化刷新间隔
optimizeRefreshInterval(indexPattern);
// 2. 优化事务日志设置
optimizeTranslogSettings(indexPattern);
// 3. 优化分片分配
optimizeShardAllocation(indexPattern);
// 4. 优化批量写入设置
optimizeBulkSettings(indexPattern);
}
/**
* 优化存储效率
*/
private void optimizeStorageEfficiency(String indexPattern) throws IOException {
log.info("Optimizing storage efficiency for: {}", indexPattern);
// 1. 执行强制合并
forceMergeIndices(indexPattern);
// 2. 优化压缩设置
optimizeCompressionSettings(indexPattern);
// 3. 优化文档值设置
optimizeDocValuesSettings(indexPattern);
// 4. 清理无用字段
cleanupUnusedFields(indexPattern);
}
/**
* 优化分片数量
*/
private void optimizeShardCount(String indexPattern) throws IOException {
// 分析数据量和查询模式
// 基于分析结果调整分片数量
// 对于时间序列数据，通常每个分片 20-50GB 是比较合适的
}
/**
* 执行强制合并
*/
private void forceMergeIndices(String indexPattern) throws IOException {
String[] indices = getIndicesMatchingPattern(indexPattern);
for (String index : indices) {
// 只对历史索引执行强制合并
if (isHistoricalIndex(index)) {
ForceMergeRequest forceMergeRequest = new ForceMergeRequest(index)
.maxNumSegments(1)
.onlyExpungeDeletes(false)
.flush(true);
ForceMergeResponse response = client.indices()
.forcemerge(forceMergeRequest, RequestOptions.DEFAULT);
log.info("Force merge completed for index: {} - Total shards: {}, Successful: {}",
index, response.getTotalShards(), response.getSuccessfulShards());
}
}
}
/**
* 优化刷新间隔
*/
private void optimizeRefreshInterval(String indexPattern) throws IOException {
// 对于写入密集型索引，可以增加刷新间隔
UpdateSettingsRequest settingsRequest = new UpdateSettingsRequest(indexPattern)
.settings(Settings.builder()
.put("index.refresh_interval", "60s")
.build());
client.indices().putSettings(settingsRequest, RequestOptions.DEFAULT);
log.info("Updated refresh interval for: {}", indexPattern);
}
/**
* 优化缓存设置
*/
private void optimizeCacheSettings(String indexPattern) throws IOException {
UpdateSettingsRequest settingsRequest = new UpdateSettingsRequest(indexPattern)
.settings(Settings.builder()
.put("indices.queries.cache.size", "15%")
.put("indices.fielddata.cache.size", "20%")
.build());
client.indices().putSettings(settingsRequest, RequestOptions.DEFAULT);
log.info("Updated cache settings for: {}", indexPattern);
}
private String[] getIndicesMatchingPattern(String pattern) throws IOException {
GetIndexRequest request = new GetIndexRequest(pattern);
GetIndexResponse response = client.indices().get(request, RequestOptions.DEFAULT);
return response.getIndices();
}
private boolean isHistoricalIndex(String index) {
// 基于索引名称或创建时间判断是否为历史索引
// 这里简化处理，实际应用中需要更复杂的逻辑
return index.contains("2023") || index.contains("2024.01");
}
private void applyOptimizationRecommendations(String indexPattern, IndexPerformanceAnalysis analysis) {
// 应用具体的优化建议
log.info("Applying optimization recommendations for: {}", indexPattern);
// 可以生成详细的优化报告
String optimizationReport = generateOptimizationReport(analysis);
log.info("Optimization report generated:\n{}", optimizationReport);
}
private String generateOptimizationReport(IndexPerformanceAnalysis analysis) {
StringBuilder report = new StringBuilder();
report.append("=== Rolling Index Optimization Report ===\n");
report.append("Generated at: ").append(LocalDateTime.now()).append("\n\n");
report.append("Query Performance Issues:\n");
for (String index : analysis.getPoorQueryPerformanceIndices()) {
report.append("- ").append(index).append(": ")
.append(String.format("%.2f ms avg query time",
analysis.getAverageQueryTime(index))).append("\n");
}
report.append("\nWrite Performance Issues:\n");
for (String index : analysis.getPoorWritePerformanceIndices()) {
report.append("- ").append(index).append(": ")
.append(String.format("%.2f ms avg index time",
analysis.getAverageIndexTime(index))).append("\n");
}
report.append("\nStorage Efficiency Issues:\n");
for (String index : analysis.getPoorStorageEfficiencyIndices()) {
report.append("- ").append(index).append(": ")
.append(String.format("%.2f bytes per doc",
analysis.getBytesPerDocument(index))).append("\n");
}
return report.toString();
}
}
7. 实际应用案例
日志收集系统
# 日志收集系统配置
PUT _index_template/log_collection_template
{
"index_patterns": ["app-logs-*"],
"template": {
"settings": {
"number_of_shards": 3,
"number_of_replicas": 1,
"index.lifecycle.name": "log_collection_policy",
"index.lifecycle.rollover_alias": "app-logs",
# 性能优化
"index.refresh_interval": "30s",
"index.translog.durability": "async",
"index.codec": "best_compression",
# 分片策略
"index.routing.allocation.total_shards_per_node": 2,
"index.routing.allocation.awareness.attributes": "rack,zone"
},
"mappings": {
"properties": {
"@timestamp": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"level": {
"type": "keyword"
},
"logger": {
"type": "keyword"
},
"message": {
"type": "text",
"analyzer": "standard"
},
"application": {
"type": "keyword"
},
"host": {
"type": "keyword"
},
"trace_id": {
"type": "keyword"
},
"user_id": {
"type": "keyword"
}
}
}
},
"data_stream": {}
}
# 日志收集生命周期策略
PUT _ilm/policy/log_collection_policy
{
"policy": {
"phases": {
"hot": {
"min_age": "0ms",
"actions": {
"rollover": {
"max_age": "1d",
"max_size": "30GB",
"max_docs": 50000000
},
"set_priority": {
"priority": 100
}
}
},
"warm": {
"min_age": "3d",
"actions": {
"allocate": {
"number_of_replicas": 0,
"require": {
"data": "warm"
}
},
"forcemerge": {
"max_num_segments": 1
},
"shrink": {
"number_of_shards": 1
},
"set_priority": {
"priority": 50
}
}
},
"cold": {
"min_age": "7d",
"actions": {
"allocate": {
"require": {
"data": "cold"
}
},
"freeze": {},
"set_priority": {
"priority": 0
}
}
},
"delete": {
"min_age": "30d",
"actions": {
"delete": {}
}
}
}
}
}
指标监控系统
# 指标监控系统配置
PUT _index_template/metrics_template
{
"index_patterns": ["metrics-*"],
"template": {
"settings": {
"number_of_shards": 2,
"number_of_replicas": 1,
"index.lifecycle.name": "metrics_policy",
"index.lifecycle.rollover_alias": "metrics",
# 针对指标数据的优化
"index.refresh_interval": "60s",
"index.translog.durability": "async",
"index.codec": "best_compression",
# 时间序列优化
"index.mode": "time_series",
"index.routing_path": ["host", "metricset"],
# 存储优化
"index.store.preload": ["*"],
"index.max_terms_count": 65536
},
"mappings": {
"properties": {
"@timestamp": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"host": {
"type": "keyword",
"time_series_dimension": true
},
"metricset": {
"type": "keyword",
"time_series_dimension": true
},
"cpu": {
"properties": {
"usage": {
"type": "double",
"time_series_metric": "gauge"
}
}
},
"memory": {
"properties": {
"usage": {
"type": "double",
"time_series_metric": "gauge"
},
"total": {
"type": "double",
"time_series_metric": "counter"
}
}
}
}
}
},
"data_stream": {}
}
# 指标数据生命周期策略
PUT _ilm/policy/metrics_policy
{
"policy": {
"phases": {
"hot": {
"min_age": "0ms",
"actions": {
"rollover": {
"max_age": "6h",
"max_size": "10GB",
"max_docs": 10000000
},
"set_priority": {
"priority": 100
}
}
},
"warm": {
"min_age": "1d",
"actions": {
"allocate": {
"number_of_replicas": 0,
"require": {
"data": "warm"
}
},
"forcemerge": {
"max_num_segments": 1
},
"set_priority": {
"priority": 50
}
}
},
"cold": {
"min_age": "7d",
"actions": {
"allocate": {
"require": {
"data": "cold"
}
},
"freeze": {},
"set_priority": {
"priority": 0
}
}
},
"frozen": {
"min_age": "30d",
"actions": {
"searchable_snapshot": {
"snapshot_repository": "metrics_repository"
}
}
},
"delete": {
"min_age": "365d",
"actions": {
"delete": {
"delete_searchable_snapshot": true
}
}
}
}
}
}
总结
Elasticsearch 滚动索引是处理时间序列数据和不断增长数据集的强大工具。通过合理的滚动策略、生命周期管理和性能优化，可以实现：
- 高性能：通过索引分区减少查询范围，提高查询效率
- 低成本：通过冷热数据分离和压缩优化降低存储成本
- 易管理：通过自动化生命周期管理减少运维工作量
- 可扩展：支持水平扩展和并行处理大规模数据
掌握滚动索引的原理和最佳实践，对于构建高性能、高可用的 Elasticsearch 应用至关重要。
扫一扫
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
