容器资源监控与自动调整系统实现
项目地址: https://gitcode.com/gh_mirrors/do/docker-py 一、系统架构设计
容器资源监控与自动调整系统基于以下三层架构设计：
1.** 数据采集层 ：通过Docker SDK定期获取容器资源使用数据（CPU、内存、网络I/O等） 2. 分析决策层 ：基于预设策略分析资源数据，决定是否需要扩缩容或调整资源配置 3. 执行控制层 **：通过Docker API执行容器生命周期操作和资源调整
import asyncio
import time
import psutil
import docker
from docker.models.containers import Container
from typing import Dict, List, Optional, Tuple, Callable
import uuid
import json
import logging
# 配置日志
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger("ContainerMonitor")
class ResourceMetrics:
"""容器资源指标采集与计算"""
def __init__(self, container: Container):
self.container = container
self.previous_stats = None
async def get_current_stats(self) -> Dict:
"""获取容器当前资源使用情况"""
try:
# 使用Docker SDK获取容器统计信息
stats = await asyncio.get_event_loop().run_in_executor(
None,
self.container.stats,
stream=False,
decode=True
)
return stats
except Exception as e:
logger.error(f"获取容器 {self.container.name} 统计信息失败: {str(e)}")
return {}
async def calculate_metrics(self) -> Dict:
"""计算容器资源指标（CPU使用率、内存使用率等）"""
current_stats = await self.get_current_stats()
if not current_stats or not self.previous_stats:
self.previous_stats = current_stats
return {}
metrics = {}
# 计算CPU使用率
cpu_usage = self._calculate_cpu_usage(current_stats)
metrics['cpu_usage'] = cpu_usage
# 计算内存使用率
mem_usage = self._calculate_memory_usage(current_stats)
metrics['mem_usage'] = mem_usage
# 计算网络I/O
network_usage = self._calculate_network_usage(current_stats)
metrics['network_usage'] = network_usage
self.previous_stats = current_stats
return metrics
def _calculate_cpu_usage(self, current_stats: Dict) -> float:
"""计算CPU使用率（百分比）"""
try:
# 获取CPU使用数据
cpu_stats = current_stats['cpu_stats']
precpu_stats = current_stats['precpu_stats']
# 计算CPU总使用时间
cpu_delta = cpu_stats['cpu_usage']['total_usage'] - precpu_stats['cpu_usage']['total_usage']
system_delta = cpu_stats['system_cpu_usage'] - precpu_stats['system_cpu_usage']
if system_delta > 0 and 'online_cpus' in cpu_stats:
online_cpus = cpu_stats['online_cpus']
cpu_usage = (cpu_delta / system_delta) * online_cpus * 100
return max(0.0, min(100.0, cpu_usage)) # 限制在0-100%之间
return 0.0
except Exception as e:
logger.error(f"计算CPU使用率失败: {str(e)}")
return 0.0
def _calculate_memory_usage(self, current_stats: Dict) -> float:
"""计算内存使用率（百分比）"""
try:
mem_stats = current_stats['memory_stats']
usage = mem_stats['usage']
limit = mem_stats['limit']
if limit > 0:
mem_usage = (usage / limit) * 100
return max(0.0, min(100.0, mem_usage)) # 限制在0-100%之间
return 0.0
except Exception as e:
logger.error(f"计算内存使用率失败: {str(e)}")
return 0.0
def _calculate_network_usage(self, current_stats: Dict) -> Dict:
"""计算网络I/O使用率"""
try:
network_stats = current_stats['networks']
if not network_stats:
return {}
stats = {}
for iface, data in network_stats.items():
stats[iface] = {
'rx_bytes': data['rx_bytes'],
'tx_bytes': data['tx_bytes'],
'rx_packets': data['rx_packets'],
'tx_packets': data['tx_packets']
}
return stats
except Exception as e:
logger.error(f"计算网络使用率失败: {str(e)}")
return {}
class AutoScalingPolicy:
"""自动扩缩容策略配置"""
def __init__(self):
# 初始配置
self.min_instances = 1
self.max_instances = 10
self.scale_up_threshold = 70 # CPU使用率阈值（%）
self.scale_down_threshold = 30 # CPU使用率阈值（%）
self.scale_interval = 60 # 检查间隔（秒）
self.scale_factor = 1 # 每次调整实例数
self.cpu_usage_history = [] # 存储最近的CPU使用率数据
self.history_window = 5 # 历史数据窗口大小
def update_policy(self, new_config: Dict):
"""更新自动扩缩容策略"""
if 'min_instances' in new_config:
self.min_instances = new_config['min_instances']
if 'max_instances' in new_config:
self.max_instances = new_config['max_instances']
if 'scale_up_threshold' in new_config:
self.scale_up_threshold = new_config['scale_up_threshold']
if 'scale_down_threshold' in new_config:
self.scale_down_threshold = new_config['scale_down_threshold']
if 'scale_interval' in new_config:
self.scale_interval = new_config['scale_interval']
if 'scale_factor' in new_config:
self.scale_factor = new_config['scale_factor']
if 'history_window' in new_config:
self.history_window = new_config['history_window']
class ContainerManager:
"""容器管理器"""
def __init__(self, client: docker.DockerClient,
service_name: str,
metrics_interval: int = 10):
self.client = client
self.service_name = service_name
self.metrics_interval = metrics_interval # 指标采集间隔（秒）
self.running = False
self.policy = AutoScalingPolicy()
self.container_map = {} # 容器ID到容器对象的映射
self.metrics_collector = {} # 容器ID到ResourceMetrics的映射
self._stop_event = asyncio.Event()
async def start(self):
"""启动容器管理器"""
self.running = True
logger.info(f"启动容器管理器: {self.service_name}")
# 启动监控任务
asyncio.create_task(self._monitor_loop())
async def stop(self):
"""停止容器管理器"""
self.running = False
self._stop_event.set()
logger.info(f"停止容器管理器: {self.service_name}")
async def _monitor_loop(self):
"""监控主循环"""
while not self._stop_event.is_set():
# 检查容器状态
await self._check_container_status()
# 收集资源指标
await self._collect_metrics()
# 检查是否需要扩缩容
await self._check_auto_scaling()
# 等待指定间隔
try:
await asyncio.wait_for(
self._stop_event.wait(),
timeout=self.policy.scale_interval
)
except asyncio.TimeoutError:
continue
async def _check_container_status(self):
"""检查容器状态，自动重启异常容器"""
try:
# 获取所有匹配服务名称的容器
filters = {"name": self.service_name}
containers = await self._get_containers(filters)
# 检查每个容器状态
for container in containers:
container_id = container.id
metrics = self.metrics_collector.get(container_id)
if container.status != "running":
logger.warning(f"容器 {container.name} 状态异常: {container.status}，尝试重启...")
try:
# 尝试重启容器
await self._restart_container(container)
logger.info(f"容器 {container.name} 重启成功")
# 更新容器元数据
if container_id in self.container_map:
self.container_map[container_id] = container
except Exception as e:
logger.error(f"重启容器 {container.name} 失败: {str(e)}")
# 如果重启失败，检查是否达到最大重启次数
# 这里简化处理，实际应用中可添加重试机制
except Exception as e:
logger.error(f"检查容器状态时出错: {str(e)}")
async def _collect_metrics(self):
"""收集容器资源指标"""
try:
# 获取所有运行中的容器
filters = {"name": self.service_name, "status": "running"}
containers = await self._get_containers(filters)
# 为每个容器收集指标
for container in containers:
container_id = container.id
# 初始化指标收集器（如果不存在）
if container_id not in self.metrics_collector:
self.metrics_collector[container_id] = ResourceMetrics(container)
# 计算并存储指标
metrics = await self.metrics_collector[container_id].calculate_metrics()
# 存储到历史数据
if 'cpu_usage' in metrics and 'mem_usage' in metrics:
self.policy.cpu_usage_history.append({
'timestamp': time.time(),
'cpu': metrics['cpu_usage'],
'mem': metrics['mem_usage']
})
# 保持历史数据窗口大小
if len(self.policy.cpu_usage_history) > self.policy.history_window:
self.policy.cpu_usage_history.pop(0)
except Exception as e:
logger.error(f"收集容器指标时出错: {str(e)}")
async def _check_auto_scaling(self):
"""根据资源指标检查是否需要扩缩容"""
try:
# 获取平均CPU使用率
avg_cpu = self._calculate_average_cpu()
if avg_cpu == 0:
logger.warning("无法计算平均CPU使用率，跳过扩缩容检查")
return
current_instances = len(await self._get_containers())
logger.info(f"当前 {current_instances} 个实例，平均CPU使用率: {avg_cpu:.2f}%")
# 确定是否需要扩缩容
if avg_cpu > self.policy.scale_up_threshold and current_instances < self.policy.max_instances:
logger.info(f"CPU使用率 {avg_cpu:.2f}% 超过阈值 {self.policy.scale_up_threshold}%，准备扩容")
await self._scale_up(current_instances)
elif avg_cpu < self.policy.scale_down_threshold and current_instances > self.policy.min_instances:
logger.info(f"CPU使用率 {avg_cpu:.2f}% 低于阈值 {self.policy.scale_down_threshold}%，准备缩容")
await self._scale_down(current_instances)
else:
logger.debug(f"CPU使用率 {avg_cpu:.2f}% 在阈值范围内，无需调整实例数")
except Exception as e:
logger.error(f"自动扩缩容检查时出错: {str(e)}")
def _calculate_average_cpu(self) -> float:
"""计算平均CPU使用率"""
if not self.policy.cpu_usage_history:
return 0
# 仅考虑最近history_window个样本
relevant_history = self.policy.cpu_usage_history[-self.policy.history_window:]
return sum(metric['cpu'] for metric in relevant_history) / len(relevant_history)
async def _scale_up(self, current_instances: int):
"""扩容容器实例"""
new_instances = current_instances + self.policy.scale_factor
new_instances = min(new_instances, self.policy.max_instances)
try:
# 拉取最新镜像
image_tag = f"{self.service_name}:latest"
logger.info(f"开始拉取最新镜像 {image_tag} 用于扩容")
await self._pull_image(image_tag)
# 创建新容器
new_container = await self._create_container(image_tag)
new_container_id = new_container.id
# 更新容器映射
self.container_map[new_container_id] = new_container
self.metrics_collector[new_container_id] = ResourceMetrics(new_container)
logger.info(f"成功扩容容器 {new_container.name}，当前实例数: {new_instances}")
except Exception as e:
logger.error(f"扩容失败: {str(e)}")
# 扩容失败时回滚，实际应用中可添加更复杂的回滚逻辑
async def _scale_down(self, current_instances: int):
"""缩容容器实例"""
new_instances = current_instances - self.policy.scale_factor
new_instances = max(new_instances, self.policy.min_instances)
try:
# 获取容器列表并按CPU使用率排序
containers = await self._get_containers()
sorted_containers = sorted(containers, key=lambda c: await self._get_container_cpu(c), reverse=True)
# 选择CPU使用率最高的容器进行缩容
containers_to_remove = sorted_containers[:current_instances - new_instances]
for container in containers_to_remove:
container_id = container.id
logger.info(f"缩容操作: 停止并删除容器 {container.name}")
# 停止容器
await self._stop_container(container)
# 删除容器
await self._remove_container(container)
# 从映射中移除
if container_id in self.container_map:
del self.container_map[container_id]
if container_id in self.metrics_collector:
del self.metrics_collector[container_id]
logger.info(f"成功缩容至 {new_instances} 个实例")
except Exception as e:
logger.error(f"缩容失败: {str(e)}")
async def _get_containers(self, filters: Optional[Dict] = None) -> List[Container]:
"""获取容器列表"""
try:
containers = self.client.containers.list(filters=filters or {})
return containers
except Exception as e:
logger.error(f"获取容器列表失败: {str(e)}")
return []
async def _get_container_cpu(self, container: Container) -> float:
"""获取单个容器的CPU使用率"""
metrics = ResourceMetrics(container)
stats = await metrics.calculate_metrics()
return stats.get('cpu_usage', 0.0)
async def _create_container(self, image_tag: str) -> Container:
"""创建新容器"""
container_name = f"{self.service_name}-{uuid.uuid4().hex[:6]}"
# 可以在这里添加容器配置参数
container = self.client.containers.run(
image_tag,
name=container_name,
detach=True,
restart_policy={"Name": "always"}
)
return container
async def _pull_image(self, image_tag: str):
"""拉取镜像"""
try:
self.client.images.pull(image_tag)
logger.info(f"镜像 {image_tag} 拉取成功")
except Exception as e:
logger.error(f"镜像拉取失败: {str(e)}")
raise
async def _start_container(self, container: Container):
"""启动容器"""
try:
container.start()
logger.info(f"容器 {container.name} 启动成功")
except Exception as e:
logger.error(f"容器 {container.name} 启动失败: {str(e)}")
raise
async def _stop_container(self, container: Container):
"""停止容器"""
try:
container.stop()
logger.info(f"容器 {container.name} 停止成功")
except Exception as e:
logger.error(f"容器 {container.name} 停止失败: {str(e)}")
raise
async def _restart_container(self, container: Container):
"""重启容器"""
try:
container.restart()
logger.info(f"容器 {container.name} 重启成功")
except Exception as e:
logger.error(f"容器 {container.name} 重启失败: {str(e)}")
raise
async def _remove_container(self, container: Container):
"""删除容器"""
try:
container.remove()
logger.info(f"容器 {container.name} 删除成功")
except Exception as e:
logger.error(f"容器 {container.name} 删除失败: {str(e)}")
raise
class DockerAutoScaler:
"""Docker自动扩缩容主类"""
def __init__(self, config: Optional[Dict] = None):
# 初始化Docker客户端
self.client = docker.from_env()
self.managers = {} # 服务名称到ContainerManager的映射
self.config = {
"metrics_interval": 10,
"scale_interval": 60,
"scale_factor": 1,
"min_instances": 1,
"max_instances": 10,
"scale_up_threshold": 70,
"scale_down_threshold": 30
}
# 应用配置
if config:
self.config.update(config)
async def start_service_monitor(self, service_name: str):
"""启动特定服务的监控"""
if service_name in self.managers:
logger.warning(f"服务 {service_name} 已在监控中")
return
# 创建容器管理器
manager = ContainerManager(
self.client,
service_name,
self.config["metrics_interval"]
)
# 更新自动扩缩容策略
manager.policy.update_policy({
"scale_interval": self.config["scale_interval"],
"scale_factor": self.config["scale_factor"],
"min_instances": self.config["min_instances"],
"max_instances": self.config["max_instances"],
"scale_up_threshold": self.config["scale_up_threshold"],
"scale_down_threshold": self.config["scale_down_threshold"]
})
# 启动管理器
await manager.start()
self.managers[service_name] = manager
logger.info(f"已启动服务 {service_name} 的自动扩缩容监控")
async def stop_service_monitor(self, service_name: str):
"""停止特定服务的监控"""
if service_name in self.managers:
await self.managers[service_name].stop()
del self.managers[service_name]
logger.info(f"已停止服务 {service_name} 的自动扩缩容监控")
async def stop_all_monitors(self):
"""停止所有服务的监控"""
for service_name in list(self.managers.keys()):
await self.stop_service_monitor(service_name)
async def get_service_status(self, service_name: str) -> Dict:
"""获取服务状态信息"""
if service_name not in self.managers:
return {"status": "stopped", "message": "服务监控未运行"}
manager = self.managers[service_name]
containers = await manager._get_containers()
metrics = []
for container in containers:
container_id = container.id
metrics_collector = manager.metrics_collector.get(container_id)
cpu_usage = 0.0
mem_usage = 0.0
if metrics_collector:
stats = await metrics_collector.calculate_metrics()
cpu_usage = stats.get('cpu_usage', 0.0)
mem_usage = stats.get('mem_usage', 0.0)
metrics.append({
"container_id": container.id[:12],
"name": container.name,
"status": container.status,
"cpu_usage": cpu_usage,
"mem_usage": mem_usage,
"created_at": container.attrs['Created']
})
return {
"service_name": service_name,
"status": "running",
"metrics": metrics,
"policy": {
"min_instances": manager.policy.min_instances,
"max_instances": manager.policy.max_instances,
"scale_up_threshold": manager.policy.scale_up_threshold,
"scale_down_threshold": manager.policy.scale_down_threshold
}
}
# 演示代码
async def main():
# 创建自动扩缩容实例
scaler = DockerAutoScaler()
# 启动服务监控
service_name = "my-app"
await scaler.start_service_monitor(service_name)
# 模拟运行5分钟
await asyncio.sleep(5 * 60)
# 获取服务状态
status = await scaler.get_service_status(service_name)
logger.info(json.dumps(status, indent=2))
# 停止监控
await scaler.stop_service_monitor(service_name)
if __name__ == "__main__":
try:
asyncio.run(main())
except KeyboardInterrupt:
print("程序被用户中断")
finally:
print("清理资源...")
二、系统核心功能说明
1.** 资源监控与指标计算 **- 实时采集容器CPU、内存和网络使用数据
- 计算容器资源使用率并进行趋势分析
- 支持历史数据窗口，提供更准确的资源趋势判断
2.** 自动扩缩容策略 **- 基于CPU使用率阈值触发扩缩容
- 可配置最小/最大实例数、扩容因子
- 支持自定义阈值和历史数据窗口大小
3.** 容器生命周期管理 **- 异常容器自动重启
- 基于CPU使用率智能选择缩容容器
- 支持最新镜像拉取和容器创建
4.** 状态监控与查询 **- 提供服务状态和资源使用情况的实时查询接口
- 支持容器级别的详细指标展示
三、性能优化与扩展建议
1.** 资源优化 **- 容器CPU使用率计算采用历史平均而非瞬时值
- 批量处理容器操作，减少API调用次数
- 配置适当的线程池大小，避免资源耗尽
2.** 扩展性考虑 **- 引入Kubernetes API实现更复杂的集群级调度
- 集成Prometheus+Grafana实现更全面的监控可视化
- 添加负载均衡机制，支持多实例服务的流量分配
3.** 高可用增强 **- 实现主从监控节点冗余
- 添加容器健康检查和自愈机制
- 支持动态配置更新，无需重启监控系统
该系统通过异步编程模型和Docker SDK高效集成，实现了容器资源的实时监控和智能扩缩容，适用于云原生应用和微服务架构的弹性伸缩需求。
创作声明：本文部分内容由AI辅助生成（AIGC），仅供参考
