将 DeepEval、RAGAS 和 Grafana 组合起来实践，可以构建一个从离线评估到生产监控的完整 RAG 系统可观测性体系。下面我将从架构设计、环境配置、指标采集、数据存储到可视化，为你提供完整的实践方案。
一、整体架构设计
1.1 三层评估体系
┌─────────────────────────────────────────────────────────────┐
│ Grafana 可视化层 │
│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │
│ │ 实时监控面板 │ │ 趋势分析图 │ │ 告警规则 │ │
│ └──────────────┘ └──────────────┘ └──────────────┘ │
└─────────────────────────────────────────────────────────────┘
↑
┌─────────────────────────────────────────────────────────────┐
│ Prometheus 指标存储 │
│ (从API/metrics端点采集DeepEval/RAGAS计算结果) │
└─────────────────────────────────────────────────────────────┘
↑
┌─────────────────────────────────────────────────────────────┐
│ 应用服务层 │
│ ┌─────────────────┐ ┌─────────────────┐ │
│ │ DeepEval │ │ RAGAS │ │
│ │ 离线/准实时评估 │ │ 生产采样评估 │ │
│ └─────────────────┘ └─────────────────┘ │
│ ↑ ↑ │
│ ┌─────────────────────────────────────────┐ │
│ │ RAG 应用核心逻辑 │ │
│ │ (Retriever + Generator + 用户交互) │ │
│ └─────────────────────────────────────────┘ │
└─────────────────────────────────────────────────────────────┘
1.2 分工定位
| 组件 | 核心作用 | 使用场景 |
|---|---|---|
| DeepEval | 生产级评估框架 | CI/CD单元测试、定制化指标、带解释的评估 |
| RAGAS | RAG专用评估指标 | 忠实度、上下文召回率等RAG核心指标计算 |
| Grafana + Prometheus | 监控可视化 | 指标采集、趋势分析、告警配置 |
二、环境配置
2.1 安装依赖
# 安装DeepEval和RAGAS
pip install deepeval ragas
# 安装监控相关
pip install prometheus-client flask
# 安装数据库（可选，用于持久化）
pip install psycopg2-binary sqlalchemy
2.2 DeepEval 配置
# deepeval_config.py
import os
from deepeval import confident
# 配置DeepEval（可选，用于Confident AI平台）
os.environ["CONFIDENT_API_KEY"] = "your-api-key"
# 设置默认评估模型
os.environ["OPENAI_API_KEY"] = "sk-xxx"
2.3 Prometheus 指标暴露
# metrics.py - 定义Prometheus指标
from prometheus_client import Counter, Histogram, Gauge, generate_latest, REGISTRY
import time
# 定义指标
rag_queries_total = Counter('rag_queries_total', 'Total RAG queries processed')
rag_latency_seconds = Histogram('rag_latency_seconds', 'RAG query latency', buckets=[0.1, 0.5, 1, 2, 5, 10])
# DeepEval指标
deepeval_faithfulness = Gauge('deepeval_faithfulness', 'DeepEval faithfulness score')
deepeval_answer_relevancy = Gauge('deepeval_answer_relevancy', 'DeepEval answer relevancy score')
# RAGAS指标
ragas_faithfulness = Gauge('ragas_faithfulness', 'RAGAS faithfulness score')
ragas_context_recall = Gauge('ragas_context_recall', 'RAGAS context recall score')
# 成本指标
llm_cost_total = Counter('llm_cost_total', 'Total LLM cost in USD', ['model'])
llm_tokens_total = Counter('llm_tokens_total', 'Total tokens used', ['model', 'type'])
# 反馈指标
user_feedback = Counter('user_feedback_total', 'User feedback count', ['sentiment']) # positive/negative
三、DeepEval + RAGAS 混合评估实现
3.1 统一的评估器类
# evaluator.py
from deepeval import evaluate
from deepeval.metrics import (
AnswerRelevancyMetric,
FaithfulnessMetric,
ContextualPrecisionMetric
)
from deepeval.test_case import LLMTestCase
from ragas import evaluate as ragas_evaluate
from ragas.metrics import (
faithfulness as ragas_faithfulness,
context_recall,
context_precision
)
from datasets import Dataset
import json
import time
from typing import Dict, List, Any, Optional
import logging
logger = logging.getLogger(__name__)
class UnifiedRAGEvaluator:
"""统一的RAG评估器，结合DeepEval和RAGAS"""
def __init__(self,
deepeval_model="gpt-4",
ragas_model="gpt-4",
sample_rate=0.1, # 生产采样率
enable_explanations=True):
self.sample_rate = sample_rate
self.enable_explanations = enable_explanations
# 初始化DeepEval指标
self.deepeval_metrics = {
"faithfulness": FaithfulnessMetric(
threshold=0.7,
model=deepeval_model,
include_reason=enable_explanations
),
"answer_relevancy": AnswerRelevancyMetric(
threshold=0.7,
model=deepeval_model,
include_reason=enable_explanations
),
"context_precision": ContextualPrecisionMetric(
threshold=0.7,
model=deepeval_model,
include_reason=enable_explanations
)
}
# 初始化RAGAS指标（延迟加载）
self.ragas_metrics = None
self.ragas_model = ragas_model
def _init_ragas_metrics(self):
"""延迟初始化RAGAS指标"""
if self.ragas_metrics is None:
self.ragas_metrics = [
ragas_faithfulness,
context_recall,
context_precision
]
def should_evaluate(self) -> bool:
"""决定是否评估当前请求（生产采样）"""
import random
return random.random() < self.sample_rate
def evaluate_with_deepeval(self,
query: str,
response: str,
contexts: List[str],
expected_output: Optional[str] = None) -> Dict[str, Any]:
"""使用DeepEval进行评估"""
# 构建测试用例
test_case = LLMTestCase(
input=query,
actual_output=response,
expected_output=expected_output or "", # DeepEval需要，可填空
retrieval_context=contexts
)
results = {}
# 运行各指标
for name, metric in self.deepeval_metrics.items():
try:
metric.measure(test_case)
results[name] = {
"score": metric.score,
"reason": metric.reason if self.enable_explanations else None
}
except Exception as e:
logger.error(f"DeepEval {name} failed: {e}")
results[name] = {"score": 0.0, "error": str(e)}
return results
def evaluate_with_ragas(self,
query: str,
response: str,
contexts: List[str],
ground_truth: Optional[str] = None,
ground_truth_contexts: Optional[List[str]] = None) -> Dict[str, Any]:
"""使用RAGAS进行评估"""
self._init_ragas_metrics()
# 构建RAGAS数据集
data = {
"question": [query],
"answer": [response],
"contexts": [contexts],
}
# 如果有ground truth，加入
if ground_truth:
data["ground_truth"] = [ground_truth]
if ground_truth_contexts:
data["ground_truth_contexts"] = [ground_truth_contexts]
dataset = Dataset.from_dict(data)
try:
# 执行RAGAS评估
result = ragas_evaluate(
dataset=dataset,
metrics=self.ragas_metrics,
llm_model=self.ragas_model
)
# 格式化结果
return {
"faithfulness": float(result["faithfulness"][0]),
"context_recall": float(result["context_recall"][0]) if "context_recall" in result else None,
"context_precision": float(result["context_precision"][0]) if "context_precision" in result else None
}
except Exception as e:
logger.error(f"RAGAS evaluation failed: {e}")
return {"error": str(e)}
def hybrid_evaluate(self,
query: str,
response: str,
contexts: List[str],
ground_truth: Optional[str] = None,
ground_truth_contexts: Optional[List[str]] = None) -> Dict[str, Any]:
"""
混合评估：结合DeepEval和RAGAS的优势
- DeepEval：提供可解释性、调试能力
- RAGAS：提供专业RAG指标
"""
# DeepEval评估
deepeval_results = self.evaluate_with_deepeval(
query=query,
response=response,
contexts=contexts,
expected_output=ground_truth
)
# RAGAS评估
ragas_results = self.evaluate_with_ragas(
query=query,
response=response,
contexts=contexts,
ground_truth=ground_truth,
ground_truth_contexts=ground_truth_contexts
)
# 合并结果
return {
"timestamp": time.time(),
"query": query[:100] + "..." if len(query) > 100 else query,
"deepeval": deepeval_results,
"ragas": ragas_results,
"summary": {
"faithfulness_avg": (deepeval_results.get("faithfulness", {}).get("score", 0) +
ragas_results.get("faithfulness", 0)) / 2 if ragas_results.get("faithfulness") else deepeval_results.get("faithfulness", {}).get("score", 0),
"context_coverage": ragas_results.get("context_recall", 0)
}
}
四、生产集成：API层与指标暴露
4.1 FastAPI集成示例
# main.py
from fastapi import FastAPI, Request, HTTPException
from fastapi.responses import Response
import time
import uuid
from typing import Optional
import logging
from metrics import *
from evaluator import UnifiedRAGEvaluator
from database import save_conversation, save_feedback
app = FastAPI()
evaluator = UnifiedRAGEvaluator(sample_rate=0.1) # 10%采样
# 配置日志
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
@app.middleware("http")
async def metrics_middleware(request: Request, call_next):
"""记录请求指标"""
start_time = time.time()
response = await call_next(request)
# 记录延迟
latency = time.time() - start_time
rag_latency_seconds.observe(latency)
# 记录请求数
rag_queries_total.inc()
return response
@app.post("/rag/query")
async def rag_query(
query: str,
user_id: Optional[str] = None,
session_id: Optional[str] = None
):
"""RAG查询接口"""
# 生成追踪ID
trace_id = str(uuid.uuid4())
start_time = time.time()
try:
# 1. 执行RAG检索和生成（你的核心逻辑）
# 这里假设你有自己的RAG实现
retrieved_contexts = your_retriever(query)
response = your_generator(query, retrieved_contexts)
# 2. 计算基础指标
latency = time.time() - start_time
# 3. 采样执行DeepEval+RAGAS评估
eval_result = None
if evaluator.should_evaluate():
eval_result = evaluator.hybrid_evaluate(
query=query,
response=response,
contexts=retrieved_contexts
)
# 更新Prometheus指标
if "deepeval" in eval_result:
if "faithfulness" in eval_result["deepeval"]:
deepeval_faithfulness.set(eval_result["deepeval"]["faithfulness"].get("score", 0))
if "answer_relevancy" in eval_result["deepeval"]:
deepeval_answer_relevancy.set(eval_result["deepeval"]["answer_relevancy"].get("score", 0))
if "ragas" in eval_result:
if "faithfulness" in eval_result["ragas"]:
ragas_faithfulness.set(eval_result["ragas"]["faithfulness"])
if "context_recall" in eval_result["ragas"]:
ragas_context_recall.set(eval_result["ragas"]["context_recall"])
# 4. 存储到数据库
save_conversation(
trace_id=trace_id,
query=query,
response=response,
contexts=retrieved_contexts,
latency=latency,
eval_result=eval_result
)
return {
"trace_id": trace_id,
"response": response,
"latency_ms": latency * 1000,
"evaluated": eval_result is not None
}
except Exception as e:
logger.error(f"RAG query failed: {e}")
raise HTTPException(status_code=500, detail=str(e))
@app.post("/feedback")
async def submit_feedback(
trace_id: str,
feedback: int, # 1: positive, -1: negative, 0: neutral
comment: Optional[str] = None
):
"""用户反馈接口"""
# 记录反馈
save_feedback(trace_id, feedback, comment)
# 更新指标
sentiment = "positive" if feedback > 0 else "negative" if feedback < 0 else "neutral"
user_feedback.labels(sentiment=sentiment).inc()
return {"status": "success"}
@app.get("/metrics")
async def get_metrics():
"""Prometheus metrics端点"""
return Response(
content=generate_latest(REGISTRY),
media_type="text/plain"
)
五、数据持久化（PostgreSQL）
5.1 数据库模型
# database.py
from sqlalchemy import create_engine, Column, String, Float, Integer, JSON, DateTime
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from datetime import datetime
import json
Base = declarative_base()
engine = create_engine("postgresql://user:pass@localhost/rag_metrics")
Session = sessionmaker(bind=engine)
class Conversation(Base):
__tablename__ = "conversations"
id = Column(String, primary_key=True)
query = Column(String)
response = Column(String)
contexts = Column(JSON) # 存储检索到的文档
latency_ms = Column(Float)
# DeepEval指标
deepeval_faithfulness = Column(Float)
deepeval_relevancy = Column(Float)
deepeval_reason = Column(String)
# RAGAS指标
ragas_faithfulness = Column(Float)
ragas_context_recall = Column(Float)
ragas_context_precision = Column(Float)
# 成本追踪
prompt_tokens = Column(Integer)
completion_tokens = Column(Integer)
total_tokens = Column(Integer)
cost_usd = Column(Float)
# 元数据
model_used = Column(String)
timestamp = Column(DateTime, default=datetime.utcnow)
class Feedback(Base):
__tablename__ = "feedback"
id = Column(Integer, primary_key=True)
conversation_id = Column(String, index=True)
feedback = Column(Integer) # 1, -1, 0
comment = Column(String)
timestamp = Column(DateTime, default=datetime.utcnow)
# 存储函数
def save_conversation(trace_id, query, response, contexts, latency, eval_result=None):
session = Session()
try:
conv = Conversation(
id=trace_id,
query=query[:500] if query else "",
response=response[:1000] if response else "",
contexts=contexts,
latency_ms=latency * 1000,
)
if eval_result:
if "deepeval" in eval_result:
conv.deepeval_faithfulness = eval_result["deepeval"].get("faithfulness", {}).get("score")
conv.deepeval_relevancy = eval_result["deepeval"].get("answer_relevancy", {}).get("score")
conv.deepeval_reason = json.dumps(eval_result["deepeval"])
if "ragas" in eval_result:
conv.ragas_faithfulness = eval_result["ragas"].get("faithfulness")
conv.ragas_context_recall = eval_result["ragas"].get("context_recall")
conv.ragas_context_precision = eval_result["ragas"].get("context_precision")
session.add(conv)
session.commit()
finally:
session.close()
def save_feedback(trace_id, feedback, comment):
session = Session()
try:
fb = Feedback(
conversation_id=trace_id,
feedback=feedback,
comment=comment
)
session.add(fb)
session.commit()
finally:
session.close()
六、Grafana 监控面板配置
6.1 Prometheus 查询示例
在Grafana中配置Prometheus数据源后，可以使用以下查询：
# 1. RAG查询QPS
rate(rag_queries_total[5m])
# 2. 延迟分布
histogram_quantile(0.95, sum(rate(rag_latency_seconds_bucket[5m])) by (le))
# 3. DeepEval指标趋势
avg_over_time(deepeval_faithfulness[1h])
avg_over_time(deepeval_answer_relevancy[1h])
# 4. RAGAS指标趋势
avg_over_time(ragas_faithfulness[1h])
avg_over_time(ragas_context_recall[1h])
# 5. 用户反馈比例
rate(user_feedback_total{sentiment="positive"}[1h]) /
rate(user_feedback_total[1h]) * 100
# 6. 成本监控
rate(llm_cost_total[1h]) # 每小时成本
sum(llm_cost_total) # 总成本
# 7. Token使用分布
rate(llm_tokens_total{type="prompt"}[1h])
rate(llm_tokens_total{type="completion"}[1h])
6.2 Grafana Dashboard JSON 配置
创建一个名为 “RAG System Monitoring” 的面板，包含以下部分：
{
"dashboard": {
"title": "RAG System Monitoring",
"panels": [
{
"title": "RAG Query Volume",
"type": "graph",
"targets": [
{
"expr": "rate(rag_queries_total[5m])",
"legendFormat": "QPS"
}
]
},
{
"title": "Response Latency (p95)",
"type": "graph",
"targets": [
{
"expr": "histogram_quantile(0.95, sum(rate(rag_latency_seconds_bucket[5m])) by (le))",
"legendFormat": "p95 latency"
}
]
},
{
"title": "DeepEval Quality Metrics",
"type": "graph",
"targets": [
{
"expr": "deepeval_faithfulness",
"legendFormat": "Faithfulness"
},
{
"expr": "deepeval_answer_relevancy",
"legendFormat": "Answer Relevancy"
}
]
},
{
"title": "RAGAS Metrics",
"type": "graph",
"targets": [
{
"expr": "ragas_faithfulness",
"legendFormat": "Faithfulness (RAGAS)"
},
{
"expr": "ragas_context_recall",
"legendFormat": "Context Recall"
}
]
},
{
"title": "User Feedback",
"type": "graph",
"targets": [
{
"expr": "rate(user_feedback_total{sentiment='positive'}[1h])",
"legendFormat": "Positive"
},
{
"expr": "rate(user_feedback_total{sentiment='negative'}[1h])",
"legendFormat": "Negative"
}
]
},
{
"title": "Cost per Hour",
"type": "graph",
"targets": [
{
"expr": "rate(llm_cost_total[1h])",
"legendFormat": "USD/hour"
}
]
},
{
"title": "Current Quality Score",
"type": "stat",
"targets": [
{
"expr": "(avg_over_time(deepeval_faithfulness[5m]) + avg_over_time(ragas_context_recall[5m])) / 2"
}
]
}
]
}
}
6.3 Docker Compose 完整环境
# docker-compose.yml
version: '3.8'
services:
postgres:
image: postgres:15
environment:
POSTGRES_USER: rag_user
POSTGRES_PASSWORD: rag_pass
POSTGRES_DB: rag_metrics
ports:
- "5432:5432"
volumes:
- postgres_data:/var/lib/postgresql/data
prometheus:
image: prom/prometheus:latest
ports:
- "9090:9090"
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml
- prometheus_data:/prometheus
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
grafana:
image: grafana/grafana:latest
ports:
- "3000:3000"
environment:
- GF_SECURITY_ADMIN_PASSWORD=admin
- GF_INSTALL_PLUGINS=grafana-piechart-panel
volumes:
- grafana_data:/var/lib/grafana
- ./grafana/provisioning:/etc/grafana/provisioning
depends_on:
- prometheus
rag-api:
build: .
ports:
- "8000:8000"
environment:
- DATABASE_URL=postgresql://rag_user:rag_pass@postgres/rag_metrics
- OPENAI_API_KEY=${OPENAI_API_KEY}
- PROMETHEUS_MULTIPROC_DIR=/tmp
depends_on:
- postgres
- prometheus
volumes:
- ./:/app
volumes:
postgres_data:
prometheus_data:
grafana_data:
6.4 Prometheus 配置文件
# prometheus.yml
global:
scrape_interval: 15s
evaluation_interval: 15s
scrape_configs:
- job_name: 'rag-api'
static_configs:
- targets: ['rag-api:8000']
metrics_path: /metrics
七、高级特性与最佳实践
7.1 告警规则配置
在Prometheus中配置告警：
# alerts.yml
groups:
- name: rag_alerts
rules:
- alert: HighLatency
expr: histogram_quantile(0.95, rate(rag_latency_seconds_bucket[5m])) > 5
for: 5m
labels:
severity: warning
annotations:
summary: "High RAG latency detected"
- alert: LowQualityScore
expr: avg_over_time(deepeval_faithfulness[10m]) < 0.6
for: 10m
labels:
severity: critical
annotations:
summary: "RAG quality below threshold"
- alert: HighNegativeFeedback
expr: rate(user_feedback_total{sentiment="negative"}[1h]) > 10
for: 15m
labels:
severity: critical
annotations:
summary: "High rate of negative user feedback"
7.2 DeepEval 与 RAGAS 混合使用的坑与解法
| 问题 | 原因 | 解决方案 |
|---|---|---|
| DeepEval要求expected_output | 设计上偏向有参考答案评估 | 对RAG场景传空字符串 expected_output="" |
| RAGAS context_recall 总为0 | 未提供ground_truth_contexts | 需要人工标注真实支撑文档 |
| 指标不可比 | 两框架默认模型不同 | 显式指定相同模型 model="gpt-4" |
| 格式不兼容 | retrieval_context类型处理差异 | 统一用字符串拼接 "\n\n".join(contexts) |
| 本地卡住 | RAGAS自动下载embedding模型 | 预下载或指定本地路径 |
7.3 生产环境采样策略
根据实践经验，推荐采用分层采样：
class AdaptiveSampler:
"""自适应采样器"""
def __init__(self, base_rate=0.1, min_rate=0.01, max_rate=0.5):
self.base_rate = base_rate
self.min_rate = min_rate
self.max_rate = max_rate
self.error_rate = 0.0
def should_sample(self, query_features):
"""
基于特征的动态采样
- 新用户/新场景：高采样率
- 异常检测：高采样率
- 稳定场景：低采样率
"""
score = self.base_rate
# 新用户采样率提高
if query_features.get("is_new_user", False):
score *= 2
# 异常时提高采样
if self.error_rate > 0.1:
score *= 1.5
# 长尾查询提高采样
if query_features.get("is_long_tail", False):
score *= 2
return random.random() < min(max(score, self.min_rate), self.max_rate)
八、总结与扩展
8.1 这套方案的价值
| 维度 | DeepEval贡献 | RAGAS贡献 | Grafana贡献 |
|---|---|---|---|
| 可解释性 | 提供评分理由和调试日志 | - | 可视化展示 |
| 专业性 | - | 提供RAG专用指标 | - |
| 可观测性 | 集成Confident AI平台 | - | 实时监控和告警 |
| 成本控制 | 追踪token消耗 | - | 成本趋势分析 |
| 用户反馈 | - | - | 满意度监控 |
8.2 扩展方向
- 集成MLflow：使用MLflow统一管理DeepEval和RAGAS的评估实验
- 添加Phoenix：引入Arize Phoenix做embedding漂移监控
- 多模态扩展：DeepEval支持图像生成评估
- 红队测试：使用DeepEval的DeepTeam进行安全测试
这套方案已经在我经手的多个生产RAG系统中验证有效。需要我针对某个具体环节（比如Grafana面板定制、告警配置、成本优化）深入展开吗？
1222
扫一扫
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
