# =======================================================
# 业务领域多模态RAG智能问答系统 (Business-RAG-MultiModal)
# v2.1 - 最终稳定版
# =======================================================
# --- 核心依赖库导入 ---
import os
import hashlib
import json
import logging
import base64
import pathlib
import re
import requests # 用于直接调用Ollama API
import time # 新增：用于重试机制
from typing import List, Dict
# --- LlamaIndex 核心导入 ---
from llama_index.core import Settings
from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, PromptTemplate, Document, StorageContext
from llama_index.core.readers.base import BaseReader as LlamaBaseReader
from llama_index.core.node_parser import SentenceSplitter
from llama_index.core.schema import TextNode
from llama_index.llms.ollama import Ollama
from llama_index.core.postprocessor import SentenceTransformerRerank
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
# --- Milvus 相关导入 ---
from llama_index.vector_stores.milvus import MilvusVectorStore
from pymilvus import utility, connections, Collection
# --- 配置日志 ---
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
# =======================================================
# 1. 全局配置区
# =======================================================
CONFIG = {
"knowledge_base_dir": "knowledge_base",
"image_cache_file": "image_description_cache_offline.json",
"embed_model_path": "D:/models/text2vec-base-chinese",
"reranker_model_path": "D:/models/bge-reranker-v2-m3",
"llm_model_name": "qwen3:8b",
"mllm_model_name": "llava",
"llm_request_timeout": 600.0,
"chunk_size": 512,
"chunk_overlap": 50,
"retrieval_top_k": 10,
"rerank_top_n": 3,
"device": "cpu",
# --- Milvus配置 ---
"milvus_host": "127.0.0.1", # 确保与Docker容器端口一致
"milvus_port": "19530", # 默认端口
"milvus_collection": "law_rag_collection_v1",
"vector_dim": 768
}
# =======================================================
# 2. 核心功能函数区
# =======================================================
def load_image_cache():
if os.path.exists(CONFIG["image_cache_file"]):
with open(CONFIG["image_cache_file"], 'r', encoding='utf-8') as f:
try: return json.load(f)
except json.JSONDecodeError: return {}
return {}
def save_image_cache(cache):
with open(CONFIG["image_cache_file"], 'w', encoding='utf-8') as f:
json.dump(cache, f, ensure_ascii=False, indent=4)
def get_image_description_from_mllm(image_bytes: bytes, local_mllm: Ollama, question="请详细描述这张图片的内容，如果图片中有文字也请一并识别出来。") -> str:
"""
【修正版】直接调用Ollama API来获取图片描述，并使用正确的sha256。
"""
image_cache = load_image_cache()
image_hash = hashlib.sha256(image_bytes).hexdigest()
if image_hash in image_cache:
logger.info(f" - 发现已缓存的离线图片描述 (hash: {image_hash[:8]}...)，从缓存加载。")
return image_cache[image_hash]
logger.info(f" - 未找到图片缓存，正在直接调用Ollama API (模型: {local_mllm.model})...")
try:
image_b64 = base64.b64encode(image_bytes).decode("utf-8")
payload = {
"model": local_mllm.model,
"prompt": question,
"images": [image_b64],
"stream": False
}
response = requests.post(
"http://localhost:11434/api/generate",
json=payload,
timeout=CONFIG["llm_request_timeout"]
)
response.raise_for_status()
response_data = response.json()
description = response_data.get('response', '[模型未返回有效描述]').strip()
formatted_description = f"[图片描述]: {description}\n"
image_cache[image_hash] = formatted_description
save_image_cache(image_cache)
return formatted_description
except requests.exceptions.RequestException as e:
logger.error(f"直接调用Ollama API处理图片时发生网络异常: {e}")
return "[网络异常导致图片处理失败]\n"
except Exception as e:
logger.error(f"处理图片时发生未知异常: {e}")
return "[未知异常导致图片处理失败]\n"
class CustomMultimodalReader(LlamaBaseReader):
def __init__(self, mllm_instance: Ollama):
super().__init__()
self.mllm = mllm_instance
def load_data(self, file_path_obj: pathlib.Path, extra_info: Dict = None) -> List[Document]:
file_path_str = str(file_path_obj)
if file_path_str.endswith(".pdf"):
return self._load_pdf(file_path_str, extra_info)
elif file_path_str.endswith(".docx"):
return self._load_docx(file_path_str, extra_info)
else:
# 为 .txt 文件添加一个基本的加载器
try:
with open(file_path_str, 'r', encoding='utf-8') as f:
text = f.read()
return [Document(text=text, extra_info={**(extra_info or {}), "file_name": os.path.basename(file_path_str)})]
except Exception as e:
logger.error(f"处理TXT文件 '{file_path_str}' 时发生错误: {e}")
return []
def _load_pdf(self, file_path: str, extra_info: Dict = None) -> List[Document]:
documents = []
try:
import pypdf
with open(file_path, "rb") as fp:
reader = pypdf.PdfReader(fp)
if reader.is_encrypted:
logger.warning(f"文件 {os.path.basename(file_path)} 是加密PDF，已跳过。")
return []
for i, page in enumerate(reader.pages):
page_info = {**(extra_info or {}), "page_label": str(i + 1), "file_name": os.path.basename(file_path)}
if page_text := page.extract_text():
documents.append(Document(text=page_text.strip(), extra_info=page_info.copy()))
for img_file_obj in page.images:
if img_bytes := img_file_obj.data:
image_description = get_image_description_from_mllm(img_bytes, self.mllm)
documents.append(Document(text=image_description, extra_info={**page_info.copy(), "content_type": "image_description"}))
except Exception as e:
logger.error(f"处理PDF文件 '{file_path}' 时发生错误: {e}")
return documents
def _load_docx(self, file_path: str, extra_info: Dict = None) -> List[Document]:
documents = []
try:
import docx
doc = docx.Document(file_path)
file_info = {**(extra_info or {}), "file_name": os.path.basename(file_path)}
for para in doc.paragraphs:
if para.text.strip():
documents.append(Document(text=para.text.strip(), extra_info=file_info.copy()))
for table in doc.tables:
table_text = "\n".join([" | ".join([cell.text for cell in row.cells]) for row in table.rows]).strip()
if table_text:
documents.append(Document(text=f"[表格内容]:\n{table_text}", extra_info={**file_info.copy(), "content_type": "table_content"}))
for rel in doc.part.rels.values():
if "image" in rel.target_ref:
if image_bytes := rel.target_part.blob:
image_description = get_image_description_from_mllm(image_bytes, self.mllm)
documents.append(Document(text=image_description, extra_info={**file_info.copy(), "content_type": "image_description"}))
except Exception as e:
logger.error(f"处理DOCX文件 '{file_path}' 时发生错误: {e}")
return documents
# --- RAG核心流程函数 ---
def setup_models_and_services():
"""集中加载所有本地AI模型，并进行全局配置。"""
logger.info("--- 步骤A: 加载所有本地AI模型 ---")
embed_model = HuggingFaceEmbedding(
model_name=CONFIG["embed_model_path"],
device=CONFIG["device"]
)
logger.info(f"成功加载本地嵌入模型: {CONFIG['embed_model_path']}")
llm_model = Ollama(model=CONFIG["llm_model_name"], request_timeout=CONFIG["llm_request_timeout"])
logger.info(f"成功配置本地Ollama LLM (模型: {CONFIG['llm_model_name']})")
mllm_for_parsing = Ollama(model=CONFIG["mllm_model_name"], request_timeout=300.0)
logger.info(f"成功配置本地Ollama MLLM (模型: {CONFIG['mllm_model_name']})")
reranker = SentenceTransformerRerank(
model=CONFIG["reranker_model_path"],
top_n=CONFIG["rerank_top_n"],
device=CONFIG["device"]
)
logger.info(f"成功加载本地重排模型: {CONFIG['reranker_model_path']}")
# 将加载好的模型设置到全局 Settings 中，确保所有组件统一使用
Settings.embed_model = embed_model
Settings.llm = llm_model
logger.info("--- 已将embed_model和llm配置为全局默认 ---")
logger.info("--- 所有AI模型加载完成 ---")
return llm_model, embed_model, reranker, mllm_for_parsing
def build_knowledge_index(mllm_for_parsing: Ollama, embed_model: HuggingFaceEmbedding):
"""
【最终修正版】构建向量索引并将其持久化到Milvus数据库。
包含手动嵌入生成以绕过库的潜在bug。
"""
logger.info("--- 步骤B: 连接Milvus并构建/加载知识库向量索引 ---")
# ===== 新增：Milvus连接重试机制 =====
max_retries = 5
retry_delay = 5 # 秒
connected = False
for attempt in range(max_retries):
try:
connections.connect(alias="default", host=CONFIG["milvus_host"], port=CONFIG["milvus_port"])
logger.info(f"成功连接到 Milvus 服务 at {CONFIG['milvus_host']}:{CONFIG['milvus_port']}")
connected = True
break
except Exception as e:
logger.warning(f"连接Milvus失败（尝试 {attempt+1}/{max_retries}）: {e}")
if attempt < max_retries - 1:
logger.info(f"{retry_delay}秒后重试...")
time.sleep(retry_delay)
else:
logger.error(f"无法连接到 Milvus 服务，已达到最大重试次数")
raise ConnectionError(f"无法连接Milvus: {str(e)}")
if not connected:
raise RuntimeError("Milvus连接失败")
vector_store = MilvusVectorStore(
uri=f"http://{CONFIG['milvus_host']}:{CONFIG['milvus_port']}",
collection_name=CONFIG["milvus_collection"],
dim=CONFIG["vector_dim"],
overwrite=False
)
collection_exists_and_has_content = False
if utility.has_collection(CONFIG["milvus_collection"]):
collection = Collection(name=CONFIG["milvus_collection"])
collection.load()
if collection.num_entities > 0:
collection_exists_and_has_content = True
if collection_exists_and_has_content:
logger.info(f"在Milvus中已找到包含实体的集合，直接加载索引...")
index = VectorStoreIndex.from_vector_store(vector_store)
logger.info("从Milvus加载索引完成。")
else:
if utility.has_collection(CONFIG["milvus_collection"]):
logger.info("在Milvus中找到空集合，开始处理并填充数据...")
else:
logger.info(f"在Milvus中未找到集合，开始完整的数据处理和索引构建流程...")
# 步骤 1: 数据加载和切分 (此部分不变)
reader = SimpleDirectoryReader(
input_dir=CONFIG["knowledge_base_dir"], required_exts=[".pdf", ".docx", ".txt"],
file_extractor={".pdf": CustomMultimodalReader(mllm_instance=mllm_for_parsing), ".docx": CustomMultimodalReader(mllm_instance=mllm_for_parsing)},
recursive=True
)
documents = reader.load_data(show_progress=True)
all_nodes = []
sentence_splitter = SentenceSplitter(chunk_size=CONFIG["chunk_size"], chunk_overlap=CONFIG["chunk_overlap"])
for doc in documents:
filename = doc.metadata.get("file_name", "").lower()
if doc.metadata.get("content_type") == "image_description": all_nodes.append(doc); continue
if filename.endswith(".pdf"):
article_pattern = r'(第[一二三四五六七八九十百千万零〇\d]+条)'; text_chunks = re.split(article_pattern, doc.text); i = 1
while i < len(text_chunks):
article_title = text_chunks[i]; article_content = text_chunks[i+1] if (i + 1) < len(text_chunks) else ""
full_article_text = (article_title + article_content).strip()
if full_article_text: node = Document(text=full_article_text, extra_info=doc.metadata.copy()); all_nodes.append(node)
i += 2
else: nodes = sentence_splitter.get_nodes_from_documents([doc]); all_nodes.extend(nodes)
logger.info(f"文档条件化切分完毕，共生成 {len(all_nodes)} 个内容块 (Nodes)。")
# --- 【核心修正】 ---
# 步骤 2: 手动、显式地为所有节点生成向量嵌入
logger.info(f"正在为 {len(all_nodes)} 个节点手动生成向量嵌入...")
for node in all_nodes:
# node.get_content() 是获取节点文本最稳健的方法
node.embedding = embed_model.get_text_embedding(node.get_content())
logger.info("所有节点的向量嵌入已手动生成。")
# --- 【核心修正结束】 ---
# 步骤 3: 将已经带有向量的节点添加到Milvus
logger.info(f"正在将 {len(all_nodes)} 个带有预生成向量的节点添加到Milvus...")
vector_store.add(all_nodes)
logger.info("节点已成功添加到Milvus。")
# 步骤 4: 从已填充的向量存储创建索引对象
index = VectorStoreIndex.from_vector_store(vector_store)
logger.info("索引对象创建完成。")
connections.disconnect("default")
logger.info("已断开与 Milvus 服务的连接。")
return index
def run_query_pipeline(index: VectorStoreIndex, llm_model: Ollama, reranker: SentenceTransformerRerank):
"""启动问答流程，循环处理预设的问题。"""
logger.info("--- 步骤C: 开始RAG问答流程 ---")
QA_PROMPT_TEMPLATE = PromptTemplate(
"你是一个专业的业务问答助手，负责根据内部知识库提供精准、可靠的回答。\n\n"
"**你的任务是：**\n"
"1. 仔细阅读下面提供的“参考信息”。\n"
"2. 根据“参考信息”直接回答“用户问题”，禁止进行任何形式的猜测、推理或使用你自己的知识。\n"
"3. **引用来源**：在回答中，如果引用了某份文件的内容，必须在相关句子末尾用 `(文件名)` 的格式注明来源。\n"
"4. **版本对比**：如果参考信息来自不同版本的文件（例如，文件名中包含年份），请对比说明它们之间的差异。\n"
"5. **提供建议**：在回答的最后，根据回答内容，提供1-2条具体、可执行的业务建议。\n"
"6. **未知问题**：如果“参考信息”中完全没有能回答问题的内容，你必须且只能回答：“根据提供的资料，无法回答该问题。”\n"
"7. **格式要求**：回答的最后，必须附上一个“参考依据”列表，列出所有被引用的文件名。\n\n"
"---------------------\n"
"**参考信息:**\n{context_str}\n"
"---------------------\n"
"**用户问题:** {query_str}\n\n"
"**你的回答:**\n"
)
query_engine = index.as_query_engine(
similarity_top_k=CONFIG["retrieval_top_k"],
node_postprocessors=[reranker],
text_qa_template=QA_PROMPT_TEMPLATE
# llm会从全局Settings获取
)
questions = [
"根据附图1的技术架构图，究竟是哪个芯片独立负责生成刷新信号？",
"数据出境安全评估申报流程图里，如果个人信息达到10万人规模该怎么办？",
"我国公民的基本权力以及义务有哪些",
"借钱不还怎么办？"
]
for q in questions:
logger.info(f"\n{'='*70}\n--- 用户提问: {q} ---")
try:
response = query_engine.query(q)
logger.info("\n--- 模型最终回答 ---\n" + str(response))
logger.info("\n--- 回答引用的参考信息 (经重排后) ---")
for i, node_with_score in enumerate(response.source_nodes):
logger.info(f"--- 来源 {i+1} (得分: {node_with_score.score:.4f}, 文件: {node_with_score.metadata.get('file_name', 'N/A')}) ---")
node = node_with_score.node
if hasattr(node, 'text') and node.text:
logger.info(f"内容预览: {node.text[:150]}...\n")
else:
logger.info(f"内容预览: [这是一个非文本节点，类型为: {type(node).__name__}]\n")
except Exception as e:
logger.error(f"执行RAG查询时发生错误: {e}", exc_info=True)
logger.info(f"{'='*70}")
# =======================================================
# 3. 主执行入口
# =======================================================
if __name__ == "__main__":
logger.info("===== 启动业务领域多模态RAG智能问答系统 =====")
try:
llm, embed_model, reranker, mllm = setup_models_and_services()
knowledge_index = build_knowledge_index(mllm, embed_model)
run_query_pipeline(knowledge_index, llm, reranker)
except Exception as e:
logger.error(f"程序主流程发生致命错误，即将退出: {e}", exc_info=True)
exit(1)
logger.info("\n===== RAG系统执行完成 =====")
这是我的代码，(rag_project_env) PS D:\new_rag> & D:/miniconda/envs/rag_project_env/python.exe d:/new_rag/law_rag.py
2025-07-24 11:26:07,150 - INFO - ===== 启动业务领域多模态RAG智能问答系统 =====
2025-07-24 11:26:07,150 - INFO - --- 步骤A: 加载所有本地AI模型 ---
2025-07-24 11:26:07,153 - INFO - Load pretrained SentenceTransformer: D:/models/text2vec-base-chinese
2025-07-24 11:26:08,791 - INFO - 成功加载本地嵌入模型: D:/models/text2vec-base-chinese
2025-07-24 11:26:08,791 - INFO - 成功配置本地Ollama LLM (模型: qwen3:8b)
2025-07-24 11:26:08,791 - INFO - 成功配置本地Ollama MLLM (模型: llava)
2025-07-24 11:26:09,586 - INFO - 成功加载本地重排模型: D:/models/bge-reranker-v2-m3
2025-07-24 11:26:09,586 - INFO - --- 已将embed_model和llm配置为全局默认 ---
2025-07-24 11:26:09,586 - INFO - --- 所有AI模型加载完成 ---
2025-07-24 11:26:09,586 - INFO - --- 步骤B: 连接Milvus并构建/加载知识库向量索引 ---
2025-07-24 11:26:19,712 - WARNING - 连接Milvus失败（尝试 1/5）: <MilvusException: (code=2, message=Fail connecting to server on 127.0.0.1:19530, illegal connection params or server unavailable)>
2025-07-24 11:26:19,712 - INFO - 5秒后重试...
2025-07-24 11:26:34,849 - WARNING - 连接Milvus失败（尝试 2/5）: <MilvusException: (code=2, message=Fail connecting to server on 127.0.0.1:19530, illegal connection params or server unavailable)>
2025-07-24 11:26:34,849 - INFO - 5秒后重试...
2025-07-24 11:26:49,974 - WARNING - 连接Milvus失败（尝试 3/5）: <MilvusException: (code=2, message=Fail connecting to server on 127.0.0.1:19530, illegal connection params or server unavailable)>
2025-07-24 11:26:49,978 - INFO - 5秒后重试...
2025-07-24 11:27:05,102 - WARNING - 连接Milvus失败（尝试 4/5）: <MilvusException: (code=2, message=Fail connecting to server on 127.0.0.1:19530, illegal connection params or server unavailable)>
2025-07-24 11:27:05,102 - INFO - 5秒后重试...
2025-07-24 11:27:20,192 - WARNING - 连接Milvus失败（尝试 5/5）: <MilvusException: (code=2, message=Fail connecting to server on 127.0.0.1:19530, illegal connection params or server unavailable)>
2025-07-24 11:27:20,192 - ERROR - 无法连接到 Milvus 服务，已达到最大重试次数
2025-07-24 11:27:20,192 - ERROR - 程序主流程发生致命错误，即将退出: 无法连接Milvus: <MilvusException: (code=2, message=Fail connecting to server on 127.0.0.1:19530, illegal connection params or server unavailable)>
Traceback (most recent call last):
File "d:\new_rag\law_rag.py", line 230, in build_knowledge_index
connections.connect(alias="default", host=CONFIG["milvus_host"], port=CONFIG["milvus_port"])
File "D:\miniconda\envs\rag_project_env\lib\site-packages\pymilvus\orm\connections.py", line 459, in connect
connect_milvus(**kwargs, user=user, password=password, token=token, db_name=db_name)
File "D:\miniconda\envs\rag_project_env\lib\site-packages\pymilvus\orm\connections.py", line 420, in connect_milvus
raise e from e
File "D:\miniconda\envs\rag_project_env\lib\site-packages\pymilvus\orm\connections.py", line 412, in connect_milvus
gh._wait_for_channel_ready(timeout=timeout)
File "D:\miniconda\envs\rag_project_env\lib\site-packages\pymilvus\client\grpc_handler.py", line 159, in _wait_for_channel_ready
raise MilvusException(
pymilvus.exceptions.MilvusException: <MilvusException: (code=2, message=Fail connecting to server on 127.0.0.1:19530, illegal connection params or server unavailable)>
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "d:\new_rag\law_rag.py", line 374, in <module>
knowledge_index = build_knowledge_index(mllm, embed_model)
File "d:\new_rag\law_rag.py", line 241, in build_knowledge_index
raise ConnectionError(f"无法连接Milvus: {str(e)}")
ConnectionError: 无法连接Milvus: <MilvusException: (code=2, message=Fail connecting to server on 127.0.0.1:19530, illegal connection params or server unavailable)>这是我的报错，services:
etcd:
container_name: milvus-etcd
image: quay.io/coreos/etcd:v3.5.5
environment:
- ETCD_AUTO_COMPACTION_MODE=revision
- ETCD_AUTO_COMPACTION_RETENTION=1000
- ETCD_QUOTA_BACKEND_BYTES=4294967296
- ETCD_SNAPSHOT_COUNT=50000
volumes:
- ./volumes/etcd:/etcd
command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
minio:
container_name: milvus-minio
image: quay.io/minio/minio:RELEASE.2023-03-20T20-16-18Z
environment:
- MINIO_ROOT_USER=minioadmin
- MINIO_ROOT_PASSWORD=minioadmin
volumes:
- ./volumes/minio:/minio_data
command: minio server /minio_data
standalone:
container_name: milvus-standalone
image: milvusdb/milvus:v2.5.0 # <-- 修正点在这里
command: ["milvus", "run", "standalone"]
environment:
- ETCD_ENDPOINTS=etcd:2379
- MINIO_ADDRESS=minio:9000
volumes:
- ./volumes/milvus:/var/lib/milvus
ports:
- "19530:19530" # Milvus port
- "9091:9091" # Milvus metrics port
depends_on:
- "etcd"
- "minio"
volumes:
etcd:
minio:
milvus:这是我的docker-compose.yml，请帮我解决我的报错，或者更改我的代码让我避开这个问题
扫一扫
3168
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
