2025-05-20 模型下载--文本向量化--Faiss检索

模型下载

• 使用Python脚本进行下载

from huggingface_hub import snapshot_download
# import os

# os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"
# 自定义下载目录（Windows 路径建议用 raw string 或 pathlib）
download_dir = r"../model/V2/jina-embeddings-v3"

local_dir = snapshot_download(
    repo_id="jinaai/jina-embeddings-v3",
    local_dir=download_dir,
    local_dir_use_symlinks=False  # 避免 Windows 下软链接问题
)

print("模型保存路径：", local_dir)

• 当使用跑数据集的时候，还是会出现错误，找不到各种文件，原因是依赖了其他的文件
• 地址 https://huggingface.co/jinaai/xlm-roberta-flash-implementation/tree/main

文本向量化

import pandas as pd
from sentence_transformers import SentenceTransformer
import os

# =========================
# 📍 Step 1: 加载数据
# =========================
input_datapath = './data/fine_food_reviews_1k.csv'

df = pd.read_csv(input_datapath, index_col=0)
df = df[["Time", "ProductId", "UserId", "Score", "Summary", "Text"]]
df = df.dropna()

# 合并文本字段
df["combined"] = (
    "Title: " + df.Summary.str.strip() + "; Content: " + df.Text.str.strip()
)

# =========================
# 📍 Step 2: 加载本地模型
# =========================
model_path = 'model/jina-embeddings-v3/models--jinaai--jina-embeddings-v3/snapshots/f1944de8402dcd5f2b03f822a4bc22a7f2de2eb9'
model = SentenceTransformer(model_path, trust_remote_code=True)

# =========================
# 📍 Step 3: 批量编码文本
# =========================
texts = df["combined"].tolist()

# tqdm 显示进度条，convert_to_numpy 提供 array 输出
embeddings = model.encode(texts, show_progress_bar=True, convert_to_numpy=True)

# =========================
# 📍 Step 4: 存储向量数据
# =========================
df["embedding"] = embeddings.tolist()

# 保存为 parquet（高效）或 csv
output_path = './output/fine_food_reviews_with_embeddings.parquet'
os.makedirs(os.path.dirname(output_path), exist_ok=True)
df.to_parquet(output_path, index=False)

print(f"✅ 向量化完成，共生成 {len(df)} 条记录，已保存至：{output_path}")

Faiss检索

import pandas as pd
import numpy as np
import faiss
from sentence_transformers import SentenceTransformer

# ========== 1. 加载数据 ==========
df = pd.read_parquet('./output/fine_food_reviews_with_embeddings.parquet')
embeddings = np.array(df['embedding'].to_list())  # 转成 numpy array

# ========== 2. 加载模型 ==========
model_path = 'model/jina-embeddings-v3/models--jinaai--jina-embeddings-v3/snapshots/f1944de8402dcd5f2b03f822a4bc22a7f2de2eb9'
model = SentenceTransformer(model_path, trust_remote_code=True)

# ========== 3. 构建向量索引 ==========
index = faiss.IndexFlatL2(embeddings.shape[1])
index.add(embeddings)

# ========== 4. 查询示例 ==========
query = "蜂蜜"
query_vec = model.encode([query], convert_to_numpy=True)

# 查询 TopK 相似文本
D, I = index.search(query_vec, k=3)

print("🔍 查询结果：")
for idx in I[0]:
    print(f"\n📌 Score: {df.iloc[idx]['Score']}")
    print(f"📄 Text: {df.iloc[idx]['combined']}")