基于Transformer对文本进行向量搜索_sentence-transformer实现文本智能检索-优快云博客

本文链接：https://blog.youkuaiyun.com/lxb_champagne/article/details/122302907

本文探讨了如何利用Transformer模型将文本转化为向量，进而实现高效精准的搜索功能。通过深度学习技术，Transformer能够捕获文本的复杂语义信息，提升文本检索的准确性。在实际应用中，这种方法对于大量文本数据的处理具有重要价值。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

import csv
from datetime import datetime

import numpy as np
import pandas as pd

import os
from pathlib import Path

from sentence_transformers import SentenceTransformer, CrossEncoder, util
from concurrent.futures import ThreadPoolExecutor, wait
import multiprocessing

# Query data from QE platform


def concurrency_encode_text(text_arr):
    record_count = text_arr.size
    task_count = int(record_count / thread_num) + 1

    future_tasks = []
    for index in range(thread_num):
        first_index = task_count * index
        second_index = task_count * (index + 1)
        if second_index > record_count:
            second_index = record_count
        task_text_arr = text_arr[first_index:second_index].copy()
        future_tasks.append(thread_pool_executor.submit(encode_text, task_text_arr))
        print("sub task starr, batch {}, start {}, end {}".format(index, first_index, second_index))

    wait(future_tasks)

    encoded_embedding = np.concatenate(([task.result() for task in future_tasks]))
    return encoded_embedding

def encode_text(text_arr):
    print("encode text array size {} records".format(text_arr.size))
    text_embedding = np.empty((0,512), dtype="float32")

    batch_size = 1000
    batch_count = int(text_arr.size / batch_size) + 1

    for batch_index in range(batch_count):
        first_index = batch_size * batch_index
        second_index = batch_size * (batch_index + 1)
        if second_index > text_arr.size:
            second_index = text_arr.size

        bat