list(map(tokenizer.tokenize, text))

最新推荐文章于 2025-04-09 10:42:30 发布

原创最新推荐文章于 2025-04-09 10:42:30 发布 · 377 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#前端 #大数据

论文复现记录专栏收录该内容

34 篇文章

订阅专栏

本文介绍如何使用tokenizer进行text2tokens处理，并通过实例演示如何将实体列表转化为token_spans，重点在于subtokenizer与raw_text的对比与实际应用中的复原技巧。

这个用法比较重要，可以做subtokenizer和raw_text的对比和复原

text2tokens = self.tokenizer.tokenize(text, add_special_tokens=self.add_special_tokens)
text_ = text.split(' ')
subwords = list(map(tokenizer.tokenize, text_))

class Preprocessor(object):
def init(self, tokenizer):
super(Preprocessor, self).init()
self.tokenizer = tokenizer
self.add_special_tokens = True

def get_ent2token_spans(self, text, entity_list):
    """实体列表转为token_spans

    Args:
        text (str): 原始文本
        entity_list (list): [(start, end, ent_type),(start, end, ent_type)...]
    """
    ent2token_spans = []

    inputs = self.tokenizer(text, add_special_tokens=True, return_offsets_mapping=True)
    token2char_span_mapping = inputs["offset_mapping"]
    text2tokens = self.tokenizer.tokenize(text, add_special_tokens=self.add_special_tokens)
    text_ = text.split(' ')
    subwords = list(map(tokenizer.tokenize, text_))
    toks, index = get_index(text2tokens)
    for en_span in entity_list:
        if en_span[0]!=0:
            subh = sum([len(i) for i in subwords[:(en_span[0] )]])
            subt = sum([len(i) for i in subwords[:(en_span[0]+1)]])
        else:
            subh = sum([len(i) for i in subwords[:(en_span[0] )]])
            subt = sum([len(i) for i in subwords[:(en_span[0]+1)]])
        if en_span[1]!=0:
            objh = sum([len(i) for i in subwords[:(en_span[1])]])
            objt = sum([len(i) for i in subwords[:(en_span[1]+1)]])
        else:
            objh = sum([len(i) for i in subwords[:(en_span[1] )]])
            objt = sum([len(i) for i in subwords[:(en_span[1]+1)]])

        start_index = (subh + 1, subt + 1)
        end_index = (objh + 1, objt + 1)

        token_span = (start_index, end_index, en_span[2])
        ent2token_spans.append(token_span)

    return ent2token_spans