【深度学习】sentencepiece工具之BPE训练使用

最新推荐文章于 2025-07-07 23:48:20 发布

UESTC_20172222

最新推荐文章于 2025-07-07 23:48:20 发布

阅读量6.1k

点赞数

CC 4.0 BY-SA版权

分类专栏：深度学习（NLP领域）文章标签： sentencepiece bpe 训练使用

本文链接：https://blog.youkuaiyun.com/qq_32806793/article/details/90904146

深度学习（NLP领域）专栏收录该内容

19 篇文章

订阅专栏

本文介绍BPE（Byte Pair Encoding）在机器翻译领域的应用，详细解释了BPE的工作原理，即通过迭代合并字符串中出现频率最高的子串来简化文本。文章提供了使用sentencepiece库进行BPE模型训练的代码示例，包括数据预处理、模型训练和数据分割的具体步骤。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

为什么要使用BPE,BPE是什么

BPE：迭代的将字符串里出现频率最高的子串进行合并
训练过程
在这里插入图片描述

使用教程

代码使用的语料在这里

# -*- coding: utf-8 -*-
#/usr/bin/python3


import os
import errno
import sentencepiece as spm
import re
import logging

logging.basicConfig(level=logging.INFO)

def prepro(hp):
    print("# Check if raw files exist")
    train1 = "iwslt2016/de-en/train.tags.de-en.de"
    train2 = "iwslt2016/de-en/train.tags.de-en.en"
    eval1 = "iwslt2016/de-en/IWSLT16.TED.tst2013.de-en.de.xml"
    eval2 = "iwslt2016/de-en/IWSLT16.TED.tst2013.de-en.en.xml"
    test1 = "iwslt2016/de-en/IWSLT16.TED.tst2014.de-en.de.xml"
    test2 = "iwslt2016/de-en/IWSLT16.TED.tst2014.de-en.en.xml"
    for f in (train1, train2, eval1, eval2, test1, test2):
        if not os.path.isfile(f):
            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), f)

    print("# Preprocessing")
    # train
    _prepro = lambda x:  [line.strip() for line in open(x, mode='r',encoding="utf-8").read().split("\n") \
                      if not line.startswith("<")]
    prepro_train1, prepro_train2 = _prepro(train1), _prepro(train2)
    assert len(prepro_train1)==len(prepro_train2), "Check if train source and target files match."

    # eval
    _prepro = lambda x: [re.sub("<[^>]+>", "", line).strip() \
                     for line in open(x, mode='r',encoding="utf-8").read().split("\n") \
                     if line.startswith("<seg id")]
    prepro_eval1, prepro_eval2 = _prepro(eval1), _prepro(eval2)
    assert len(prepro_eval1) == len(prepro_eval2), "Check if eval source and target files match."

    # test
    prepro_test1, prepro_test2 = _prepro(test1), _prepro(test2)
    assert len(prepro_test1) == len(prepro_test2), "Check if test source and target files match."

    print("Let's see how preprocessed data look like")
    print("prepro_train1:", prepro_train1[0])
    print("prepro_train2:", prepro_train2[0])
    print("prepro_eval1:", prepro_eval1[0])
    print("prepro_eval2:", prepro_eval2[0])
    print("prepro_test1:", prepro_test1[0])
    print("prepro_test2:", prepro_test2[0])

    print("# write preprocessed files to disk")
    os.makedirs("iwslt2016/prepro", exist_ok=True)
    def _write(sents, fname):
        with open(fname, mode='w',encoding="utf-8") as fout:
            fout.write("\n".join(sents))

    _write(prepro_train1, "iwslt2016/prepro/train.de")
    _write(prepro_train2, "iwslt2016/prepro/train.en")
    _write(prepro_train1+prepro_train2, "iwslt2016/prepro/train")
    _write(prepro_eval1, "iwslt2016/prepro/eval.de")
    _write(prepro_eval2, "iwslt2016/prepro/eval.en")
    _write(prepro_test1, "iwslt2016/prepro/test.de")
    _write(prepro_test2, "iwslt2016/prepro/test.en")

    print("# Train a joint BPE model with sentencepiece")
    os.makedirs("iwslt2016/segmented", exist_ok=True)
    train = '--input=iwslt2016/prepro/train --pad_id=0 --unk_id=1 \
             --bos_id=2 --eos_id=3\
             --model_prefix=iwslt2016/segmented/bpe --vocab_size={} \
             --model_type=bpe'.format(hp.vocab_size)
    spm.SentencePieceTrainer.Train(train)

    print("# Load trained bpe model")
    sp = spm.SentencePieceProcessor()
    sp.Load("iwslt2016/segmented/bpe.model")

    print("# Segment")
    def _segment_and_write(sents, fname):
        with open(fname,mode= "w",encoding="utf-8") as fout:
            for sent in sents:
                pieces = sp.EncodeAsPieces(sent)
                fout.write(" ".join(pieces) + "\n")

    _segment_and_write(prepro_train1, "iwslt2016/segmented/train.de.bpe")
    _segment_and_write(prepro_train2, "iwslt2016/segmented/train.en.bpe")
    _segment_and_write(prepro_eval1, "iwslt2016/segmented/eval.de.bpe")
    _segment_and_write(prepro_eval2, "iwslt2016/segmented/eval.en.bpe")
    _segment_and_write(prepro_test1, "iwslt2016/segmented/test.de.bpe")

    print("Let's see how segmented data look like")
    print("train1:", open("iwslt2016/segmented/train.de.bpe",mode='r',encoding="utf-8").readline())
    print("train2:", open("iwslt2016/segmented/train.en.bpe", mode='r',encoding="utf-8").readline())
    print("eval1:", open("iwslt2016/segmented/eval.de.bpe", mode='r',encoding="utf-8").readline())
    print("eval2:", open("iwslt2016/segmented/eval.en.bpe", mode='r',encoding="utf-8").readline())
    print("test1:", open("iwslt2016/segmented/test.de.bpe", mode='r',encoding="utf-8").readline())

if __name__ == '__main__':
    hparams = Hparams()
    parser = hparams.parser
    hp = parser.parse_args()
    prepro(hp)
    print("Done")