文本数据增强-回译尝试

最新推荐文章于 2025-03-29 15:45:40 发布

jacky_wxl(微信同号）

最新推荐文章于 2025-03-29 15:45:40 发布

阅读量1.2k

点赞数

CC 4.0 BY-SA版权

分类专栏：深度学习文章标签：自然语言处理语言模型 transformer

本文链接：https://blog.youkuaiyun.com/wxl781227/article/details/121241889

深度学习专栏收录该内容

6 篇文章

订阅专栏

该博客介绍了两种AI翻译方法：一是通过调用百度翻译API实现中英文互译；二是利用transformers库中的mbart-large-50模型进行多语言翻译，包括中文到英文和英文到中文的转换。这两种方法都是基于现代机器学习技术，为跨语言沟通提供了便利。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

方法一：使用百度翻译api

import requests
import random
import json
from hashlib import md5

def cntoen(query):
    # Set your own appid/appkey.
    appid = 'x'
    appkey = 'x'

    # For list of language codes, please refer to `https://api.fanyi.baidu.com/doc/21`
    from_lang = 'zh'
    to_lang =  'en'

    endpoint = 'http://api.fanyi.baidu.com'
    path = '/api/trans/vip/translate'
    url = endpoint + path

    # Generate salt and sign
    def make_md5(s, encoding='utf-8'):
        return md5(s.encode(encoding)).hexdigest()

    salt = random.randint(32768, 65536)
    sign = make_md5(appid + query + str(salt) + appkey)

    # Build request
    headers = {'Content-Type': 'application/x-www-form-urlencoded'}
    payload = {'appid': appid, 'q': query, 'from': from_lang, 'to': to_lang, 'salt': salt, 'sign': sign}

    # Send request
    r = requests.post(url, params=payload, headers=headers)
    result = r.json()

    # Show response
    print(json.dumps(result, indent=4, ensure_ascii=False))

    query=result.get("trans_result")[0].get("dst")

    return query

def entocn(query):
    # Set your own appid/appkey.
    appid = 'x'
    appkey = 'x'

    # For list of language codes, please refer to `https://api.fanyi.baidu.com/doc/21`
    from_lang = 'en'
    to_lang =  'zh'

    endpoint = 'http://api.fanyi.baidu.com'
    path = '/api/trans/vip/translate'
    url = endpoint + path

    # Generate salt and sign
    def make_md5(s, encoding='utf-8'):
        return md5(s.encode(encoding)).hexdigest()

    salt = random.randint(32768, 65536)
    sign = make_md5(appid + query + str(salt) + appkey)

    # Build request
    headers = {'Content-Type': 'application/x-www-form-urlencoded'}
    payload = {'appid': appid, 'q': query, 'from': from_lang, 'to': to_lang, 'salt': salt, 'sign': sign}

    # Send request
    r = requests.post(url, params=payload, headers=headers)
    result = r.json()

    # Show response
    print(json.dumps(result, indent=4, ensure_ascii=False))

    query=result.get("trans_result")[0].get("dst")

    return query

方法二：使用facebook的mbart-large-50-many-to-many-mmt

from transformers import MBartForConditionalGeneration, MBart50TokenizerFast
#其它语言 ar_AR,cs_CZ,de_DE,en_XX,es_XX,et_EE,fi_FI,fr_XX,gu_IN,hi_IN,it_IT,ja_XX,kk_KZ,ko_KR,lt_LT,lv_LV,my_MM,ne_NP,nl_XX,ro_RO,ru_RU,si_LK,tr_TR,vi_VN,zh_CN

model = MBartForConditionalGeneration.from_pretrained("facebook/mbart-large-50-many-to-many-mmt")
tokenizer = MBart50TokenizerFast.from_pretrained("facebook/mbart-large-50-many-to-many-mmt")


def cn2en(article_cn):
    tokenizer.src_lang = "zh_CN"
    encoded_hi = tokenizer(article_cn, return_tensors="pt")
    generated_tokens = model.generate(**encoded_hi, forced_bos_token_id=tokenizer.lang_code_to_id["en_XX"])
    result = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)
    print(f"{article_cn}的翻译结果: {result[0]}")
    return result[0]

def en2cn(article_en):
    tokenizer.src_lang = "en_XX"
    encoded_hi = tokenizer(article_en, return_tensors="pt")
    generated_tokens = model.generate(**encoded_hi, forced_bos_token_id=tokenizer.lang_code_to_id["zh_CN"])
    result = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)
    print(f"{article_en}的翻译结果: {result[0]}")
    return result[0]