中文维基百科的语料库下载和繁体转换为简体的操作

中文维基百科的语料库下载和繁体转换为简体的操作

• 维基百科数据下载源https://dumps.wikimedia.org/backup-index.html】

# from gensim.corpora.wikicorpus import extract_pages,filter_wiki
# import bz2file
# import sys
# import re
# import opencc   # 中文繁转简
# from tqdm import tqdm
# import codecs
import time
from gensim.corpora.wikicorpus import extract_pages,filter_wiki
import bz2file
import re
import opencc
from tqdm import tqdm
import codecs


wiki = extract_pages(bz2file.open('zhwiki-20220820-pages-articles.xml.bz2'))
print("读取压缩包成功")
# cc=opencc('t2s')
def wiki_replace(d,converter):
    s = d[1]
    s = re.sub(':*{\|[\s\S]*?\|}', '', s)
    s = re.sub('<gallery>[\s\S]*?</gallery>', '', s)
    s = re.sub('(.){{([^{}\n]*?\|[^{}\n]*?)}}', '\\1[[\\2]]', s)
    s = filter_wiki(s)
    s = re.sub('\* *\n|\'{2,}', '', s)
    s = re.sub('\n+', '\n', s)
    s = re.sub('\n[:;]|\n +', '\n', s)
    s = re.sub('\n==', '\n\n==', s)
    s = u'【' + d[0] + u'】\n' + s
    return converter.convert(s).strip()
if __name__ == '__main__':
    i = 0
    f = codecs.open('zhwiki.txt', 'w', encoding='utf-8')
    w = tqdm(wiki, desc=u'已获取0篇文章')
    converter=opencc.OpenCC('t2s.json')
    for dd in w:
        if not re.findall('^[a-zA-Z]+:', dd[0]) and dd[0] and not re.findall(u'^#', dd[1]):
            # s = wiki_replace(d)
            s=wiki_replace(dd,converter)
            f.write(s+'\n\n\n')
            i += 1
            w.set_description(u'已获取%s篇文章' % i)
            # if i % 100 == 0:
            #     w.set_description(u'已获取%s篇文章'%i)
    print(i)
    time.sleep(5);
    f.close()