Python从同一文件进行数据不落地的取高频处理

最新推荐文章于 2025-07-08 10:53:55 发布

原创最新推荐文章于 2025-07-08 10:53:55 发布 · 1k 阅读

0 ·

CC 4.0 BY-SA版权

python_method 专栏收录该内容

31 篇文章

订阅专栏

该博客介绍如何使用Python从文件中高效地提取多音字的高频词汇，特别是那些在文本中出现频率较高的词汇，如'的'、'是'等，实现了数据不落地的处理方式。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

数据格式：（词，拼音，词频）

的 de 148709248
的 di 1193135
了 le 62873377
了 liao 3199200
是 shi 62432861
一 yi 58994539
不 bu 57479625
不 fou 1136895

将文件中多音字的高频词汇提取并返回

def chose_high_freq_word():
    '''key:word,value:[pinyin](high frequence)'''
    filepath = os.path.dirname(os.path.abspath(__file__))
    filename = os.path.join(filepath,"dict.txt")
    if not os.path.isfile(filename):
        raise ValueError("No such file:{}".format(filename))
    all_word_pinyin_freq_dic = {}
    multi_word_pinyin_freq_dic = {}
    with codecs.open(filename,encoding="utf-8") as f:
        for line in f.readlines():
            if line.startswith(";"):
                pass
            else:
                splited_line = line.split("\t")
                if len(splited_line) is not 3:
                    raise ValueError("splited lenth is not 3,in file HZout_NoTone.txt")
                word = splited_line[0]
                pinyin = splited_line[1]
                freq = splited_line[2].strip()
                check_multi = all_word_pinyin_freq_dic.get(word)
                if check_multi is not None:
                    check_multi.append([pinyin,freq])
                    multi_word_pinyin_freq_dic[word] = check_multi
                    all_word_pinyin_freq_dic[word] = check_multi
                else:
                    all_word_pinyin_freq_dic[word] = [[pinyin,freq]]
    # print len(multi_word_pinyin_freq_dic)
    pattern = re.compile(r"\d")
    for word in multi_word_pinyin_freq_dic:
        # print word,multi_word_pinyin_freq_dic[word]
        multi_word_pinyin_freq_dic[word] = max(multi_word_pinyin_freq_dic[word],key=lambda x:int(x[1]))
        multi_word_pinyin_freq_dic[word] = [pattern.sub("",multi_word_pinyin_freq_dic[word][0])]
    # for word in multi_word_pinyin_freq_dic:
    #     print word,multi_word_pinyin_freq_dic[word]
    return multi_word_pinyin_freq_dic