python正向最大匹配分词和逆向最大匹配分词

最新推荐文章于 2024-08-19 03:14:04 发布

Yan456jie

最新推荐文章于 2024-08-19 03:14:04 发布

阅读量3k

点赞数 1

CC 4.0 BY-SA版权

分类专栏： python NLP

本文链接：https://blog.youkuaiyun.com/Yan456jie/article/details/78788783

python 同时被 2 个专栏收录

73 篇文章

订阅专栏

NLP

34 篇文章

订阅专栏

本文介绍了一种基于词典的文本分词方法，包括正向最大匹配和逆向最大匹配两种算法，并提供了详细的实现流程及示例代码。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

正向最大匹配

# -*- coding:utf-8 -*-

CODEC='utf-8'

def u(s, encoding):
    'converted other encoding to unicode encoding'
    if isinstance(s, unicode):
        return s
    else:
        return unicode(s, encoding)

def fwd_mm_seg(wordDict, maxLen, str):
    'forward max match segment'
    wordList = []
    segStr = str
    segStrLen = len(segStr)
    for word in wordDict:
        print 'word: ', word
    print "\n"
    while segStrLen > 0:
        if segStrLen > maxLen:
            wordLen = maxLen
        else:
            wordLen = segStrLen
        subStr = segStr[0:wordLen]
        print "subStr: ", subStr
        while wordLen > 1:
            if subStr in wordDict:
                print "subStr1: %r" % subStr
                break
            else:
                print "subStr2: %r" % subStr
                wordLen = wordLen - 1
                subStr = subStr[0:wordLen]
#            print "subStr3: ", subStr
        wordList.append(subStr)
        segStr = segStr[wordLen:]
        segStrLen = segStrLen - wordLen
    for wordstr in wordList:
        print "wordstr: ", wordstr
    return wordList
        
            
def main():
    fp_dict = open('words.dic')
    wordDict = {}
    for eachWord in fp_dict:
        wordDict[u(eachWord.strip(), 'utf-8')] = 1
    segStr = u'你好世界hello world'
    print segStr
    wordList = fwd_mm_seg(wordDict, 10, segStr)
    print "==".join(wordList)
    

if __name__ == '__main__':
    main()

逆向最大匹配

# -*- coding:utf-8 -*-


def u(s, encoding):
    'converted other encoding to unicode encoding'
    if isinstance(s, unicode):
        return s
    else:
        return unicode(s, encoding)

CODEC='utf-8'

def bwd_mm_seg(wordDict, maxLen, str):
    'forward max match segment'
    wordList = []
    segStr = str
    segStrLen = len(segStr)
    for word in wordDict:
        print 'word: ', word
    print "\n"
    while segStrLen > 0:
        if segStrLen > maxLen:
            wordLen = maxLen
        else:
            wordLen = segStrLen
        subStr = segStr[-wordLen:None]
        print "subStr: ", subStr
        while wordLen > 1:
            if subStr in wordDict:
                print "subStr1: %r" % subStr
                break
            else:
                print "subStr2: %r" % subStr
                wordLen = wordLen - 1
                subStr = subStr[-wordLen:None]
#            print "subStr3: ", subStr
        wordList.append(subStr)
        segStr = segStr[0: -wordLen]
        segStrLen = segStrLen - wordLen
    wordList.reverse()
    for wordstr in wordList:
        print "wordstr: ", wordstr
    return wordList
        
            
def main():
    fp_dict = open('words.dic')
    wordDict = {}
    for eachWord in fp_dict:
        wordDict[u(eachWord.strip(), 'utf-8')] = 1
    segStr = ur'你好世界hello world'
    print segStr
    wordList = bwd_mm_seg(wordDict, 10, segStr)
    print "==".join(wordList)

if __name__ == '__main__':
    main()