【中文分词】 FMM & BMM （python）

1. 语料库准备

     （需要的私聊博主）

2.处理语料文档

2.1处理多余字符

def string_process(x):    #处理字符串
    a=re.sub(r'\d{8}-\d{2}-\d{3}-\d{3}/m|[/a-z！。”“，、——\[\]（）：《》……A-Z？]', "", x)
#前面时间的正则表达式,后面部分删掉字母和其他符号
    b=a.replace("  "," ")
    return b.rstrip()

2.2 处理字典数据集

def file_process():
    s=""
    with open('199801-train.txt','r',encoding='UTF-8') as f:
        for line1 in f:
            a=string_process(line1)
            a=string_process(a)
            s+=a+"\n"
    f.close()
    with open("process_1.txt",'w',encoding='UTF-8') as w:
        w.write(s)

    w.close()

2.3处理测试数据集

def test_file_process(): 
    s = ""
    with open('199801-test.txt', 'r', encoding='UTF-8') as f:
        for line1 in f:
            a = string_process(line1)
            a = string_process(a)
            s += a + "\n"
    f.close()
    with open("test_process_1.txt", 'w', encoding='UTF-8') as w:
        w.write(s)

    w.close()

2.4处理测试集空格

def delete_space():
    f = open('test_process_1.txt','r', encoding='utf-8')
    f2 = open('test_endfile.txt', 'w', encoding='utf-8')
    s = f.read()
    r = ''.join(s.split())
    f2.write(r)

3.FMM

training_words = 'result.txt'

def get_dic(training_words):
    with open(training_words,'r',encoding='utf-8') as f:
        try:
            file_content = f.read().split()
        finally:
            f.close()
    chars = list(set(file_content))
    return chars

dic = get_dic(training_words)


def readfile():
    max_length=0
    for i in dic:
        max_length=max(max_length,len(i))##获得最大长度
    zz=max_length
    f=open("test_endfile.txt",'r',encoding='utf-8')
    ff=open("生成文档正向最大匹配(真).txt",'w',encoding='utf-8')
    lines=f.readlines()
    f.close()
    for line in lines:#分别对每一行进行正向最大匹配处理
        max_length=zz
        my_list=[]
        len_hang=len(line)
        while len_hang>0:
            tryW=line[0:max_length]##切割字符串
            while tryW not in dic:
                if len(tryW)==1:#长度为1的时候就直接退出
                    break;
                tryW=tryW[0:len(tryW)-1]
            my_list.append(tryW)
            line=line[len(tryW):]
            len_hang=len(line)
        for i in my_list:
            ff.write(i+"/")
    ff.close()
    print("complete !!!")
readfile()

4.BMM

training_words = 'result.txt'

def get_dic(training_words):
    with open(training_words,'r',encoding='utf-8') as f:
        try:
            file_content = f.read().split()
        finally:
            f.close()
    chars = list(set(file_content))
    return chars


dic = get_dic(training_words)


def readfile():
    max_length=0
    for i in dic:
        max_length=max(max_length,len(i))##获得最大长度
    zz=max_length
    f=open("测试.txt",'r',encoding='utf-8')
    ff=open("生成文档逆向最大匹配.txt",'w',encoding='utf-8')
    lines=f.readlines()
    f.close()
    for line in lines:#分别对每一行进行逆向最大匹配处理
        max_length=zz
        my_list = []
        len_hang=len(line)
        while len_hang>0:
            tryW=line[max(0,len_hang-max_length):len_hang]#防止溢出
            while tryW not in dic:
                if len(tryW)==1:
                    break;
                tryW=tryW[1:len(tryW)]#这里注意，一直是从1开始的
            my_list.append(tryW)
            line=line[0:len_hang-len(tryW)]
            len_hang=len(line)
    while len(my_list):
        tt=my_list.pop()#这里类似栈的操作
        ff.write(tt+"/")
    ff.close()
readfile()##012345