文本分类应用到工作实践

日常工作中会用到文本分类，如果量比较少人工分类一下也可实现，但是准确率不是很高。如果量多就需要用到NLP相关功能。直接上代码实践。
运行环境：python3 paddlepaddle2.5.0

加载数据集

# 加载当前数据集
cd /home/aistudio/data/data238254/

安装组件

# 安装组件
pip install fasttext -i https://pypi.tuna.tsinghua.edu.cn/simple

导入依赖包

import fasttext
import pandas as pd
import numpy as np
import jieba as jb
import re
import csv
from sklearn.utils import shuffle    

读取文件

#读取数据
df = pd.read_csv('/home/aistudio/data/data238254/train_data.csv',encoding='utf-8')
df=df[['cat','review']]
print("数据总量: %d ." % len(df))
print("在 cat 列中总共有 %d 个空值." % df['cat'].isnull().sum())
print("在 review 列中总共有 %d 个空值." % df['review'].isnull().sum())
df[df.isnull().values==True]
df = df[pd.notnull(df['review'])]

统计数据

#统计各类别数据量
d = {'cat':df['cat'].value_counts().index, 'count': df['cat'].value_counts()}
df_cat = pd.DataFrame(data=d).reset_index(drop=True)
df['cat_id'] = df['cat'].factorize()[0]
print(df.tail(5))

加载停用词

#加载停用词
 stopwords = [line.strip() for line in open("/home/aistudio/data/data238254/stopwords.txt", 'r', encoding='utf-8').readlines()]

去除特殊符号

#定义删除除字母,数字，汉字以外的所有符号的函数
def remove_punctuation(line):
    line = str(line)
    if line.strip()=='':
        return ''
    rule = re.compile(u"[^a-zA-Z\u4E00-\u9FA5]")
    line = rule.sub('',line)
    line = re.sub("d", "", line)
    line=line.replace('->' , '')
    return line

df['clean_review'] = df['review'].apply(remove_punctuation)
df['clean_cat'] = df['cat'].apply(remove_punctuation)
df['cut_review'] = df['clean_review'].apply(lambda x: " ".join([w for w in list(jb.cut(x)) if w not in stopwords]))
df.head(1)

df=shuffle(df)
train_num=int(len(df)*0.8)
train_set=df[0:train_num]
test_set=df[train_num:-1]
with open('/home/aistudio/data/data238254/train.txt','a') as train:
    for i, p in train_set.iterrows():
        train.write("__label__"+str(p['cat_id'])+" "+str(p['cut_review'])+"\n")
with open('/home/aistudio/data/data238254/test.txt','a') as test:
    for i, p in test_set.iterrows():
        test.write("__label__"+str(p['cat_id'])+" "+str(p['cut_review'])+"\n")

生成字典

#生成字典
with open('/home/aistudio/data/data238254/dic.txt','a') as dic:
    for i, p in test_set.iterrows():
        dic.write("__label__"+str(p['cat_id'])+":"+str(p['cat'])+"\n")   

模型训练

#模型训练
dur=100
model = fasttext.train_supervised(input='/home/aistudio/data/data238254/train.txt', autotuneValidationFile='/home/aistudio/data/data238254/test.txt', autotuneDuration=dur)
#保存模型
model.save_model("/home/aistudio/data/sheet_model.bin")

样本分析

train_result=model.test('/home/aistudio/data/data238254/train.txt')
print('train_precision:', train_result[1])
print('train_recall:', train_result[2])
print('Number of train examples:', train_result[0])
test_result=model.test('/home/aistudio/data/data238254/test.txt')
print('test_precision:', test_result[1])
print('test_recall:', test_result[2])
print('Number of test examples:', test_result[0]) 

获取字典值

#获取字典
file_path = "/home/aistudio/data/data238254/dic.txt"
content_dict = {}
with open(file_path, "r") as f:
    for line in f:
        key, value = line.split(":")
        content_dict[key] = value.strip()
print(content_dict)

结果预测

#模型预测
fasttext.FastText.eprint = lambda x: None
model= fasttext.load_model("/home/aistudio/data/sheet_model.bin")
text="手机卡安装后显示无服务"
format_sec=text.join([w for w in list(jb.cut(remove_punctuation(text))) if w not in stopwords])
result=model.predict(format_sec,k=5)
print( "结果:",result)

print( "结果:",content_dict[str(result[0][0])])
print( "结果:",content_dict[str(result[0][1])])
print( "结果:",content_dict[str(result[0][2])])
print( "结果:",content_dict[str(result[0][3])])
print( "结果:",content_dict[str(result[0][4])])

dur=100 标示训练轮数。如果设置过大，普通电脑训练时间会过长。