文章目录
文本分类【极简】模板
from jieba import cut
from sklearn.linear_model import LogisticRegression
from sklearn.feature_extraction.text import TfidfVectorizer
"""语料"""
texts = [
'三个只有一个是新鲜的，其它两个都坏了，买得最差的一次水果',
'京东，才发现你是个骗子，服务更是一样的烂',
'火龙果很小，而且还有一个烂了，发霉了',
'物流慢，收到时有两个底部已开始腐烂',
'屏幕清晰度不高，而且运行速度巨慢',
'贝质量很好，款式时尚，大小合适，做工精致，穿着舒服，服务很好',
'质量很好的，款式也不错，看起来高档大气，卖家服务还好，不错',
'裤子收到了，质量不错，价格便宜，穿着舒服，下次我还会来买的',
'宝贝收到了，试了一下，穿上挺舒服的，是正品，综合给好评',
'裤子挺好看，质量也不错，老公搭配衣服挺好看也很喜欢穿',
]
y = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
"""向量转换器"""
vectorizer = TfidfVectorizer(tokenizer=cut, stop_words=set('的在了是和也有为就都说等与才这，'))
X = vectorizer.fit_transform(texts)
"""分类模型"""
clf = LogisticRegression()
clf.fit(X, y)
print(clf.score(X, y))
多元文本分类
场景：文本包含标题和正文，将两者合并输入则为一元，分开输入则为多元
from jieba import cut
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from warnings import filterwarnings
filterwarnings('ignore')
def get_data():
y, x, xh, xb = [], [], [], []
with open('policy.txt', encoding='utf-8') as f:
for rows in f.read().strip().split('\n\n'):
label, head, body = rows.split('\n', 2)
y.append(label)
x.append(head + '\n' + body)
xh.append(head)
xb.append(body)
return y, x, xh, xb
def cut1(text):
"""cut1"""
for word in cut(text):
if word.strip():
yield word.strip()
def cut2(line):
"""cut2"""
head, body = line.split('\n', 1)
for word in cut1(head):
yield word + 'H'
for word in cut1(body):
yield word + 'B'
def clf_text(name, x, y, tokenizer=cut1):
# 数据切分
x_train, x_test, y_train, y_test = train_test_split(x, y, random_state=7)
# 向量化
vectorizer = TfidfVectorizer(tokenizer=tokenizer, min_df=2)
x_train = vectorizer.fit_transform(x_train)
x_test = vectorizer.transform(x_test)
# 建模
clf = LogisticRegression()
clf.fit(x_train, y_train)
print(name, tokenizer.__doc__, clf.score(x_test, y_test))
_y, _x, _xh, _xb = get_data()
clf_text('head', _xh, _y)
clf_text('body', _xb, _y)
clf_text('h++b', _x, _y)
clf_text('h++b', _x, _y, cut2)
head cut1 0.8768472906403941
body cut1 0.8866995073891626
h++b cut1 0.8916256157635468
h++b cut2 0.8866995073891626
文本编码
sklearn文本向量化
| parameter | explanation | default |
|---|---|---|
| lowercase | 是否转小写 | True |
| stop_words | 停词过滤 | None |
| tokenizer | 文本切分（默认正则表达式） | None |
| token_pattern | 文本切分的正则表达式 | r"(?u)\b\w\w+\b" |
| min_df | 低频词过滤（保留>=min_df） | 1 |
| max_features | 最大特征数（按词频降序前max_features个词） | None |
| ngram_range | 语言模型 | (1, 1) |
| vocabulary | 词表（只保留词表中存在的） | None |
CountVectorizer
from sklearn.feature_extraction.text import CountVectorizer
corpus = ['air air ball call', 'air ball ball', 'air air air']
vectorizer = CountVectorizer() # 创建文本向量转换器
vectorizer.fit(corpus) # 训练
X = vectorizer.transform(corpus) # 转换
print(X, type(X), X[0], type(X[0]), sep='\n')
(0, 0) 2
(0, 1) 1
(1, 0) 1
(1, 1) 2
(2, 0) 3
<class 'scipy.sparse.csr.csr_matrix'>
(0, 0) 2
(0, 1) 1
<class 'scipy.sparse.csr.csr_matrix'>
TfidfVectorizer（继承CountVectorizer）
import jieba
from sklearn.feature_extraction.text import TfidfVectorizer
# 语料
texts = ['小米苹果华为小米', '苹果小米苹果', '小米小米']
# 向量化
vectorizer = TfidfVectorizer(tokenizer=jieba.cut)
X = vectorizer.fit_transform(texts)
print(X)
print(X.toarray())
引入中文分词器
默认分词器是正则表达式分词，参数是tokenizer和token_pattern
stop_words过滤不需要的词
max_features限制最大词汇量，过滤低频词
import jieba
from sklearn.feature_extraction.text import CountVectorizer
# 语料
texts = ['小米、小米、苹果、华为', '小米和苹果、1+和苹果', '华为和小米']
# 分词器
jieba.add_word('1+', 2, 'nz')
tokenizer = lambda s: jieba.cut(s, HMM=False)
# 停词
stop_words = {'、', '和', '的'}
# 向量化
vectorizer = CountVectorizer(tokenizer=tokenizer, stop_words=stop_words, max_features=3)
X = vectorizer.fit_transform(texts)
print(X)
print(list(tokenizer('小米华为苹果1+')))
print(vectorizer.transform(['小米、华为、苹果和1+']))
知识补充：压缩稀疏矩阵
- Compressed Sparse Row Matrix
- Compressed Sparse Column Matrix
from numpy import array
from scipy.sparse import csr_matrix, csc_matrix
a = array([[0, 0, -2, 0],
[-1, 0, 3, 0],
[0, 0, 0, -9]])
csr = csr_matrix(a)
csc = csc_matrix(a)
print(a, csr, type(csr), csc, type(csc), csc.toarray(), sep='\n\n')
word2id
from collections import Counter
from jieba import cut
texts = ['酸奶蛋糕', '酸奶芝士', '芝士酸奶']
counter = Counter(word for text in texts for word in cut(text))
print(counter)
word2id = {c[0]: i for i, c in enumerate(counter.most_common())}
print(word2id)
-
print
-
Counter({‘酸奶’: 3, ‘芝士’: 2, ‘蛋糕’: 1})
{‘酸奶’: 0, ‘芝士’: 1, ‘蛋糕’: 2}
one hot representation
length = len(word2id)
for word, wid in word2id.items():
vector = [0] * length
vector[wid] = 1
print(word, vector)
-
print
-
酸奶 [1, 0, 0]
芝士 [0, 1, 0]
蛋糕 [0, 0, 1]
共现矩阵
语言模型+共现矩阵
共现矩阵+SVD
# 编造文本、分词
from jieba import lcut
sentences = ['我吴彦祖', '我张学友', '吴彦祖我', '张学友我刘德华吴彦祖',
'酸奶芝士', '芝士酸奶', '芝士蛋糕', '酸奶芝士蛋糕']
ls_of_words = [lcut(sentence) for sentence in sentences]
# 生成字典和词ID
from gensim.corpora import Dictionary
dt = Dictionary(ls_of_words).token2id
ls_of_wids = [[dt[word] for word in words] for words in ls_of_words]
# 共现矩阵
import numpy as np
dimension = len(dt) # 维数
matrix = np.matrix([[0] * dimension] * dimension)
def co_occurrence_matrix(ls):
length = len(ls)
for i in range(length):
for j in range(length):
if i != j:
matrix[[ls[i]], [ls[j]]] += 1
for ls in ls_of_wids:
co_occurrence_matrix(ls)
print(matrix)
# 奇异值分解（Singular Value Decomposition）
U, s, Vh = np.linalg.svd(matrix, full_matrices=False)
# 聚类
X = -U[:, 0:2]
from sklearn.cluster import KMeans
labels = KMeans(n_clusters=2).fit(X).labels_
colors = ('y', 'g')
# 可视化
import matplotlib.pyplot as mp
mp.rcParams['font.sans-serif'] = ['SimHei'] # 显示中文
for word in dt.keys():
i = dt[word]
mp.scatter(X[i, 1], X[i, 0], c=colors[labels[i]], s=400, alpha=0.4)
mp.text(X[i, 1], X[i, 0], word, ha='center', va='center')
mp.show()
附录
| en | cn |
|---|---|
| feature | 特征 |
| extraction | 提取 |
| SVD | Singular Value Decomposition |
| occurrence | n. 发生、出现 |
| dimension | 维数 |
| scatter | nv 分散 |
2599
扫一扫
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
