机器学习19-自然语言处理包（NLTK）

使用词袋法（Bag-of-Words）对示例文本进行特征向量化

from sklearn.feature_extraction.text import CountVectorizer

sent1 = 'The cat is waking in the bedroom'
sent2 =  'A dog was running across the kitchen'

count_vec = CountVectorizer()

sentences = [sent1, sent2]

print(count_vec.fit_transform(sentences).toarray())
#out[]:
# [[0 1 1 0 1 1 0 0 2 1 0]
#  [1 0 0 1 0 0 1 1 1 0 1]]

#输出向量各个维度的特征含义
print(count_vec.get_feature_names())
#out[]:
# ['across', 'bedroom', 'cat', 'dog', 'in', 'is', 'kitchen', 'running', 'the', 'waking', 'was']

使用NLTK对示例文本进行语言分析

在cmd中安装nltk包：

1. pip install nltk
2. 安装NLTK数据: import nltk   nltk.download()
   
   然后弹出NLTK Downloader，选择All packages并点击下载。
   

code

import nltk

nltk.download('punkt')
sent1 = 'The cat is waking in the bedroom'
sent2 =  'A dog was running across the kitchen'
#对句子进行词汇分割和正规化，有些情况如are'nt需要分割为are和n't;
#或者I'm要分割为I和'm
tokens_1 = nltk.word_tokenize(sent1)
print(tokens_1)
#out[]:['The', 'cat', 'is', 'waking', 'in', 'the', 'bedroom']

tokens_2 = nltk.word_tokenize(sent2)
print(tokens_2)
#out[]:['A', 'dog', 'was', 'running', 'across', 'the', 'kitchen']
#整理两句的词表，并且按照ASCII的排序输出。
vocab_1 = sorted(set(tokens_1))
print(vocab_1)
#out[]:['The', 'bedroom', 'cat', 'in', 'is', 'the', 'waking']
vocab_2 = sorted(set(tokens_2))
print(vocab_2)
#out[]:['A', 'across', 'dog', 'kitchen', 'running', 'the', 'was']
#初始化stemmer寻找各个词汇最原始的词根。
stemmer = nltk.stem.PorterStemmer()
stem_1 = [stemmer.stem(t) for t in tokens_1]
print(stem_1)
#out[]:['the', 'cat', 'is', 'wake', 'in', 'the', 'bedroom']
stem_2 = [stemmer.stem(t) for t in tokens_2]
print(stem_2)
#out[]:['A', 'dog', 'wa', 'run', 'across', 'the', 'kitchen']
#初始化词性标注器，对每个词汇进行标注。
pos_tag_1 = nltk.tag.pos_tag(tokens_2)
print(pos_tag_1)
#out[]:[('A', 'DT'), ('dog', 'NN'), ('was', 'VBD'), ('running', 'VBG'), ('across', 'IN'), ('the', 'DT'), ('kitchen', 'NN')]
pos_tag_2 = nltk.tag.pos_tag(tokens_2)
print(pos_tag_2)
#out[]:[('A', 'DT'), ('dog', 'NN'), ('was', 'VBD'), ('running', 'VBG'), ('across', 'IN'), ('the', 'DT'), ('kitchen', 'NN')]