配置--NLP常用第三放依赖的安装和使用

jieba分词

参考: https://www.jianshu.com/p/8d455e70f724

ubuntu下安装jieba

pip install jieba

使用

import jieba
import jieba.posseg as psg

content = "张三非常喜欢打篮球，他认为打篮球和有趣."

print("精确分词模式结果：")
segs_1 = jieba.cut(content, cut_all=False)
print("/".join(segs_1))

print("全模式分词模式结果：")
segs_3 = jieba.cut(content, cut_all = True)
print("/".join(segs_3))

print("搜索引擎模式结果：")
segs_4 = jieba.cut_for_search(content)
print("/".join(segs_4))

print("获取分词后的list：")
segs_5 = jieba.lcut(content)
print(segs_5)

print("获取词性")
print([(x.word, x.flag) for x in psg.lcut(content)])

gensim

利用gensim得到对应单词的词向量

利用gensim和jieba生成中文词向量

https://blog.youkuaiyun.com/shuihupo/article/details/85162237

针对中文文档生成分词得到的每个单词对应的词向量。

import os
import jieba
import jieba.analyse
from gensim.models import Word2Vec
from gensim.models.word2vec import LineSentence

class ChineseW2V:
    def __init__(self, filename, outputPath="./output/", size=300, window=10, min_count=1, hs=1):
        self.filename = filename # 文件路径
        self.outputPath = outputPath # 文件输出路径
        self.size = size
        self.window = window
        self.min_count = min_count
        self.hs = hs
        self.jieba_cut_outfile = None

    def readFile(self):
        with open(self.filename, encoding="utf-8") as f:
            document = f.read()
            document_cut = jieba.cut(document)
            result = ' '.join(document_cut)

            if not os.path.exists(self.outputPath):
                os.mkdir(self.outputPath)

            with open(self.outputPath+"jieba_cut.txt", "w", encoding="utf-8") as outf:
                outf.write(result)
        self.jieba_cut_outfile = self.outputPath+"jieba_cut.txt"

    def genEmbedding(self):
        self.readFile()
        sentences = LineSentence(self.jieba_cut_outfile)
        model = Word2Vec(sentences, hs=self.hs, min_count=self.min_count, window=self.window, size=self.size)
        model.save("./output/Chinese_word2vec.model")
        print("=="*3+"Done"+"=="*3)


if __name__ == '__main__':
    filename = './data/三体txt.txt'
    genmodel = ChineseW2V(filename)
    genmodel.genEmbedding()