genism 实现LDA

理论知识可以参考这个文章

这里只看简单实现： 调用 gensim 可以直接实现。

from gensim import corpora, models, similarities

import  numpy  as np

#将所有的语料 放入一个list中 用逗号隔开  每一个逗号 表示一篇文章
documents = ["Human machine interface for lab abc computer applications",
             "A survey of user opinion of computer system response time",
             "The EPS user interface management system",
             "System and human system engineering testing of EPS",
             "Relation of user perceived response time to error measurement",
             "The generation of random binary unordered trees",
             "The intersection graph of paths in trees",
             "Graph minors IV Widths of trees and well quasi ordering",
             "Graph minors A survey"]

#切割文章 变成 list 形式 [ [ 单词1， 单词2]，  [ 单词3 ，单词4]]
texts = [[word for word in document.lower().split()] for document in documents]
M = len(texts)
print("texts==",texts)

# 词典 将文章里面所有的词 按照 顺序  和 词对应起来
#dict= (0, 'abc')(1, 'applications')(2, 'computer')(3, 'for')

dictionary = corpora.Dictionary(texts)


for  k  in  dictionary.iteritems():
    print("dict=",k)
# 词库，以(词，词频)方式存贮
#(0, 1), (1, 1), (2, 1), (3, 1)
corpus = [dictionary.doc2bow(text) for text in texts]

print("corpus==",corpus)

#初始化语料
tfidf = models.TfidfModel(corpus)

#计算每个词的  tf-idf
corpus_tfidf = tfidf[corpus]

# (0, 0.39510679503439006), (1, 0.39510679503439006), (2, 0.270464478621662),
for doc in corpus_tfidf:
    print("doc",doc)

num_topics=3
lda=models.LdaModel(corpus_tfidf,num_topics,id2word=dictionary,alpha=0.01,eta=0.01,minimum_phi_value=0.01,update_every=1,chunksize=3,passes=1)

#获取若干主题
doc_topics = lda.get_document_topics(corpus_tfidf)


num_show_term = 3
num_show_topic=3
idx = np.arange(M)

np.random.shuffle(idx)
idx = idx[:10]
#主题先排序
for i in idx:
    topic = np.array(doc_topics[i])
    print("topic==",topic)
    topic_distribute = np.array(topic[:, 1])
    # print topic_distribute
    topic_idx = topic_distribute.argsort()[:-num_show_topic-1:-1]
    print ('第%d个文档的前%d个主题：' % (i, num_show_topic)), topic_idx
    print(topic_distribute[topic_idx])


#展示topic
for topic_id in range(num_topics):

    term_distribute_all = lda.get_topic_terms(topicid=topic_id)
    print('主题#%d：\t' % topic_id)
    term_distribute = term_distribute_all[:num_show_term]
    print("term_distribute",term_distribute)
    term_distribute = np.array(term_distribute)
    term_id = term_distribute[:, 0].astype(np.int)
    print('词：\t', )
    for t in term_id:
        print(dictionary.id2token[t], )
    print('\n概率：\t', term_distribute[:, 1])