自然语言处理

就是对龙王传说这本小，查找关键字，并且按照，权重的大小排序

# coding: utf-8

# In[1]:


import json
import jieba
import pandas as pd
import os
import codecs
import sys
import re
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix
from imp import reload 
reload(sys)
import chardet
from tqdm import tqdm
from sklearn.feature_extraction.text import TfidfVectorizer


# In[2]:


s = open('龙王传说23.txt','r',encoding = 'utf-16')


# In[3]:


def clean(text):
    #text = re.sub('[0-9]{2,}','',text)#消除两位以上的数字
    #text= re.sub('(?:(?:\d+,?)+(?:\.?\d+)?)','',text)#消除数字
    text = text.replace(' ','')#消除空格
    #text = re.sub('http[s]?://(?:[a-z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-f][0-9a-f]))+','',text)#消除网页链接
    text = re.sub('[a-zA-Z]','',text)#消除英文字母
    words  = list(jieba.lcut(text))
    stopwords = [line.rstrip() for line in open('stopwords.txt','r',encoding="utf-8")]
    words_nostop = [w for w in words if w not in stopwords]
    return ' '.join(words_nostop)


# In[4]:


Sents = [line.rstrip('\r\n') for line in s]#/r/n 回车加换行 


# In[5]:


Sents


# In[6]:


n = 2
raw_Sents = ['\r\n'.join(Sents[i:i+n]) for i in range(len(Sents)-n+1)]


# In[7]:


Sents = raw_Sents


# In[8]:


Sents = [clean(i) for i in tqdm(iter(Sents))]


# In[9]:


raw_Sents


# In[10]:


Sents


# In[11]:


from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import CountVectorizer


# In[12]:


corpu = Sents
vectorizer=CountVectorizer()#该类会将文本中的词语转换为词频矩阵，矩阵元素a[i][j] 表示j词在i类文本下的词频
transformer=TfidfTransformer()#该类会统计每个词语的tf-idf权值
#x = vectorizer.fit_transform(corpus)
#l=vectorizer.get_feature_names()
#for u in l:
    #print (u)
tfidf=transformer.fit_transform(vectorizer.fit_transform(corpu))#第一个fit_transform是计算tf-idf，第二个fit_transform是将文本转为词频矩阵
word=vectorizer.get_feature_names()#获取词袋模型中的所有词语
weight=tfidf.toarray()#将tf-idf矩阵抽取出来，元素a[i][j]表示j词在i类文本中的tf-idf权重


# In[13]:


from collections import Counter


# In[14]:


def search(s):
    result = {}
    answer = []
    score = []
    _ = jieba.lcut(s)
    for q in _:
        if q in word:
            for i in range(len(weight)):
                for j in range(len(word)):
                    if q == word[j]:
                        result[i] = result.get(i,0) + weight[i][j]
    result = Counter(result).most_common(10)
    for n in range(len(result)):
        answer.append(raw_Sents[result[n][0]])
        score.append(result[n][1])
    c = {
        "answer" : answer,
        "score"  : score}
    tf = pd.DataFrame(c)
    return(tf)


# In[15]:


print (search(u'神级机甲有多强大'))


# In[37]:


print (search(u'魔鬼群岛的七位前辈')) 

 下面是代码运行的结果：