使用概率分布进行分类,根据概率大小决定分类结果
有标签的数据分类,构建词向量和标签
def loadDataSet():
postingList =[['my', 'dog', 'has', 'flea', 'problems', 'help', 'please'],
['maybe', 'not', 'take', 'him', 'to', 'dog', 'park', 'stupid'],
['my', 'dalmation', 'is', 'so', 'cute', 'I', 'love', 'him'],
['stop', 'posting', 'stupid', 'worthless', 'garbage'],
['mr', 'licks', 'ate', 'my', 'steak', 'how', 'to', 'stop', 'him'],
['quit', 'buying', 'worthless', 'dog', 'food', 'stupid']]
classVec = [0, 1, 0, 1, 0, 1]
return postingList, classVec
创建词汇表,词汇表中包含文本的所有词汇
# 创建包含在所有文档中出现的不重复词列表
def createVocabList(dataSet):
# set去重复,无序
vocabSet = set([])
for document in dataSet:
# | 用于求俩个集合的并集
vocabSet = vocabSet | set(document)
return list(vocabSet)
# 将输入的文档转化为与词汇表等长的向量
def setOfWordsToVec(vocabList, inputSet):
returnVec = [0] * len(vocabList)
for word in inputSet:
if word in vocabList:
returnVec[vocabList.index(word)] = 1
else:
print("The word: %s is not in my vocabulary" % word)
return returnVec
对测试的文本进行处理,即将测试文本转化为与词汇表等长的向量,结果为一串0,1组成的列表
# 将输入的文档转化为与词汇表等长的向量
def setOfWordsToVec(vocabList, inputSet):
returnVec = [0] * len(vocabList)
for word in inputSet:
if word in vocabList:
returnVec[vocabList.index(word)] = 1
else:
print("The word: %s is not in my vocabulary" % word)
return returnVec
利用朴素贝叶斯计算不同类别下的概率
# 朴素贝叶斯分类器训练函数
def trainNB0(trainMatrix, trainCategory):
# 总文档数
numTrainDocs = len(trainMatrix)
# print(numTrainDocs)
# 词条总数(词汇表的词汇个数)
numWords = len(trainMatrix[0])
# print(numWords)
# 文档属于侮辱性文档的概率
pAbusive = sum(trainCategory) / float(numTrainDocs)
# 按照词汇表大小初始化
p0Num = ones(numWords)
# print(p0Num)
p1Num = ones(numWords)
# 初始化每个类别总数
p0Denom = 2.0
p1Denom = 2.0
# 对每个文档循环
for i in range(numTrainDocs):
if trainCategory[i] == 1:
# 对于属于侮辱性的每篇文档,如果某个词在该文档中出现,则对应位置增加1,侮辱性文档的词条数目加1
p1Num += trainMatrix[i]
# print(p1Num)
p1Denom += sum(trainMatrix[i])
# print(p1Denom)
else:
p0Num += trainMatrix[i]
p0Denom += sum(trainMatrix[i])
"""
p1Vect = p1Num / p1Denom
p0Vect = p0Num / p0Denom
"""
# 使用自然对数,避免下溢出或者浮点数舍入错误(采用自然对数进行处理不会有损失)
p1Vect = log(p1Num / p1Denom)
p0Vect = log(p0Num / p0Denom)
return p0Vect, p1Vect, pAbusive
# 朴素贝叶斯分类函数,vec2Classify表示要分类的向量
def classifyNB(vec2Classify, p0Vec, p1Vec, pClass1 ):
p1 = sum(vec2Classify * p1Vec) + log(pClass1)
p0 = sum(vec2Classify * p0Vec) + log(1.0 - pClass1)
if p1 > p0:
return 1
else:
return 0
# 封装前面的函数,简化操作
def testingNB():
listOPosts , listClasses = loadDataSet()
myVocabList = createVocabList(listOPosts)
trainMat = []
for postinDoc in listOPosts:
trainMat.append(setOfWordsToVec(myVocabList, postinDoc))
p0V ,p1V, pAb = trainNB0(array(trainMat), array(listClasses))
testEntry = ['love', 'my', 'dalmation']
thisDoc = array(setOfWordsToVec(myVocabList, testEntry))
print(testEntry, 'classify as :', classifyNB(thisDoc, p0V, p1V, pAb))
# testingNB()
# 词袋模型,与setOfWordsToVec()所不同的是,词袋模型,每出现一个单词,增加词向量中的对应值,而不是将该值设为1
def bagOfWordsToVec(vocabList, inputSet):
returnVec = [0] * len(vocabList)
for word in inputSet:
if word in vocabList:
returnVec[vocabList.index(word)] += 1
return returnVec
# 使用朴素贝叶斯过滤垃圾邮件
# 将大写的字符串转化为字符串列表,去掉少于俩个字符的字符串,将所有的字符串转化为小写
def textParse(bigString):
# 使用正则表达式切分,分隔符是除单词,数字外的任意字符串
import re
listOfTokens = re.split('\W*', bigString)
return [tok.lower() for tok in listOfTokens if len(tok) > 2]
# 贝叶斯垃圾邮件分类器
def spamTest():
docList = []
classList = []
fullText = []
# 将ham和spam中的内容解析为词列表,并给垃圾邮件和非垃圾邮件分类
for i in range(1, 26):
wordList = textParse(open('E:\机器学习\machinelearninginaction\Ch04\email\spam\%d.txt' % i).read())
docList.append(wordList)
fullText.extend(wordList)
classList.append(1)
wordList = textParse(open('E:/机器学习/machinelearninginaction/Ch04/email/ham/%d.txt' % i, encoding='gb18030', errors='ignore').read())
docList.append(wordList)
fullText.extend(wordList)
classList.append(0)
# 创建不重复词的词列表
vocabList = createVocabList(docList)
# 一共有五十份邮件,初始化训练集为50
trainingSet = list(range(50))
testSet = []
# 随机选择测试集,并将选择好的测试文档从训练集中删除
for i in range(10):
randIndex = int(random.uniform(0, len(trainingSet)))
testSet.append(trainingSet[randIndex])
del trainingSet[randIndex]
# 训练集
trainMat = []
trainClasses = []
for docIndex in trainingSet:
trainMat.append(setOfWordsToVec(vocabList, docList[docIndex]))
trainClasses.append(classList[docIndex])
p0V, p1V, pSpam = trainNB0(array(trainMat), array(trainClasses))
# 计算错误率
errorCount = 0.0
for docIndex in testSet:
wordVector = setOfWordsToVec(vocabList, docList[docIndex])
if classifyNB(array(wordVector), p0V, p1V, pSpam) != classList[docIndex]:
errorCount += 1
print("错误率", float(errorCount) / len(testSet))
导入Rss源,从个人广告中获取区域倾向
import feedparser
from chapter4.bayes import textParse
from chapter4.bayes import createVocabList
from numpy import *
from chapter4.bayes import bagOfWordsToVec
from chapter4.bayes import trainNB0
from chapter4.bayes import classifyNB
# 统计词汇表中的每个词在文本中出现的次数,返回出现次数最多的三十个词
def calcMostFreq(vocabList, fullText):
import operator
freqDict = {}
for token in vocabList:
freqDict[token] = fullText.count(token)
sortedFreq = sorted(freqDict.items(), key=operator.itemgetter(1), reverse=True)
return sortedFreq[:30]
# 使用俩个Rss源作为参数,交叉验证,计算错误率
def localWords(feed1, feed0):
import feedparser
docList = []
classList = []
fullText = []
minLen = min(len(feed1['entries']), len(feed0['entries']))
for i in range(minLen):
wordList = textParse(feed1['entries'][i]['summary'])
docList.append(wordList)
fullText.extend(wordList)
classList.append(1)
wordList = textParse(feed0['entries'][i]['summary'])
docList.append(wordList)
fullText.extend(wordList)
classList.append(0)
vocabList = createVocabList(docList)
top30Words = calcMostFreq(vocabList, fullText)
# 去掉高频词汇,语言中的冗余和结构辅助性内容占据了词汇表的很大一部分,这些词对训练没有好处
for pairW in top30Words:
if pairW[0] in vocabList:
vocabList.remove(pairW[0])
trainingSet = list(range(2 * minLen))
testSet = []
for i in range(20):
randIndex = int(random.uniform(0, len(trainingSet)))
testSet.append(trainingSet[randIndex])
del trainingSet[randIndex]
trainMat = []
trainClasses = []
for docIndex in trainingSet:
trainMat.append(bagOfWordsToVec(vocabList, docList[docIndex]))
trainClasses.append(classList[docIndex])
p0V, p1V, pAb = trainNB0(array(trainMat), array(trainClasses))
errorCount = 0.0
for docIndex in testSet:
wordVector = bagOfWordsToVec(vocabList, docList[docIndex])
if classifyNB(array(wordVector), p0V, p1V, pAb) != classList[docIndex]:
errorCount += 1
print("错误率", errorCount)
return vocabList, p0V, p1V
"""
xa = feedparser.parse('http://xian.craigslist.org/stp/index.rss')
nj = feedparser.parse('http://nanjing.craigslist.org/stp/index.rss')
vocablist, p0V, p1V = localWords(xa, nj)
"""
# 根据某个词出现的条件概率,选择出大于某个阈值的所有词
def getTopWords(local1, local2):
import operator
vocabList, p0V, p1V = localWords(local1, local2)
top1 = []
top2 = []
for i in range(len(p0V)):
if p0V[i] > -6.0:
top1.append((vocabList[i], p0V[i]))
if p1V[i] > -6.0:
top2.append((vocabList[i], p1V[i]))
# 列表数组从0开始,所以按照第二个元素进行排序,即按照条件概率的大小从大到小排序
sorted1 = sorted(top1, key=lambda pair: pair[1], reverse=True)
print("---------------------")
for item in sorted1:
# 打印词汇
print(item[0])
sorted2 = sorted(top2, key=lambda pair: pair[1], reverse=True)
print("--------------------")
for item in sorted2:
print(item[0])