决策树

决策树用于标称型数据的分类，首先要构造决策树，使用信息增益选择最好的数据集划分方式。

from math import log
# 计算给定数据集的香农熵
def calcShannonEnt(dataSet):
    # 实例总数
    numEntries = len(dataSet)
    # 对于数据集中的每一个数据，获得最后一列的键值，如果该键值在数据字典中不存在，扩展数据字典。
    labelCounts = {}
    for featVec in dataSet:
        currentLabel = featVec[-1]
        if currentLabel not in labelCounts.keys():
            labelCounts[currentLabel] = 0
        labelCounts[currentLabel] += 1
    # 计算每个类别的概率，并计算香农熵
    shannonEnt = 0.0
    for key in labelCounts:
        prob = float(labelCounts[key])/numEntries
        shannonEnt -= prob * log(prob,2)
    return shannonEnt


# 数据集的划分，以axis所在的索引位置划分数据集
def splitDataSet(dataSet, axis, value):
    retDataSet = []
    # 数据集中的每一个数据，如果划分数据的特征与需要返回的特征值相等，则将除axis索引位置所在列的其他列的数据返回
    for featVec in dataSet:
        if featVec[axis] == value:
            # 相当于一个空列表
            reduceFeatVec = featVec[:axis]
            #print(reduceFeatVec)
            #reduceFeatVec = []
            # python在函数内部对列表对象的修改，会影响列表对象的整个生命周期
            reduceFeatVec.extend(featVec[axis + 1:])
            retDataSet.append(reduceFeatVec)
            #print(retDataSet)
    return retDataSet


"""
dataSet = [[1,1,'yes'],[1,1,'yes'],[1,0,'no'],[0,1,'no'],[0,1,'no']]
splitDataSet(dataSet,0,1)
"""

# 选择最好的数据集划分方式,数据必须是由列表元素组成的列表，所有的列表元素要有相同的长度，每个实例的最后一个元素是该实例的类别标签
def chooseBestFeatureToSplit(dataSet):
    # 判定当前数据集包含多少特征属性，一定有一个属性是类别，而且为了之后的for循环遍历方便
    numFeatures = len(dataSet[0]) - 1
    # 原始香农熵，保存最初的无序度量值
    baseEntropy = calcShannonEnt(dataSet)
    bestInfoGain = 0.0
    bestFeature = -1
    # featList = []
    for i in range(numFeatures):
        """
        for example in dataSet:
            featList.append(example[i])
        """
        # 获得不相同的特征值
        featList = [example[i] for example in dataSet]
        uniqueVals = set(featList)
        newEntropy = 0.0
        # 对于该特征的每一个唯一属性值划分数据集，并计算新的香农熵值，并对所有唯一属性的熵值求和
        for value in uniqueVals:
            subDataSet = splitDataSet(dataSet, i, value)
            prob = len(subDataSet)/float(len(dataSet))
            newEntropy += prob * calcShannonEnt(subDataSet)
        # 计算信息增益，即计算数据无序度的减少
        infoGain = baseEntropy - newEntropy
        # 比较所有特征中的信息增益，返回最好特征划分的索引值
        if(infoGain > bestInfoGain):
            bestInfoGain = infoGain
            bestFeature = i
    return bestFeature


"""
dataSet = [[1, 1, 'yes'], [1, 1, 'yes'], [1, 0, 'no'], [0, 1, 'no'], [0, 1, 'no']]
bestFeature = chooseBestFeatureToSplit(dataSet)
print(bestFeature)
"""

"""
for i in range(9):
    print(i)

print(len(range(9)))
"""

使用递归来创建决策树，如果类标签不能唯一决定数据集的划分时，采用多数表决来决定叶子节点的分类

# majorityCnt()函数用
import operator
# 如果数据集处理了所有属性，但类标签依然不唯一，用多数表决的方法决定叶子节点的分类
def majorityCnt(classList):
    # 存储每个类标签出现的频率
    classCount = {}
    # 如果标签不存在字典中，则创建，存在，加一
    for vote in classList:
        if vote not in classCount.keys():
            classCount[vote] = 0
        classCount[vote] += 1
    # 排序，返回最大值的标签
    sortedClassCount = sorted(classCount.items(), key=operator.itemgetter(1), reverse=True)
    return sortedClassCount[0][0]


# 创建树
def creatTree(dataSet, labels):
    # 数据集的所有类标签
    classList = [example[-1] for example in dataSet]
    # 所有的类标签完全相同，直接返回类标签
    if(classList.count(classList[0]) == len(classList)):
        return classList[0]
    # 使用完了所有特征，仍不能将数据集划分为仅包含唯一类别的分组
    if(len(dataSet) == 1):
        return majorityCnt(classList)
    # 选择出最好特征的索引和标签
    bestFeat = chooseBestFeatureToSplit(dataSet)
    # print(bestFeat)
    bestFeatLabel = labels[bestFeat]
    # print(bestFeatLabel)
    # myTree用于存储树的所有信息
    myTree = {bestFeatLabel: {}}
    # 最好特征的标签要删除
    del labels[bestFeat]
    # 得到列表包含的所有属性值
    featValues = [example[bestFeat] for example in dataSet]
    # print(featValues)
    uniqueVals = set(featValues)
    # 遍历当前特征包含的所有属性，在每个数据集上递归调用函数，并将得到的值填入myTree中
    for value in uniqueVals:
        # 复制类标签，并开辟空间，存储在新的列表变量中
        subLabels = labels[:]
        myTree[bestFeatLabel][value] = creatTree(splitDataSet(dataSet, bestFeat, value), subLabels)
    return myTree

"""
dataSet = [[1, 1, 'yes'], [1, 1, 'yes'], [1, 0, 'no'], [0, 1, 'no'], [0, 1, 'no']]
labels = ['no surfacing', 'flippers']
myTree = creatTree(dataSet,labels)
print(myTree)
"""

结果：

使用Matplotlib绘制树形图

# 树节点的绘制
import matplotlib.pyplot as plt
# 定义文本框（叶子节点和判定节点）和箭头格式
decisionNode = dict(boxstyle="sawtooth", fc="0.8")
leafNode = dict(boxstyle="round4", fc="0.8")
arrow_args = dict(arrowstyle="<-")


# 绘图功能的执行，该功能的执行需要需要有一个全局变量createPlot.ax1()定义的绘图区
def plotNode(nodeTxt,centerPt,parentPt,nodeType):
    createPlot.ax1.annotate(nodeTxt, xy=parentPt, xycoords='axes fraction',xytext=centerPt, textcoords='axes fraction', va='center',ha='center',bbox=nodeType,arrowprops=arrow_args)



# 新图形的创建，清空绘图区，绘制俩个代表不同类型的树节点
def createPlot():
    # fig = plt.figure(1,facecolor='white')
    # fig.clf()
    # 绘制一个长宽比是1：1的图形，并设置图形不可见
    createPlot.ax1 = plt.subplot(111,frameon=False)
    # 绘制俩个节点
    plotNode('决策节点', (0.5, 0.1), (0.1, 0.5), decisionNode)
    plotNode('叶节点', (0.8, 0.1), (0.3, 0.8), leafNode)
    plt.show()

# createPlot()

# 获取叶节点的数目
def getNumLeafs(myTree):
    numLeafs = 0
    firstStr = list(myTree.keys())[0]
    secondDict = myTree[firstStr]
    for key in secondDict.keys():
        if type(secondDict[key]).__name__ == 'dict':
            numLeafs += getNumLeafs(secondDict[key])
        else:
            numLeafs += 1
    return numLeafs


# 计算树的层数
def getTreeDepth(myTree):
    maxDepth = 0
    firstStr = list(myTree.keys())[0]
    secondDict = myTree[firstStr]
    for key in secondDict.keys():
        if type(secondDict[key]).__name__ == 'dict':
            thisDepth = 1 + getTreeDepth(secondDict[key])
        else:
            thisDepth = 1
        if thisDepth > maxDepth:
            maxDepth = thisDepth
    return maxDepth

# 定义俩颗预先储存好的树信息，方便测试使用
def retrieveTree(i):
    listOfTree = [{'no surfacing': {0: 'no', 1: {'flippers': {0: 'no', 1: 'yes'}}}},
                  {'no surfacing': {0: 'no', 1: {'flipper': {0: {'head': {0: 'no', 1: 'yes'}}, 1: 'no'}}}}]
    return listOfTree[i]


# myTree = retrieveTree(0)
# print(getNumLeafs(myTree))
# print(getTreeDepth(myTree))

# 在父子节点间填充文本信息，xMid 和 yMid 分别表示横纵坐标的位置，txtString表示要填充的文本信息
def plotMidText(cntrPt, parentPt, txtString):
    xMid = (parentPt[0] - cntrPt[0])/2.0 + cntrPt[0]
    yMid = (parentPt[1] - cntrPt[1])/2.0 + cntrPt[1]
    createPlot.ax1.text(xMid, yMid, txtString)

# 按照图形比例绘制树形图
def plotTree(myTree, parentPt, nodeTxt):
    numLeafs = getNumLeafs(myTree)
    depth = getTreeDepth(myTree)
    firstStr = list(myTree.keys())[0]
    # 计算决策节点的坐标，注意横坐标的变换，初始横坐标设置为 -(1/2w),父节点的横坐标为 -(1/2w) + (1 + w)/2w
    cntrPt = (plotTree.xOff + (1.0 + float(numLeafs)) / 2.0 / plotTree.totalW, plotTree.yOff)
    plotMidText(cntrPt, parentPt, nodeTxt)
    # 绘制决策节点
    plotNode(firstStr, cntrPt, parentPt, decisionNode)
    # 获取当前节点的子节点
    secondDict = myTree[firstStr]
    # 深度的变化，即坐标标变化，移到绘制子节点的位置
    plotTree.yOff = plotTree.yOff - 1.0 / plotTree.totalD
    # 遍历子节点，判断是决策节点还是叶子节点
    for key in secondDict.keys():
        # 如果是决策节点，递归调用函数
        if type(secondDict[key]).__name__ == 'dict':
            plotTree(secondDict[key], cntrPt, str(key))
        # 如果是叶子节点，计算叶子节点横坐标的位置，在 -(1/2w)的基础上每次偏移 1/w
        else:
            plotTree.xOff = plotTree.xOff + 1.0 / plotTree.totalW
            plotNode(secondDict[key], (plotTree.xOff, plotTree.yOff), cntrPt, leafNode)
            plotMidText((plotTree.xOff, plotTree.yOff), cntrPt, str(key))
    # 由于每次在循环之前就到达绘制子节点的位置，在循环里面递归时又将深度减了一次，所以要加
    plotTree.yOff = plotTree.yOff + 1.0 / plotTree.totalD


def createPlot(inTree):
    fig = plt.figure(1, facecolor='white')
    # 图形区清空，绘制图形
    fig.clf()
    # 去掉横纵坐标上的值
    axprops = dict(xticks=[],yticks=[])
    # print(axprops)
    createPlot.ax1 = plt.subplot(111,frameon=False, **axprops)
    # 树的宽度
    plotTree.totalW = float(getNumLeafs(inTree))
    # 树的宽度
    plotTree.totalD = float(getTreeDepth(inTree))
    # 为方便计算横坐标的变化而赋的初值
    plotTree.xOff = -0.5/plotTree.totalW
    # 父节点的起始纵坐标
    plotTree.yOff = 1.0
    # 绘制图形
    plotTree(inTree, (0.5, 1.0), '')
    plt.show()

# 测试函数输出结果
# myTree = retrieveTree(0)
# createPlot(myTree)

"""
# 变更字典，重新绘制图
myTree = retrieveTree(0)
myTree['no surfacing'][3] = 'maybe'
print(myTree)
createPlot(myTree)
# 变更字典，重新绘制图
myTree['no surfacing'][1]['flippers'][2] = 'head'
createPlot(myTree)
"""

结果：

决策树的使用

# 使用决策树分类
def classify(inputTree,featLabels,testVec):
    firstStr = list(inputTree.keys())[0]
    secondDict = inputTree[firstStr]
    # 特征标签列表的index方法查找当前列表中第一个匹配firstStr变量的元素
    featIndex = featLabels.index(firstStr)
    for key in secondDict.keys():
        # 如果测试集(testVec)的值与树节点的值相同，继续比较
        if testVec[featIndex] == key:
            # 判断是决策节点还是叶子节点，决策节点递归，叶子节点返回结果
            if(type(secondDict[key]).__name__ == 'dict'):
                classLabel = classify(secondDict[key], featLabels, testVec )
            else:
                classLabel = secondDict[key]
    return classLabel


"""
labels = ['no surfacing', 'flippers']
myTree = {'no surfacing': {0: 'no', 1: {'flippers': {0: 'no', 1: 'yes'}}}}
print(classify(myTree, labels, [1, 0]))
print(classify(myTree, labels, [0]))
"""

持久化分类器，将决策树使用pickle模块存储

# 使用pickle模块存储决策树
def storeTree(inputTree, filename):
    import pickle
    fw = open(filename, 'wb')
    pickle.dump(inputTree, fw)
    fw.close()


def grabTree(filename):
    import pickle
    fr = open(filename, 'rb')
    return pickle.load(fr)


"""
myTree = {'no surfacing': {0: 'no', 1: {'flippers': {0: 'no', 1: 'yes'}}}}
storeTree(myTree, 'classifierStorage.txt')
grabTree('classifierStorage.txt')
"""

实例：使用决策树预测隐形眼镜类型

from chapter3 import trees
from chapter3 import treePlotter
fr = open('E:\机器学习\machinelearninginaction\Ch03\lenses.txt')
lenses = [inst.strip().split('\t') for inst in fr.readlines()]
lensesLabels = ['age', 'prescript', 'astigmatic', 'tearRate']
lensesTree = trees.creatTree(lenses, lensesLabels)
print(lensesTree)
treePlotter.createPlot(lensesTree)

结果：