机器学习04-决策树-python

1. 原理推导

信息增益比准确计算方式

2. ID3算法实现

1. 计算信息熵

github完整代码

def calaShannonEnt(dataSet , index = -1):
    #m , n = dataSet.shape
    m = len(dataSet)

    labelCounts = {}
    # 提取每个分类数量
    for line in dataSet:
        fea = line[index]
        if fea not in labelCounts.keys():
            labelCounts[fea] = 0
        labelCounts[fea] += 1
    shannonEnt = 0
    for key in labelCounts:
        prob = float(labelCounts[key]) / m
        shannonEnt -= prob * np.log(prob)
    
    return shannonEnt

2. 分割数据

def spiltDataSet(dataSet  , axis , value ):
    '''
    para: 
        dataSet:        特征 + 标签数据
        axis:           根据那个特征进行分割数据
        value:          根据那个值进行分割数据
    return:
        指定特征指定值的数据集
    '''
    retData = []
    for fea in dataSet:
        if fea[axis] == value:
            feaVec = fea[:axis]
            feaVec.extend(fea[axis+1:])
            retData.append(feaVec)
    
    return retData

3. 选择最佳特征以及最佳特征值

def chooseBestFeatureToSplit(dataSet , imp = 0 ):
    '''
    原理：
        外循环每一个特征，内循环每一个特征属性
    para: 
        数据集 np.array
    return: 
        特征索引
    '''
    m = len(dataSet) 

    n = len(dataSet[0]) -1
    if n == 1:
        return 0
    baseEnt = calaShannonEnt(dataSet)
    bestFea = -1
    bestGain = 0
    bestGainRate = 0
    # 循环所有的特征
    for i in range(n):
        # 所有行，第i列数据
        FeaList = [ fea[i] for fea in dataSet ]
        uniqueVars = set(FeaList)
        newEnt = 0
        for value in uniqueVars:
            subData = spiltDataSet(dataSet , i , value)
            prob = len(subData) / float(m)
            newEnt += prob * calaShannonEnt(subData)
        infoGain = baseEnt - newEnt
        if imp == 0:  # 计算信息增益  ID3算法
            if infoGain > bestGain:
                bestFea = i
                bestGain = infoGain
        else:   # 这里是计算的信息增益比 c4.5算法
            #这里计算的是某个特征条件下的熵 ， 而不是集合类别的熵
            
            iv = calaShannonEnt(dataSet , i)
            if iv == 0:
                continue
            infoGainRate = infoGain / iv
            if bestGainRate < infoGainRate:
                bestFea = i
                bestGainRate = infoGainRate
    return bestFea

4. 投票的方式确定叶子节点的类别

def majorityCnt(classList):
    '''
    叶子节点判断属于什么类别  投票的方式
    '''
    classCount = {}
    for vote in classList:
        if vote not in classCount.keys():
            classCount[vote] = 0
        classCount[vote] += 1
    sortedClass = sorted(classCount.items() ,reverse = True )
    return sortedClass[0][0]

5. 创建分类树

def createTree(dataSet , labels , imp = 0):
    '''
    思路： 
        1. 选择最佳特征创建根节点 
        2. 遍历该特征所有属性，迭代创建决策树
    
    '''
    # 获取所有的数据类别列表
    classList = [lab[-1] for lab in dataSet]
    # 递归出口
    # 如果第一个类别就是全部
    if (classList.count(classList[0]) == len(classList)):
        return classList[0]
    # 所有特征全部使用完了，但是有可能还是没有完全分开，那么就投票决定了
    if len(dataSet[0]) == 1 :
        return majorityCnt(classList)

    bestFea = chooseBestFeatureToSplit(dataSet , imp)
    print '------' , bestFea
    bestFeaLabel = labels[bestFea]
    myTree = {bestFeaLabel :{}}
    print 'diyici' , myTree
    del(labels[bestFea])
    # 
    featValues = [fea[bestFea] for fea in dataSet]
    uniqueVals = set(featValues)
    
    for value in uniqueVals:
        subLabels = labels[:]
        myTree[bestFeaLabel][value] = createTree(spiltDataSet(
                dataSet , bestFea , value) , subLabels)
        print 'dierci :----' , myTree
    return myTree

6. 预测函数

def predict(myTree , labels, preLabel):
    # 获取第一个根节点
    firstFea = myTree.keys()[0]
    # 获取孩子节点
    secondDict = myTree[firstFea]
    # 获取根节点在label中的索引
    featIndex = labels.index(firstFea)
    # 遍历根节点所有的边
    for key in secondDict.keys():
        # 判断边是与给定的相等，走向对应的分支
        if preLabel[featIndex] == key:
            if isinstance(secondDict[key] , dict):
                classLabel = predict(secondDict[key] , labels , preLabel)
            else:
                classLabel = secondDict[key]
    return classLabel

3. C4.5算法实现

note: 信息增益比 ， 分母是该特征的混乱程度，而不是该特征下类别的混乱程度，很多博客上面是有问题的

if imp == 0:  # 计算信息增益  ID3算法
            if infoGain > bestGain:
                bestFea = i
                bestGain = infoGain
else:   # 这里是计算的信息增益比 c4.5算法
   #这里计算的是某个特征条件下的熵 ， 而不是集合类别的熵
   iv = calaShannonEnt(dataSet , i)
   if iv == 0:
       continue
   infoGainRate = infoGain / iv
   if bestGainRate < infoGainRate:
       bestFea = i
       bestGainRate = infoGainRate

4. CART 分类树实现

1. 基尼指数计算

def calaGini(dataSet):
    # 计算每个类别出现的次数
    label = [ fea[-1] for fea in dataSet ]
    feaDict = {}
    for fea in label:
        if fea not in feaDict:
            feaDict[fea] = 0
        feaDict[fea] += 1
    
    gini0 = 0
    # 累加基尼指数
    for key in feaDict.keys():
        prob =  feaDict[key] / float(len(label))
        gini0 += prob * prob
    return 1 - gini0

2. 分割数据

CART是二叉树，每次都是对数据进行二类分割

def spiltDataSet(dataSet , col ,value):
    retSetLeft = []
    retSetRight= []
    for fea in dataSet:
        if fea[col] == value:
            feaVec = fea[:col]
            feaVec.extend(fea[col+1 : ])
            retSetLeft.append(feaVec)
        else:
            feaVec = fea[:col]
            feaVec.extend(fea[col+1:])
            retSetRight.append(feaVec)
    return retSetLeft , retSetRight

3. 选择最佳特征以及特征值

def chooseBestFeatureToSpilt(dataSet):
    '''
    '''
    numFeat = len(dataSet[0]) -1
    bestFeat  = -1
    minGini = 100000
    bestValue = 9999
    # 遍历所有的特征
    for col in range(numFeat):
        colList = dataSet[col]
        feat = set(colList)  
        gini = 0
        # 遍历所有的特征值
        for value in feat:
            subData1 , subData2 = spiltDataSet(dataSet , col , value)
            prob1 = len(subData1) / float(len(dataSet))
            prob2 = len(subData2) / float(len(dataSet))
            gini = prob1 * calaGini(subData1) + prob2*calaGini(subData2)
            if gini < minGini:
                minGini = gini
                bestFeat = col
                bestValue = value
    return bestFeat   , bestValue , gini

3. 创建分类树

class Tree(object):
    def __init__(self , fea , value , isLeaf = False):
        self.left = None
        self.right = None
        self.isLeaf = isLeaf
        self.fea = fea
        self.value = value
        
def createTree(dataSet , labels):
    '''
    '''
    classList = [fea[-1] for fea in dataSet]
    # 递归出口
    if classList.count(classList[0]) == len(classList):
        return Tree(None , classList[0]  , True)
    if len(dataSet[0]) == 1:
        return Tree(None ,dt.majorityCnt(classList) , True)
    # 找到最佳特征以及特征值
    fea , value , gini = chooseBestFeatureToSpilt(dataSet)
	# 创建根节点
    feaLabel = labels[fea]
    myTree = Tree(feaLabel , value)
    # 切分数据，递归创建树
    dataLeft , dataRight = spiltDataSet(dataSet , fea , value)
    newLabels = labels[:fea]
    newLabels.extend(labels[fea+1:])
    myTree.left = createTree(dataLeft , newLabels)
    myTree.right = createTree(dataRight , newLabels)
    
    return myTree

4. 预测

def predict(myTree , labels ,preVec):
    # 递归出口
    if myTree.isLeaf :
        return myTree.value
    fea = myTree.fea
    value = myTree.value
    index = labels.index(fea)
    preValue = preVec[index]
    if value == preValue:
        return predict(myTree.left , labels , preVec)
    else:
        return predict(myTree.right , labels , preVec)

5. CART 回归树实现

1. 计算误差

def calaError(dataSet  , mean0 = -1):
    
    numCount = len(dataSet)
    target = [t[-1] for t in dataSet]
    if mean0 == -1:
        mean = sum(target) / numCount
    else:
        # 这里主要是在后剪枝的时候，需要使用
        mean = mean0
    
    err = sum([ (t-mean)**2  for t in target ]) 
    return err

2. 选择最佳特征以及最佳特征值

def chooseBestFeatureToSpilt(dataSet ,ops):
    numFeat = len(dataSet[0]) - 1
    bestErr = 999999
    bestIndex = -1
    bestValue = 0
    minLeftCount = ops[1]
    minErr = ops[0]
    baseErr = calaError(dataSet)
    
    # 数据不可分，限制叶子节点中样本数量
    if len(dataSet) <= 2 *  minLeftCount:
        return None,meanValue(dataSet),baseErr
    
    for i in range(numFeat):
        colList = [fea[i] for fea in dataSet]
        feaList = set(colList)
        for feaValue in feaList:
            subLeft ,subRight = spiltDataSet(dataSet , i , feaValue)
            if len(subLeft)<minLeftCount or len(subRight)<minLeftCount:
                continue
            newErr = calaError(subLeft) + calaError(subRight)
            if newErr < bestErr:
                bestErr = newErr
                bestIndex = i
                bestValue = feaValue
    # 当数据集不可分，或者误差小于指定的阈值的情况下
    if abs(baseErr - bestErr) < minErr :
        return None,meanValue(dataSet),baseErr
    
    return bestIndex , bestValue , bestErr

3. 创建回归树

def createTree(dataSet, ops=(1, 4)):
    
    index , value ,err= chooseBestFeatureToSpilt(dataSet , ops)
    
    if index is None:
        return value
    myTree = {}
    myTree['spIndex'] = index 
    myTree['spValue'] = value
    
    subLeft , subRight = spiltDataSet(dataSet , index , value)
    myTree['left'] = createTree(subLeft , ops)
    myTree['right'] = createTree(subRight , ops)
    
    return myTree

4. 预测

def predict(myTree , preVec):
    # 出口
    if isLeaf(myTree):
        return myTree
    
    index = myTree['spIndex']
    value = myTree['spValue']
    preValue = preVec[index]
    if preValue <= value:
        return predict(myTree['left'] , preVec)
    else:
        return predict(myTree['right'] , preVec)

5. 后剪枝实现

def prune(tree , testData):
    '''
    1. 把测试数据分配到回归树的叶子节点
    2. 计算合并叶子节点前后 误差的变化
    3. 如果没有分配到数据，那么就直接把子树合并起来
    '''
    
    if len(testData) == 0:
        return getMean(tree)
    
    # 根据回归树来划分我们的测试数据
    if isTree(tree['left']) or isTree(tree['right']):
        dataLeft , dataRight = spiltDataSet(testData , tree['spIndex'] , tree['spValue'])
    
    if isTree(tree['left']):
        tree['left'] = prune(tree['left'] , dataLeft)
    if isTree(tree['right']):
        tree['right'] = prune(tree['right'] , dataRight)
    
    if isLeaf(tree['left']) and isLeaf(tree['right']):
        dataLeft , dataRight = spiltDataSet(testData , tree['spIndex'] , tree['spValue'])
        #sumErr = calaError(dataLeft) + calaError(dataRight)
        
        #sumErrMer = calaError(testData)
        #value = meanValue(testData)
        ## 这里特别注意： 这里的均值就是叶子节点的值
        sumErr = calaError(dataLeft , tree['left']) + calaError(dataRight , tree['right'])
        value = tree['left'] + (tree['right'])/float(2)
        sumErrMer = calaError(testData , value)
        if sumErr > sumErrMer:
            return value
        else:
            return tree
    else:
        return tree

github地址