k-近邻算法

k-近邻算法

k-近邻算法采用不同特征之间的距离方法进行分类。在训练样本集中每条记录都存在标签，样本集中每一数据与所属分类的对应关系。输入没有标签的新数据后，将新数据的特征值和样本集中数据对应的特征进行比较。一般情况，选择k个最相似数据中出现次数最多的分类，作为新数据的分类。以电影分类为例，电影特征空间为打斗镜头和接吻镜头。

电影名称	打斗镜头	接吻镜头	电影类型
canifonia man	3	104	爱情电影
He is Not Really into Dudes	2	100	爱情电影
Beautiful Woman	1	81	爱情电影
Kevin Longblade	101	10	动作电影
Robo Slayer 3000	99	5	动作片
Amped II	98	2	动作片
Ni Dongde	100	100	爱情动作片
To be Predicted	3	60	？

最后一条记录是待预测的电影，计算待预测电影和带有标签的记录之间的距离，找到距离最近的k近邻个电影，在k近邻中类型分布最多的电影标签即为待预测电影的类型。显然知道待预测的电影类型是动作片。

k近邻算法流程
1. 收集数据
2. 准备数据
3. 分析数据
4. 计算距离
5. 距离排序
6. k近邻样本投票

示例代码

from numpy import *
import operator
# k近邻分类，对k近邻进行投票分类
def classify0(inX, dataSet, labels, k):
    # get the dimention of dataSet
    dataSetSize = dataSet.shape[0]

    # create diffMat (dataSetSize * 1)
    diffMat = tile(inX, (dataSetSize,1)) - dataSet

    sqDiffMat = diffMat**2
    sqDistance = sqDiffMat.sum(axis = 1)
    distances = sqDistance**0.5

    # get the index array of distances sorted
    sortedDistIndices = distances.argsort()

    classCount = {}
    for i in range(k):
        voteIlabel = labels[sortedDistIndices[i]]
        classCount[voteIlabel] = classCount.get(voteIlabel,0) + 1
    sortedClassCount = sorted(classCount.iteritems(), key = operator.itemgetter(1), reverse = True)
    # sortedClassCount = sorted(classCount.itertems(), key = lambda d: d[1], reverse = True)
    return sortedClassCount[0][0]

# 从文件中读入数据到数组中
def file2matrix(filename):
    fr = open(filename)
    arrayOLines = fr.readlines()
    numberOfLines = len(arrayOLines)
    returnMat = zeros((numberOfLines, 3))
    classLabelVector = []
    index = 0
    for line in arrayOLines:
        line = line.strip()
        listFromLine = line.split('\t')
        returnMat[index,:] = listFromLine[0:3]
        classLabelVector.append(listFromLine[-1])
        index += 1
    return returnMat, classLabelVector

# 不同属性分布差别较大，需进行归一化
def autoNorm(dataSet):
    minVals = dataSet.min(0)
    maxVals = dataSet.max(0)
    ranges = maxVals - minVals
    normDataSet = zeros(shape(dataSet))
    m = dataSet.shape[0]
    normDataSet = dataSet - tile(minVals,(m,1))
    normDataSet = normDataSet/tile(ranges, (m,1))
    return normDataSet,ranges, minVals

# 分类测试
def datingClassTest():
    hoRatio = 0.10
    datingDataMat, datingLabels = file2matrix('datingTestSet2.txt')
    normMat, ranges, minVals = autoNorm(datingDataMat)
    m = normMat.shape[0]
    numTestVecs = int(m*hoRatio)
    errorCount = 0.0
    for i in range(numTestVecs):
        classifierResult = classify0(normMat[i,:],normMat[numTestVecs:m,:], datingLabels[numTestVecs:m], 3)
        print('the classifier cameback with: %s, the real answer is: %s' %(classifierResult, datingLabels[i]))
        if(classifierResult!=datingLabels[i]):
            errorCount +=1.0
    print("the total error rate is: %f" %(errorCount/float(numTestVecs)))

def classifyPerson():
    resultList = ['not at all','in small doses', 'in large doses']
    percenTats = float(input('percentage of time spent playing video games?'))
    ffMiles = float(input('frequent filter miles earned per year?'))
    iceCream = float(input('liters of ice cream consumed per year?'))
    datingDataMat, datingLabels = file2matrix('datingTestSet2.txt')
    normMat, ranges, minVals = autoNorm(datingDataMat)
    inArr = array([ffMiles, percenTats, iceCream])
    classifierResult = classify0((inArr-minVals)/ranges, normMat, datingLabels, 3)
    print("You will probably like this person:", resultList[int(classifierResult) - 1])

# 图像数据转为向量，图像以文本方式存储
def img2vector(filename):
    returnVect = zeros((1, 1024))
    fr = open(filename)
    for i in range(32):
        lineStr = fr.readline()
        for j in range(32):
            returnVect[0, 32*i + j] = int(lineStr[j])
    return returnVect
# 对文本数字进行分类
def handwritingClassTest():
    hwLabels = []
    trainingFileList = listdir('trainingDigits')
    m = len(trainingFileList)
    trainingMat  = zeros((m, 1024))
    for i in range(m):
        fileNameStr = trainingFileList[i]
        fileStr = fileNameStr.split('.')[0]
        classNumStr = int(fileStr.split('_')[0])
        hwLabels.append(classNumStr)
        trainingMat[i,:] = img2vector('trainingDigits/%s' % fileNameStr)
    testFileList = listdir('testDigits')
    errorCount = 0.0
    mTest = len(testFileList)
    for i in range(mTest):
        fileNameStr = testFileList[i]
        fileStr = fileNameStr.split('.')[0]
        classNumStr = int(fileStr.split('_')[0])
        vectorUnderTest = img2vector('testDigits/%s' % fileNameStr)
        classifierResult = classify0(vectorUnderTest, trainingMat, hwLabels, 3)
        print('the classifier came back with: %d, the real answer is: %d' %(classifierResult,classNumStr))
        if(classifierResult != classNumStr):
            errorCount += 1.0
    print('\nthe total number of errors is: %d' % errorCount)
    print('\nthe total error rate is: %f' % (errorCount/float(mTest)))

    #对文本分类进行测试
    handwritingClassTest()

the total number of errors is: 13
the total error rate is: 0.013742

算法特点

优点：精度高、对异常值不敏感、无数据输入假定
缺点：计算复杂度高、空间
适用数据范围：数值型和标称型

k近邻算法思路较为简单，自己感觉效率一般