Backpropagation Algorithm Implementation

最新推荐文章于 2022-06-20 12:55:05 发布

_rho

最新推荐文章于 2022-06-20 12:55:05 发布

阅读量326

点赞数

CC 4.0 BY-SA版权

分类专栏：机器学习文章标签：神经网络反向传播算法

本文链接：https://blog.youkuaiyun.com/u012841922/article/details/78992316

机器学习专栏收录该内容

14 篇文章

订阅专栏

本文介绍了一个使用三层神经网络对手写数字进行分类的实验。该网络采用Sigmoid激活函数，并通过反向传播算法进行训练。实验使用了MNIST数据集，并详细记录了训练过程中的参数变化。

数据来源：http://yann.lecun.com/exdb/mnist/

以下代码实现了一个三层的神经网络，其激活函数为 $sigmod\ function$ 。笔者运行了多次都遇到了 $s i g m o d$ 梯度消失，模型预测效果蛮差。读者也可试试其它激活函数。阅读本程序可助于深入理解神经网络和反向传播算法。

import os
import numpy as np
import random
from mnist import MNIST
from sklearn.preprocessing import LabelBinarizer

mndata = MNIST('mnist/')
training_images, training_labels = mndata.load_training()

training_labels = LabelBinarizer().fit_transform(np.array(training_labels))
training_images = np.array(training_images)
training_images = training_images/255

testing_images, testing_labels = mndata.load_testing()

testing_labels = LabelBinarizer().fit_transform(np.array(testing_labels))
testing_images = np.array(testing_images)
testing_images = testing_images/255

def sigmod(x):
    return 1/(1+np.exp(-x))
    
def linear_act(x):
    return x

def d_sigmod(y):
    return y*(1-y)

def d_linear_act(x):
    x[True] = 1
    return x

class NerualNetwork():
    def __init__(self,layers,lr=0.1):
        self.layers = layers
        self.num_of_layers = np.shape(layers)[0]
        self.lr = lr
        self.W = [np.nan]
        self.b = [np.nan]
        self.a_last = np.array([])
        for i in range(self.num_of_layers-1):
            self.W.append(np.random.random((layers[i+1],layers[i]))*2-1)
            self.b.append(np.random.random((layers[i+1],1))*2-1)
            
    def forword_propagation(self,X):
        s = [np.nan]
        a = [X]
        for l in range(1,self.num_of_layers-1):
            s.append(np.dot(self.W[l],a[l-1])+self.b[l])
            a.append(sigmod(s[l]))
        s.append(np.dot(self.W[self.num_of_layers-1],a[self.num_of_layers-2])+self.b[self.num_of_layers-1])
        a.append(sigmod(s[self.num_of_layers-1]))
        
        return s,a
    
    def predict(self,X):
        s,a = self.forword_propagation(X)
        prediction = a[-1]
        return prediction
    
    def back_propagation(self,s,a,Y):
        d_a_last = -2*(Y-a[-1])/self.layers[-1]
        d_s_last = np.dot(np.diag(d_sigmod(a[-1][:,0])), d_a_last[:,0].reshape(10,1))
        
        for m in range(1,np.shape(Y)[1]):
            d_s_last = np.append(d_s_last,
                                 np.dot(np.diag(d_sigmod(a[-1][:,m])), d_a_last[:,m].reshape(10,1)), 1)
        
        ds = [d_s_last]
        for i in list(reversed(range(self.num_of_layers-2))):            
            d_s_hide = np.dot(np.diag(d_sigmod(a[i+1][:,0])),
                              np.dot(self.W[i+2].T,ds[-1][:,0].reshape(self.layers[i+2],1)))
            
            for m in range(1,np.shape(Y)[1]):
                d_s_hide = np.append(d_s_hide,
                                     np.dot(np.diag(d_sigmod(a[i+1][:,m])),
                                            np.dot(self.W[i+2].T,ds[-1][:,m].reshape(self.layers[i+2],1))), 1)
            ds.append(d_s_hide)
        
        ds.append(np.nan)
        ds = list(reversed(ds))
        
        with open('ds.txt','a') as f_ds:
            for l in range(1,self.num_of_layers):
                f_ds.write('\nLayer '+str(l)+'\n')
                f_ds.write('\nds:'+str(np.shape(ds[l]))+'\n')
                f_ds.write(str(ds[l])+'\n')
        
        self.update_param(a,ds)
        
    def update_param(self,a,ds):
        for i in range(1,self.num_of_layers):
            sum_of_gradient_w = np.dot(ds[i][:,0].reshape(self.layers[i],1),
                                       a[i-1][:,0].reshape(self.layers[i-1],1).T)
            sum_of_gradient_b = ds[i][:,0].reshape(self.layers[i],1)
            for m in range(1,np.shape(ds[i])[1]):
                sum_of_gradient_w += np.dot(ds[i][:,m].reshape(self.layers[i],1),
                                         a[i-1][:,m].reshape(self.layers[i-1],1).T)
                sum_of_gradient_b += ds[i][:,m].reshape(self.layers[i],1)
                
            self.W[i] -= self.lr*sum_of_gradient_w/(np.shape(ds[i])[1])
            self.b[i] -= self.lr*sum_of_gradient_b/(np.shape(ds[i])[1])
            
    def train(self,X,Y,epochs=10000, batch_size=500):
        if os.path.isfile('parameters.txt'):
            os.remove('parameters.txt')
        if os.path.isfile('activations.txt'):
            os.remove('activations.txt') 
        if os.path.isfile('costs.txt'):
            os.remove('costs.txt')
        if os.path.isfile('ds.txt'):
            os.remove('ds.txt')
            
        for i in range(epochs):
            sample_index = random.sample(list(range(np.shape(Y)[1])), batch_size)
            X_in_batch = X[:,sample_index]
            Y_in_batch = Y[:,sample_index]
            if i > 200:
                self.lr /= 10
            self.train_in_batch(X_in_batch,Y_in_batch,i)
            
    def train_in_batch(self,X,Y,i):
        s,a = self.forword_propagation(X)
        
        if 1:
            with open('parameters.txt','a') as f_parameters:
                f_parameters.write('\n\nAt iteration:'+str(i)+'\n')
                for l in range(1,self.num_of_layers):
                    f_parameters.write('\nLayer '+str(l)+'\n')
                    f_parameters.write('\nW:'+str(np.shape(self.W[l]))+'\n')
                    f_parameters.write(str(self.W[l])+'\n')

                    f_parameters.write('\nb:'+str(np.shape(self.b[l]))+'\n')
                    f_parameters.write(str(self.b[l])+'\n')

            with open('activations.txt','a') as f_activations:
                f_activations.write('\n\nAt iteration:'+str(i)+'\n')
                for l in range(0,self.num_of_layers):
                    f_activations.write('\nLayer '+str(l)+'\n')
                    f_activations.write('\ns:'+str(np.shape(s[l]))+'\n')
                    f_activations.write(str(s[l])+'\n')
                    
                    f_activations.write('\na:'+str(np.shape(a[l]))+'\n')
                    f_activations.write(str(a[l])+'\n')
            
            with open('ds.txt','a') as f_ds:
                f_ds.write('\n\nAt iteration:'+str(i)+'\n')
                
        self.a_last = a[-1]
        
        cost = np.zeros((1,1))
        for j in range(np.shape(Y)[1]):
            residual = Y[:,j].reshape(10,1)-self.a_last[:,j].reshape(10,1)
            cost += np.dot(residual.T,residual)
                
        with open('costs.txt','a') as f_costs:
            f_costs.write(str(cost/(np.shape(Y)[1]))+'\n')
        self.back_propagation(s,a,Y)

training_X = training_images.T
training_Y = training_labels.T
testing_X = testing_images.T
testing_Y = testing_labels.T
nn = NerualNetwork(np.array([784,196,49,10]))
nn.train(training_X,training_Y,epochs=1000)

import matplotlib.pyplot as plt
%matplotlib inline

with open('costs.txt') as f:
    y = []
    for eachline in f:
        y.append(float(eachline[3:][:-3]))
        
x = range(len(y))

plt.figure()
plt.plot(x,y)
plt.title('MNIST: 3-layer Nerual Networks using Sigmod Function')
plt.xlabel('steps')
plt.ylabel('cost')

这里写图片描述

testing_Y[:,list(range(5))]

array([[0, 0, 0, 1, 0],
       [0, 0, 1, 0, 0],
       [0, 1, 0, 0, 0],
       [0, 0, 0, 0, 0],
       [0, 0, 0, 0, 1],
       [0, 0, 0, 0, 0],
       [0, 0, 0, 0, 0],
       [1, 0, 0, 0, 0],
       [0, 0, 0, 0, 0],
       [0, 0, 0, 0, 0]])

nn.predict(testing_X[:,list(range(5))])

array([[ 0.08852091,  0.07712739,  0.24558161,  0.24253024,  0.09615363],
       [ 0.06009189,  0.01842345,  0.02953039,  0.10981609,  0.12261026],
       [ 0.42194989,  0.26216405,  0.0929229 ,  0.26748772,  0.22785839],
       [ 0.06863829,  0.07286736,  0.17960174,  0.18293914,  0.0487555 ],
       [ 0.02216657,  0.12481555,  0.21624219,  0.04304522,  0.04131497],
       [ 0.03851181,  0.03991082,  0.0137592 ,  0.01167841,  0.0100367 ],
       [ 0.08333821,  0.33273348,  0.100765  ,  0.11475403,  0.23539435],
       [ 0.00238281,  0.00640189,  0.00788065,  0.02162292,  0.00926097],
       [ 0.03512258,  0.00634047,  0.00719783,  0.00774136,  0.00957267],
       [ 0.17760028,  0.27654505,  0.02500085,  0.11724239,  0.08195934]])

training_Y[:,list(range(5))]

array([[0, 1, 0, 0, 0],
       [0, 0, 0, 1, 0],
       [0, 0, 0, 0, 0],
       [0, 0, 0, 0, 0],
       [0, 0, 1, 0, 0],
       [1, 0, 0, 0, 0],
       [0, 0, 0, 0, 0],
       [0, 0, 0, 0, 0],
       [0, 0, 0, 0, 0],
       [0, 0, 0, 0, 1]])

nn.predict(training_X[:,list(range(5))])

array([[ 0.14088619,  0.43447728,  0.00969862,  0.11855052,  0.3753252 ],
       [ 0.07504254,  0.18396886,  0.01375044,  0.02276849,  0.17391514],
       [ 0.18822948,  0.24832705,  0.48081013,  0.13560714,  0.13318023],
       [ 0.10294189,  0.28654818,  0.14425579,  0.14219287,  0.040433  ],
       [ 0.06517029,  0.08686016,  0.14041424,  0.38237368,  0.14986737],
       [ 0.0906671 ,  0.04954782,  0.01434389,  0.00715321,  0.01533208],
       [ 0.15485006,  0.04465222,  0.01622938,  0.18944488,  0.24251637],
       [ 0.10819804,  0.01239158,  0.03952334,  0.01231031,  0.00277187],
       [ 0.05063162,  0.01122012,  0.04046127,  0.01782585,  0.02276523],
       [ 0.34232939,  0.01950765,  0.18545528,  0.05277625,  0.03637404]])