深度学习入门

参考资料：《深度学习入门 基于python的理论与实现》

误差反向传播法的神经网络的实现

激活函数层:

Relu层:

class Relu:
    def __init__(self):
    	# 由True和False构成的Numpy数组，将正向传播时输入x小于等于0的地方保存为True，其他地方保存为False
        self.mask = None 

    def forward(self, x):
        self.mask = (x <= 0)
        out = x.copy()
        out[self.mask] = 0

        return out

    def backward(self, dout):
    
        dout[self.mask] = 0
        dx = dout

        return dx

Sigmoid层：

反向传播推导如下
而在Sigmoid函数中，y = 1 / (1 + exp(-x))，exp(-x) = 1/y - 1
代入后有 y^2 * exp(-x) = y(1 - y)
代码如下

class Sigmoid:
    def __init__(self):
    	# 用来保存前向传输时的结果
        self.out = None

    def forward(self, x):
        out = 1 / (1 + np.exp(-x))
        self.out = out

        return out
    def backward(self, dout):
        dx = dout * (1.0 - self.out) * self.out

        return dx

Affine层:

class Affine:
    def __init__(self, W, b):
        self.W = W
        self.b = b
        self.x = None # 保存输入的x
        self.dW = None # 记录dW
        self.db = None # 记录db

    def forward(self, x):
        self.x = x 
        out = np.dot(x, self.W) + self.b
        return out

    def backward(self, dout):
        dx = np.dot(dout, self.W.T)
        self.dW = np.dot(self.x.T, dout)
        # 正向传播时，偏置会被加到每一个数据上，因此反向传播时，各个数据的反向传播的值需要汇总为偏置的元素
        self.db = np.sum(dout, axis=0)
        return dx

Softmax-with-Loss层：

class SoftmanWithLoss:
    def __init__(self):
        self.y = None
        self.t = None
        self.loss = None

    def forward(self, x, t):
        self.y = softmax(x)
        self.t = t
        self.loss = cross_entropy_error(self.y, self.t)
        return self.loss
    
    def backward(self, dout=1):
        batch_size = self.t.shape[0]
        dx = (self.y - self.t) / batch_size # 传递给前面层的是单个数据的误差
        return dx
        
def softmax(x):
    if x.ndim == 2:
        x = x.T
        x = x - np.max(x, axis=0)
        y = np.exp(x) / np.sum(np.exp(x), axis=0)
        return y.T

    x = x - np.max(x)  # 溢出对策
    return np.exp(x) / np.sum(np.exp(x))

def cross_entropy_error(y, t):
    if y.ndim == 1:
        t = t.reshape(1, t.size)
        y = y.reshape(1, y.size)

    # 监督数据是one-hot-vector的情况下，转换为正确解标签的索引
    if t.size == y.size:
        t = t.argmax(axis=1)

    batch_size = y.shape[0]
    return -np.sum(np.log(y[np.arange(batch_size), t] + 1e-7)) / batch_size

神经网络的实现:

TwoLayerNet:

class TwoLayerNet:
    def __init__(self, input_size, hidden_size, output_size, weight_init_std=0.01):
        # 初始化权重
        self.params = {}
        self.params['W1'] = weight_init_std * np.random.randn(input_size, hidden_size)  # 初始化W1
        self.params['b1'] = np.zeros(hidden_size)  # 初始化b1
        self.params['W2'] = weight_init_std * np.random.randn(hidden_size, output_size)  # 初始化W2
        self.params['b2'] = np.zeros(output_size)  # 初始化b2

        # 生成层
        self.layers = OrderedDict()  # 生成有序字典，可以记住向字典里添加元素的顺序
        self.layers['Affine1'] = Affine(self.params['W1'], self.params['b1'])  # 
        self.layers['Relu1'] = Relu()  # 
        self.layers['Affine2'] = Affine(self.params['W2'], self.params['b2'])  # 

        self.lastLayer = SoftmaxWithLoss()  # 激活函数

    def predict(self, x):
        for layer in self.layers.values():
            x = layer.forward(x)
        return x

    def loss(self, x, t):
        y = self.predict(x)

        return self.lastLayer.forward(y, t)  
        
    def accuracy(self, x, t):
        y = self.predict(x)
        y = np.argmax(y, axis=1)
        if t.ndim != 1: t = np.argmax(t, axis=1)
        accuracy = np.sum(y == t) / float(x.shape[0])
        return accuracy
        
    def gradient(self, x, t):
        # forward 
        self.loss(x, t)

        # backward
        dout = 1
        dout = self.lastLayer.backward(dout)

        layers = list(self.layers.values())
        layers.reverse()
        for layer in layers:
            dout = layer.backward(dout)

        grads = {}
        grads['W1'] = self.layers['Affine1'].dW
        grads['b1'] = self.layers['Affine1'].db
        grads['W2'] = self.layers['Affine2'].dW
        grads['b2'] = self.layers['Affine2'].db

使用误差反向传播法的学习:

import sys, os
sys.path.append(os.pardir)
import numpy as np
from mnist import load_mnist
from TwoLayerNet import *

# 读入数据
(x_train, t_train), (x_test, t_test) = load_mnist(normalize=True, one_hot_label=True)

network = TwoLayerNet(input_size=784, hidden_size=50, output_size=10)

iter_num = 10000
train_size = x_train.shape[0]
batch_size = 100
learning_rate = 0.1
train_loss_list = []
train_acc_list = []
test_acc_list = []

iter_per_epoch = max(train_size / batch_size, 1)

for i in range(iter_num):
    batch_mask = np.random.choice(train_size, batch_size)
    x_batch = x_train[batch_mask]
    t_batch = t_train[batch_mask]

    # 反向误差法求梯度
    grad = network.gradient(x_batch, t_batch)

    # 更新
    for key in ('W1', 'b1', 'W2', 'b2'):
        network.params[key] -= learning_rate * grad[key]

    loss = network.loss(x_batch, t_batch)
    train_loss_list.append(loss)

    if i % iter_per_epoch == 0:
        train_acc = network.accuracy(x_train, t_train)
        test_acc = network.accuracy(x_test, t_test)
        train_acc_list.append(train_acc)
        test_acc_list.append(test_acc)
        print(train_acc, test_acc)