吴恩达机器学习 笔记五 BP神经网络

  由于硕士期间上过这门课，对神经网络已经很熟了，一些细枝末节就不从头说了。这篇文章只记录我的知识水平之上的一些要点。先祭出两篇大牛文章，我觉得不能比他们写得更好了。
1、https://blog.youkuaiyun.com/zhongkejingwang/article/details/44514073
2、https://zhuanlan.zhihu.com/p/38006693

1. 神经网络比较重点的几个地方

1.1 变换坐标

  其实确切的原因是因为各层之间权值组成的矩阵对坐标系进行了改变，比如，旋转，拉伸，降维。具体可以参考我的另外一篇关于矩阵的文章，传送门在此

1.2 激活函数

  软饱和的激活函数可以将直线掰弯。矩阵操作线性的，但是经过激活函数之后就可以非线性。

1.3 反向传播的是误差

  每个权重的梯度都等于与其相连的前一层节点的输出乘以与其相连的后一层的反向传播的输出（包含误差）

2. tensorflow实现BP神经网络的拟合

'''
Author       :  vivalazxp
Date         :  9/2/2018
Description  :  neural network non_linear regression
'''
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
'''
Description    : create data for neural network non-linear regression
Param          : numData         number of training data
                 sigma           power of noise
                 horizon_limit   horizontal limit
Return         : horizon_data    horizontal axis  shape=(1,numData)
                 vertical_data   vertical axis    shape=(1,numData) 
'''
def create_data_nn_non_lin_reg(numData,sigma, horizon_limit):
    horizon_data = horizon_limit * 2*(np.random.rand(numData)-0.5)[np.newaxis, :]
    vertical_data = np.sin(horizon_data)
    # add noise
    vertical_data = vertical_data + sigma*np.random.rand(numData)
    print('-------------- create data sucessfully --------------')
    return horizon_data, vertical_data
'''
Description  : add one neural network layer
Param        : @input_data     input data
               @input_size     number of neurons in the input layer 
               @output_size    number of nuerons in the output layer
Return       : @Weight    weight of the layer added
               @Bias      bias of the layer added
               @output_data    output of the layer added
'''
def add_layer(input_data, input_size, output_size, activation_function):
    Weight = tf.Variable(tf.random_normal([output_size, input_size]))
    Bias = tf.Variable(tf.random_normal([output_size, 1]))
    WX_plus_B = tf.matmul(Weight, input_data) + Bias
    if activation_function == None:
        ouput_data = WX_plus_B
    else:
        ouput_data = activation_function(WX_plus_B)
    print('------------- add one nn layer sucessfully --------------')
    return Weight, Bias, ouput_data
'''
Description     :  use tensorflow to complete neural network with nonlinear regression
Param           :  @hid_layer_size   number of neurons in the hidden layer
                   @out_layer_size   number of neurons in the output layer
                   @alpha    learning rate
                   @steps    sum learning steps
Return          :  @Weight_hid    weight of the hidden layer
                   @Weight_out    weight of the output layer
                   @Bias_hid      bias of the hidded layer
                   @Bias_out      bias of the output layer
                   @loss          cost variation in the training
'''
def tf_nn_clas(horizon_data, vertical_data, in_layer_size, hid_layer_size, out_layer_size, alpha, stpes):
    horizon_from_data = tf.placeholder(tf.float32)
    vertical_from_data = tf.placeholder(tf.float32)
    # hidder layer, sigmoid activate function
    Weight_hid, Bias_hid, out_hid = add_layer(horizon_from_data, in_layer_size, hid_layer_size, activation_function=tf.nn.sigmoid)
    # output layer, no activate function
    Weight_out, Bias_out, class_pred = add_layer(out_hid, hid_layer_size, out_layer_size, activation_function=None)
    # cost and optimizer
    cost = tf.reduce_mean(tf.square(vertical_from_data - class_pred))
    optimizer = tf.train.GradientDescentOptimizer(alpha).minimize(cost)
    # session
    sess = tf.Session()
    init = tf.global_variables_initializer()
    sess.run(init)
    print('----------train started------------')
    loss = np.zeros(steps)
    for step in range(steps):
        sess.run(optimizer, feed_dict={horizon_from_data:horizon_data, vertical_from_data:vertical_data})
        loss[step] = sess.run(cost, feed_dict={horizon_from_data:horizon_data, vertical_from_data:vertical_data})
    print('-----------train completed---------------')
    Weight_hid = sess.run(Weight_hid)
    Weight_out = sess.run(Weight_out)
    Bias_hid = sess.run(Bias_hid)
    Bias_out = sess.run(Bias_out)
    return Weight_hid ,Weight_out, Bias_hid, Bias_out, loss

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

if __name__ == "__main__":
    numData = 1000
    sigma = 1
    horizon_limit = 3

    alpha = 0.2
    steps = 2000

    dynam = 5
    in_layer_size = 1
    out_layer_size = 1
    hid_layer_size = np.ceil(np.sqrt(in_layer_size+out_layer_size)).astype(np.int32) + dynam

    horizon_data, vertical_data = create_data_nn_non_lin_reg(numData, sigma, horizon_limit)
    Weight_hid, Weight_out, Bias_hid, Bias_out, loss = tf_nn_clas(horizon_data,vertical_data,in_layer_size, hid_layer_size,out_layer_size,alpha,steps)

    # regression curve
    plt.figure(1)
    num_fitting = 200
    horizon_fit = np.linspace(-horizon_limit, horizon_limit, num_fitting)[np.newaxis,:]
    vertical_fit = np.zeros(num_fitting)
    output_hid = sigmoid(Weight_hid*horizon_fit + np.tile(Bias_hid,(1,num_fitting)))
    vertical_fit = np.matmul(Weight_out, output_hid) + np.tile(Bias_out,(1,num_fitting))
    plt.scatter(horizon_data, vertical_data, marker='o', label='training data')
    plt.plot(horizon_fit[0,:], vertical_fit[0,:], 'r', label='regression line')
    plt.legend()
    plt.xlabel('horizontal axis')
    plt.ylabel('vertical axis')
    plt.title('nn non linear regression')

    # cost variation
    plt.figure(2)
    plt.plot(range(steps), loss)
    plt.xlabel('step')
    plt.ylabel('loss')
    plt.title('loss variation in nn non linear regression')

    plt.show()

3. tensorflow实现BP神经网络的二分类

'''
Author       :  vivalazxp
Date         :  9/2/2018
Description  :  neural network non_linear regression
'''
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
'''
Description    : create data for neural network classification
Param          : numData         number of training data
                 vertical_limit           power of noise
                 horizon_limit   horizontal limit
Return         : axis_data       horizontal and vertical axis  shape=(2,numData)
                 class_data      vertical axis    shape=(numData,) 
'''
def create_data_nn_clas(numData, vertical_limit, horizon_limit):
    axis_data = np.zeros([2,numData])
    axis_data[0,:] = horizon_limit * 2*(np.random.rand(numData)-0.5)
    axis_data[1,:] = vertical_limit * 2*(np.random.rand(numData) - 0.5)
    class_data = np.zeros(numData)
    for index in range(numData):
        if np.sin(axis_data[0,index]) - axis_data[1,index] > 0:
            class_data[index] = 1
            plt.scatter(axis_data[0,index],axis_data[1,index],c='g')
        else:
            class_data[index] = 0
            plt.scatter(axis_data[0, index], axis_data[1, index], c='b')
    print('-------------- create data sucessfully --------------')
    return axis_data, class_data
'''
Description  : add one neural network layer
Param        : @input_data     input_data data
               @input_size     size of input_data 
               @output_size    size of output
'''
def add_layer(input_data, input_size, output_size, activation_function):
    Weight = tf.Variable(tf.random_normal([output_size, input_size]))
    Bias = tf.Variable(tf.random_normal([output_size, 1]))
    WX_plus_B = tf.matmul(Weight, input_data) + Bias
    if activation_function == None:
        ouput_data = WX_plus_B
    else:
        ouput_data = activation_function(WX_plus_B)
    print('------------- add one nn layer sucessfully --------------')
    return Weight, Bias, ouput_data
'''
Description     :  use tensorflow to complete neural network classification
Param           :  @hid_layer_size   number of neurons in the hidden layer
                   @out_layer_size   number of neurons in the output layer
                   @alpha    learning rate
                   @steps    sum learning steps
Return          :  @Weight_hid    weight of the hidden layer
                   @Weight_out    weight of the output layer
                   @Bias_hid      bias of the hidded layer
                   @Bias_out      bias of the output layer
                   @loss          cost variation in the training
'''
def tf_nn_clas(axis_data, class_data, in_layer_size, hid_layer_size, out_layer_size, alpha, stpes):
    axis_from_data = tf.placeholder(tf.float32)
    class_from_data = tf.placeholder(tf.float32)
    # hidder layer, sigmoid activate function
    Weight_hid, Bias_hid, out_hid = add_layer(axis_from_data, in_layer_size, hid_layer_size, activation_function=tf.nn.sigmoid)
    # output layer, no activate function
    Weight_out, Bias_out, class_pred = add_layer(out_hid, hid_layer_size, out_layer_size, activation_function=tf.nn.sigmoid)
    # cost and optimizer
    cost = tf.reduce_mean(- class_from_data * tf.log(class_pred) \
                          - (1 - class_from_data) * tf.log(1 - class_pred))
    optimizer = tf.train.GradientDescentOptimizer(alpha).minimize(cost)
    # session
    sess = tf.Session()
    init = tf.global_variables_initializer()
    sess.run(init)
    print('----------train started------------')
    loss = np.zeros(steps)
    for step in range(steps):
        sess.run(optimizer, feed_dict={axis_from_data:axis_data, class_from_data:class_data})
        loss[step] = sess.run(cost, feed_dict={axis_from_data:axis_data, class_from_data:class_data})
        if step % 500 == 0:
            print(step)
    print('-----------train completed---------------')
    Weight_hid = sess.run(Weight_hid)
    Weight_out = sess.run(Weight_out)
    Bias_hid = sess.run(Bias_hid)
    Bias_out = sess.run(Bias_out)
    return Weight_hid ,Weight_out, Bias_hid, Bias_out, loss

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

if __name__ == "__main__":
    numData = 1000
    vertical_limit = 2
    horizon_limit = 3

    alpha = 1
    steps = 2000

    neur_dynam = 5
    in_layer_size = 2
    out_layer_size = 1

    hid_layer_size = np.ceil(np.sqrt(in_layer_size+out_layer_size)).astype(np.int32) + neur_dynam
    axis_data, class_data = create_data_nn_clas(numData, vertical_limit, horizon_limit)
    Weight_hid, Weight_out, Bias_hid, Bias_out, loss = tf_nn_clas(axis_data,class_data,in_layer_size, hid_layer_size,out_layer_size,alpha,steps)

    # classification curve
    num_clas = 200
    horizon_clas = np.linspace(-horizon_limit, horizon_limit, num_clas)
    vertical_tmp = np.linspace(-vertical_limit, vertical_limit, num_clas)
    vertical_clas = np.zeros(num_clas)
    for index_horiz in range(num_clas):
        for index_verti in range(num_clas):
            input = np.array([horizon_clas[index_horiz],vertical_tmp[index_verti]])[:,np.newaxis]
            ouput_hid = sigmoid(np.matmul(Weight_hid, input) + Bias_hid)
            output = np.matmul(Weight_out,ouput_hid) + Bias_out
            if output > -0.2 and output < 0.2:
                vertical_clas[index_horiz] = vertical_tmp[index_verti]
                continue
    plt.plot(horizon_clas, vertical_clas, 'r', label='classification line')
    plt.legend()
    plt.xlabel('horizontal axis')
    plt.ylabel('vertical axis')
    plt.title('nn classification')

    # cost variation
    plt.figure()
    plt.plot(range(steps), loss)
    plt.xlabel('step')
    plt.ylabel('loss')
    plt.title('loss variation in nn classification')

    plt.show()