CNN学习日志

LeNet-5 模型给我的初体验非常不好，目前为止还没有实际的感觉到卷积的优点，或者是LeNET-5的优点，以下是完整训练代码：卷积层-池化层-卷积层-池化层-全连接-全连接
这里有个问题：学到现在我并不觉得深度学习的合理性很好，大部分观点都是基于日常生活经验就是猜罢了(或许日后会为今天的言论感到惭愧，但我从传统机器学习理论结束之后就觉得因果关系是未来)
代码如下：【来自实战google深度学习框架】

import tensorflow  as tf
# 配置神经网络的参数
input_node = 784
output_node = 10
layer1_node = 500
IMAGE_SHAPE = 28
NUM_CHANELS = 1
NUM_LABELS = 10
# 配置卷积层的尺寸和参数
CONV1_DEEP = 32
CONV1_SIZE = 5
CONV2_DEEP = 64
CONV2_SIZE = 5
# 全连接层的输出节点个数
FC_SIZE = 512
# 最后全连接层的参数【节点个数】
FC_SIZE = 512
def get_weight_variable(shape,regularizer):
    weights = tf.get_variable('weights',shape,initializer = tf.truncated_normal_initializer(stddev=0.1))
    if regularizer != None:
        tf.add_to_collection('losses',regularizer(weights))
    return weights

#定义神经网络前向传播的过程
    
def inference(input_tensor,train,regularizer):
    # 声明第一层卷积层
    with tf.variable_scope('layer1_conv1'):
        conv1_weights = tf.get_variable('weight',[CONV1_SIZE,CONV1_SIZE,NUM_CHANELS,CONV1_DEEP],
                                        initializer=tf.truncated_normal_initializer(stddev = 0.1))
        conv1_biases = tf.get_variable(
                'bias',[CONV1_DEEP],initializer=tf.constant_initializer(0.0))
        #使用边长为5，深度为32的过滤器，步幅为1，使用相同卷积：全0填充
        conv1 = tf.nn.conv2d(
                input_tensor,conv1_weights,strides=[1,1,1,1],padding='SAME')
        relu1 = tf.nn.relu(tf.nn.bias_add(conv1,conv1_biases))
        
    # 声明第二层池化层
    with tf.variable_scope('layer2_pool1'):
        # 采用最大池化，过滤器边长为2，步幅为2，全0填充
        pool1  = tf.nn.max_pool(
                relu1,ksize = [1,2,2,1],strides = [1,2,2,1],padding='SAME')
    # 声明第三层卷积层
    with tf.variable_scope('layer3_conv2'):
        conv2_weights = tf.get_variable('weight',[CONV2_SIZE,CONV2_SIZE,CONV1_DEEP,CONV2_DEEP],
                                        initializer=tf.truncated_normal_initializer(stddev=0.1))
        conv2_biases = tf.get_variable(
                'bias',[CONV2_DEEP],initializer=tf.constant_initializer(0.0))
        # 使用边长为5，深度为64.步幅为1，全0填充的卷积核
        conv2 = tf.nn.conv2d(
                pool1,conv2_weights,strides=[1,1,1,1],padding='SAME')
        relu2 = tf.nn.relu(tf.nn.bias_add(conv2,conv2_biases))
    # 声明第四层池化层
    with tf.variable_scope('layer4_pool2'):
        pool2 = tf.nn.max_pool(
                relu2,ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME')
        
    # 因为从第五层开始，网络结构转换成全连接层，这里需要将第四层的张量拉直为向量
    pool_shape = pool2.get_shape().as_list()
    print(pool_shape)
    nodes = pool_shape[1]*pool_shape[2]*pool_shape[3]
    reshaped = tf.reshape(pool2,[pool_shape[0],nodes])
    # 第五层的额输入将会是一组长度为3136的向量，输出长度为512
    with tf.variable_scope('layer5_fc1'):
        fc1_weights = tf.get_variable('weight',[nodes,FC_SIZE],
                                      initializer=tf.truncated_normal_initializer(stddev=0.1)) 
        if regularizer != None:
            tf.add_to_collection('losses',regularizer(fc1_weights)) 
        fc1_biases = tf.get_variable('bias',[FC_SIZE],
                                     initializer=tf.constant_initializer(0.1))

        fc1 = tf.nn.relu(tf.matmul(reshaped,fc1_weights) + fc1_biases)
        if train: tf.nn.dropout(fc1,0.5) # dropout函数防止过拟合
    with tf.variable_scope('layer6_fc2'):
        fc2_weights = tf.get_variable('weight',[FC_SIZE,NUM_LABELS],
                                      initializer=tf.truncated_normal_initializer(stddev=0.1))
        if regularizer != None:
            tf.add_to_collection('losses',regularizer(fc2_weights)) 
        fc2_biases = tf.get_variable('bais',[NUM_LABELS],
                                     initializer=tf.constant_initializer(0.1))
        logit = tf.matmul(fc1,fc2_weights)+fc2_biases
        
    return logit

下面是训练代码，上面的代码保存为 mnist_inference.py

import os
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

#==================用于作图的导入
import numpy as np
import matplotlib.pyplot as plt
plt.style.use('ggplot')
plt.rcParams['savefig.dpi']=300
plt.rcParams['figure.dpi'] = 300
from pylab import mpl
mpl.rcParams['font.sans-serif'] = ['KaiTI']

#==============加载minist.inforence.py文件里定义的常量以及前向传播的函数
import mnist_inference

# 配置神经网络的参数
batch_size = 200
training_steps = 30000
learning_rate_base = 0.03     #基础学学习率
learning_rate_decay = 0.99   #学习率的衰减率
regularaztion_rate = 0.0001  #正则化系数
moving_average_decay = 0.99   #滑动平均系数
Loss_to_see = []

# 模型保存的路径和文件名
MODEL_SAVE_PATH = 'D:/the treasure of CYY/Python.ML/深tensorflow代码'
MODEL_NAME = 'mnist.model.ckpt'

def train(mnist):
    # 定义占位符placeholder
    x  = tf.placeholder(tf.float32,
                        [batch_size,
                         mnist_inference.IMAGE_SHAPE,
                         mnist_inference.IMAGE_SHAPE,
                         mnist_inference.NUM_CHANELS],
                        name = 'x-input')
    y_ = tf.placeholder(tf.float32,[None,mnist_inference.output_node],name = 'y-input') 
    # 定义正则化器【l2正则化】
    regularizer = tf.contrib.layers.l2_regularizer(regularaztion_rate) 
    #直接使用mniist_infere进行前向传播
    y = mnist_inference.inference(x,train,regularizer)
    global_step = tf.Variable(0,trainable=False)
    
    variable_average = tf.train.ExponentialMovingAverage(moving_average_decay,global_step)
    variable_average_op = variable_average.apply(tf.trainable_variables())
    
    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,labels=tf.argmax(y_,1))
    cross_entropy_mean = tf.reduce_mean(cross_entropy)
    
    loss = cross_entropy_mean + tf.add_n(tf.get_collection('losses'))
    #设置指数衰减的学习率
    learning_rate = tf.train.exponential_decay(
            learning_rate_base,
            global_step,
            mnist.train.num_examples/batch_size,
            learning_rate_decay)
    
    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss,global_step=global_step)
    with tf.control_dependencies([train_step,variable_average_op]):
        train_op = tf.no_op(name = 'train')

    # 初始化tensorflow持久化类
    # Saver = tf.train.Saver()
    with tf.Session() as sess:
        tf.global_variables_initializer().run()
        
        for i in range(training_steps):
            xs,ys = mnist.train.next_batch(batch_size)
            reshaped_xs = np.reshape(xs,(batch_size,
                         mnist_inference.IMAGE_SHAPE,
                         mnist_inference.IMAGE_SHAPE,
                         mnist_inference.NUM_CHANELS))
            _,loss_value,step = sess.run([train_op,loss,global_step],feed_dict={x:reshaped_xs,y_:ys})
            
            # 每1000轮保存一次模型
            if i % 1500 == 0:
                # 输出当前的损失函数大小
                print('after %d training step(s),loss on training batch is %g.'%(i,loss_value))
                # 这里在保存的文件名后加入global_step变量，这样可以直观的看到所到的训练轮数
                # Saver.save(
                #        sess,os.path.join(MODEL_SAVE_PATH,MODEL_NAME),
                #        global_step=global_step)
                if i != 0:
                    Loss_to_see.append([i,loss_value])
        Loss_final = np.array(Loss_to_see)
        plt.plot(Loss_final[:,0],Loss_final[:,1],linewidth = 0.6,color = 'g',alpha = 0.6,label='train_loss')
        plt.legend(loc='best',frameon=False)
        plt.ylabel('训练误差')
        plt.xlabel('迭代轮数')
        plt.show()

def main(argv = None):
    mnist = input_data.read_data_sets("/tmp/data",one_hot= True)
    train(mnist)
if __name__ == '__main__':
    tf.app.run()

请注意：代码中的对mnist数据集调用的接口不久之后tensorflo会做出调整

总共迭代3w轮，效果非常差，调整了好多次学习率，或许是卷积核尺寸深度选择的有问题，这点暂且不谈，整体耗时2h，而我之前使用的三层全连接网络耗时4分钟，误差为0.05左右，这个结果与普遍认知相反，懒得去跑测试。

#----------------------------后续
个人认为mnist数据较为简单，卷积层没有必要使用32,64等非常深的深度
我将 卷积深度改为了 6,12，卷积核尺寸统一调整为3，学习率回调至0.3【之前是0.04】
试了3k轮迭代效果很好