AlexNet网络实现--TensorFlow

AlexNet

1. Relu激活函数应用 --解决sigmoid在网络较深时的梯度弥散问题
2. dropout机制避免模型过拟合
3. CNN使用重叠的最大池化，而AlexNet使用最大池化，避免平均池化的模糊效果
4. 提出使用LRN层，创建局部神经元竞争机制，增强模型泛化机制
5. 使用CUDA加速训练过程
6. 数据增强–PCA降维
   AlexNet：8个训练层（不包含pool和LRN层），前五层conv层，后三层full_connection层，softmax层输出1000类

from datetime import datetime
import math
import time
import tensorflow as tf 

batch_size = 8
num_batches = 10

def print_activations(t):
    '''
    打印每一个卷积层或池化层输出的tensor尺寸
    '''
    print(t.op.name,' ',t.get_shape().as_list())

#设计AlexNet网络层结构
#定义inference
def inference(images):
    '''
    输入数据images
    返回计算卷积和池化后提取的特征及权重参数
    '''
    parameters = []
    
    with tf.name_scope('conv1') as scope:
        kernel = tf.Variable(tf.truncated_normal(
            [11,11,3,64],dtype = tf.float32,stddev = 0.1),name = 'weights')
        conv = tf.nn.conv2d(images,kernel,[1,4,4,1],padding = 'SAME')
        biases = tf.Variable(
            tf.constant(0.0,shape=[64],dtype=tf.float32),
            trainable=True,name='biases')
        conv1 = tf.nn.relu(tf.nn.bias_add(conv,biases),name=scope)
       
        parameters += [kernel,biases]
    print_activations(conv1)    
    
    lrn1 = tf.nn.lrn(conv1,4,bias = 1.0,alpha = 0.001 / 9,beta=0.75,name='lrn1')
    pool1 = tf.nn.max_pool(lrn1,ksize=[1,3,3,1],strides=[1,2,2,1],
                           padding='VALID',name='pool1')
    print_activations(pool1)
    
    with tf.name_scope('conv2') as scope:
        kernel = tf.Variable(tf.truncated_normal(
            [5,5,64,256],dtype=tf.float32,stddev=0.1),name="weights")
        biases = tf.Variable(tf.constant(0.1,shape=[256],dtype=tf.float32),
                             trainable=True,name='biases')
        conv2 = tf.nn.relu(tf.nn.bias_add(
            tf.nn.conv2d(pool1,kernel,[1,1,1,1],
                         padding = 'SAME'),biases),name=scope)
        
        parameters += [kernel,biases]
    print_activations(conv2)
    
    lrn2 = tf.nn.lrn(conv2,4,bias=1.0,alpha=0.001 / 9,beta=0.75,name='lrn2')
    pool2 = tf.nn.max_pool(lrn2,ksize=[1,3,3,1],strides=[1,2,2,1],
                           padding='VALID',name='pool2')
    print_activations(pool2)
    
    with tf.name_scope('conv3') as scope:
        kernel = tf.Variable(tf.truncated_normal(
            [3,3,256,384],dtype=tf.float32,stddev=0.1),name="weights")
        biases = tf.Variable(tf.constant(0.1,shape=[384],dtype=tf.float32),
                             trainable=True,name='biases')
        conv3 = tf.nn.relu(tf.nn.bias_add(
            tf.nn.conv2d(pool2,kernel,[1,1,1,1],
                         padding = 'SAME'),biases),name=scope)
        
        parameters += [kernel,biases]
    print_activations(conv3)
    
    with tf.name_scope('conv4') as scope:
        kernel = tf.Variable(tf.truncated_normal(
            [3,3,384,384],dtype=tf.float32,stddev=0.1),name="weights")
        biases = tf.Variable(tf.constant(0.1,shape=[384],dtype=tf.float32),
                             trainable=True,name='biases')
        conv4 = tf.nn.relu(tf.nn.bias_add(
            tf.nn.conv2d(conv3,kernel,[1,1,1,1],
                         padding = 'SAME'),biases),name=scope)
        
        parameters += [kernel,biases]
    print_activations(conv4)
    
    with tf.name_scope('conv5') as scope:
        kernel = tf.Variable(tf.truncated_normal(
            [3,3,384,256],dtype=tf.float32,stddev=0.1),name="weights")
        biases = tf.Variable(tf.constant(0.1,shape=[256],dtype=tf.float32),
                             trainable=True,name='biases')
        conv5 = tf.nn.relu(tf.nn.bias_add(
            tf.nn.conv2d(conv4,kernel,[1,1,1,1],
                         padding = 'SAME'),biases),name=scope)
        
        parameters += [kernel,biases]
    print_activations(conv5)
    
    pool5 = tf.nn.max_pool(conv5,ksize=[1,3,3,1],strides=[1,2,2,1],
                           padding="VALID",name='pool5')
    print_activations(pool5)
    
    return pool5,parameters

def time_tensorflow_run(session,target,info_string):
    '''
    session:窗口
    target:评测对象
    info_string：对象名称
    '''
    num_steps_burn_in = 10#程序热身
    #用于计算方差的两个参数
    total_duration = 0.0
    total_duration_squared = 0.0
    
    for i in range(num_batches + num_steps_burn_in):
        start_time = time.time()
        _ = session.run(target)
        duration = time.time() - start_time
        if i >= num_steps_burn_in:
            if not i % 10:
                print('%s:step %d,duration = %.3f'%(
                    datetime.now(),i - num_steps_burn_in,duration))
            total_duration += duration
            total_duration_squared += duration * duration
            
    #计算平均耗时和标准差
    mean_time = total_duration / num_batches
    sd = math.sqrt(total_duration_squared / num_batches - mean_time * mean_time)
    print('%s: %s across %d steps,%.3f +/- %.3f sec / batch'%(
        datetime.now(),info_string,num_batches,mean_time,sd))

# 构建一个自己的数据集（使用ImageNet数据集训练过程十分耗时）
def run_benchmark():
    #定义一个新的图进行计算
    with tf.Graph().as_default():
        image_size = 224
        #构造随机tensor
        images = tf.Variable(tf.random_normal([batch_size,
                                               image_size,
                                               image_size,3],
                                               dtype=tf.float32,stddev=0.1))
        pool5,parameters = inference(images)
        
        init = tf.global_variables_initializer()
        sess = tf.Session()
        sess.run(init)
        
        time_tensorflow_run(sess,pool5,'Forward')
        
        objetive = tf.nn.l2_loss(pool5)#计算pool5正则化损失
        print(objetive)
        grad = tf.gradients(objetive,parameters)
        time_tensorflow_run(sess,grad,'Forward--Backward')
        

run_benchmark()

conv1   [8, 56, 56, 64]
pool1   [8, 27, 27, 64]
conv2   [8, 27, 27, 256]
pool2   [8, 13, 13, 256]
conv3   [8, 13, 13, 384]
conv4   [8, 13, 13, 384]
conv5   [8, 13, 13, 256]
pool5   [8, 6, 6, 256]
2019-08-22 16:31:17.975427:step 0,duration = 0.066
2019-08-22 16:31:18.643627: Forward across 10 steps,0.073 +/- 0.007 sec / batch
Tensor("L2Loss:0", shape=(), dtype=float32)
2019-08-22 16:31:23.732227:step 0,duration = 0.442
2019-08-22 16:31:27.658827: Forward--Backward across 10 steps,0.437 +/- 0.016 sec / batch