15-tensorflow实现各种自编码器及技巧

用编码器做PCA

• 隐层的特点：线性激活函数

• 代码：

• 预先声明的函数和包

# To support both python 2 and python 3
from __future__ import division, print_function, unicode_literals

# Common imports
import numpy as np
import os
import sys

# to make this notebook's output stable across runs
def reset_graph(seed=42):
    tf.reset_default_graph()
    tf.set_random_seed(seed)
    np.random.seed(seed)

# To plot pretty figures
%matplotlib inline
import matplotlib
import matplotlib.pyplot as plt
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12

# Where to save the figures
PROJECT_ROOT_DIR = "."
CHAPTER_ID = "autoencoders"

def save_fig(fig_id, tight_layout=True):
    path = os.path.join(PROJECT_ROOT_DIR, "images", CHAPTER_ID, fig_id + ".png")
    print("Saving figure", fig_id)
    if tight_layout:
        plt.tight_layout()
    plt.savefig(path, format='png', dpi=300)

• 模型：

#Build 3D dataset
import numpy as np
import numpy.random as rnd
rnd.seed(4)
m = 200
w1, w2 = 0.1, 0.3
noise = 0.1

angles = rnd.rand(m) * 3 * np.pi / 2 - 0.5
data = np.empty((m, 3))
data[:, 0] = np.cos(angles) + np.sin(angles)/2 + noise * rnd.randn(m) / 2
data[:, 1] = np.sin(angles) * 0.7 + noise * rnd.randn(m) / 2
data[:, 2] = data[:, 0] * w1 + data[:, 1] * w2 + noise * rnd.randn(m)

#Normalize the data:
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
X_train = scaler.fit_transform(data[:100])
X_test = scaler.transform(data[100:])

#autoencoder
import tensorflow as tf

reset_graph()

n_inputs = 3
n_hidden = 2  # codings
n_outputs = n_inputs

learning_rate = 0.01

X = tf.placeholder(tf.float32, shape=[None, n_inputs])
hidden = tf.layers.dense(X, n_hidden)
outputs = tf.layers.dense(hidden, n_outputs)

reconstruction_loss = tf.reduce_mean(tf.square(outputs - X))

optimizer = tf.train.AdamOptimizer(learning_rate)
training_op = optimizer.minimize(reconstruction_loss)

init = tf.global_variables_initializer()

n_iterations = 1000
codings = hidden

with tf.Session() as sess:
    init.run()
    for iteration in range(n_iterations):
        training_op.run(feed_dict={
   
   X: X_train})
    codings_val = codings.eval(feed_dict={
   
   X: X_test})

    #fig = matplotlib.pyplot.gcf()
    #fig.set_size_inches(18.5, 10.5)
    fig = plt.figure(figsize=(12,10))
    plt.plot(codings_val[:,0], codings_val[:, 1], "b.")
    plt.xlabel("$z_1$", fontsize=18)
    plt.ylabel("$z_2$", fontsize=18, rotation=0)
    save_fig("linear_autoencoder_pca_plot")
    plt.show()

输出：

Saving figure linear_autoencoder_pca_plot

---

堆栈编码器(stacked autoencoder)

• 特点：即有很多隐层来叠加，结构如图：
  

• 代码：

#build a stacked Autoencoder with 3 hidden layers and 1 output layer (ie. 2 stacked Autoencoders).
#We will use ELU activation, He initialization and L2 regularization.

#下载数据
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/tmp/data")
reset_graph()

from functools import partial

n_inputs = 28 * 28 # for mnist
n_hidden1 = 300
n_hidden2 = 150 #codings
n_hidden3 = n_hidden1
n_outputs = n_inputs

learning_rate = 0.01
l2_reg = 0.0001

X = tf.placeholder(tf.float32,shape=[None,n_inputs])


he_init = tf.contrib.layers.variance_scaling_initializer()# He initialization
#等价于
#he_init = lambda shape, dtype=tf.float32: tf.truncated_normal(shape, 0., stddev=np.sqrt(2/shape[0]))
l2_regularizer = tf.contrib.layers.l2_regularizer(l2_reg)
my_dense_layer = partial(tf.layers.dense,
                        activation=tf.nn.elu,
                        kernel_initializer=he_init,
                        kernel_regularizer=l2_regularizer)

hidden1 = my_dense_layer(X, n_hidden1)
hidden2 = my_dense_layer(hidden1, n_hidden2)
hidden3 = my_dense_layer(hidden2, n_hidden3)
outputs = my_dense_layer(hidden3, n_outputs,activation=None)

reconstruction_loss = tf.reduce_mean(tf.square(outputs - X))
##
reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)

#tf.add_n :Adds all input tensors element-wise.
loss = tf.add_n([reconstruction_loss] + reg_losses)

optimizer = tf.train.AdamOptimizer(learning_rate)
training_op = optimizer.minimize(loss)

init = tf.global_variables_initializer()

saver = tf.train.Saver()


#训练(y_batch is not used). This is unsupervised training.

n_epochs = 5
batch_size = 150

with tf.Session() as sess:
    init.run()
    for epoch in range(n_epochs):
        n_batches = mnist.train.num_examples // batch_size
        for iteration in range(n_batches):
            print("\r{}%".format(100 * iteration // n_batches), end="")
            sys.stdout.flush()
            X_batch, y_batch = mnist.train.next_batch(batch_size)
            sess.run(training_op,feed_dict={
  
  X:X_batch})
        loss_train = reconstruction_loss.eval(feed_dict={
  
  X:X_batch})
        print("\r{}".format(epoch),"Train MSE:",loss_train)
        saver.save(sess,"./my_model_all_layers.ckpt")

输出：
0 Train MSE: 0.020855438
1 Train MSE: 0.011372581
2 Train MSE: 0.010224564
3 Train MSE: 0.009900457
49% Train MSE: 0.010375758

#可视化其中的原始图 和重构图
def show_reconstructed_digits(X, outputs, model_path = None, n_test_digits = 2):
    with tf.Session() as sess:
        if model_path:
            saver.restore(sess,model_path)
        X_test = mnist.test.images[:n_test_digits]
        outputs_val = outputs.eval(feed_dict={X:X_test})

    fig = plt.figure(figsize=(8, 3 * n_test_digits))
    for digit_index in range(n_test_digits):
        plt.subplot(n_test_digits, 2, digit_index * 2 + 1)
        plot_image(X_test[digit_index])
        plt.subplot(n_test_digits, 2, digit_index * 2 + 2)
        plot_image(outputs_val[digit_index])

#保存图
show_reconstructed_digits(X, outputs, "./my_model_all_layers.ckpt")
save_fig("reconstruction_plot")

权重绑定(tying weights)

即把解码层(decoder layers)与编码层(encoder layers)的权重绑定在一起，即公用这个权值。
优点：防止过拟合、减半权重参数。

设编码器有$N$层(不包括input layer)，用 $\mathbf W_{L}$ 表示第$L$ 层的连接权重,即第一层是第一隐层，