highway

（1）y是input，g(wy+b)是输出
t = sigmoid(Wy + b)
z = t * g(Wy + b) + (1 - t) * y
where g is nonlinearity, t is transform gate, and (1 - t) is carry gate.（2）文章 http://arxiv.org/abs/1505.00387
参考代码https://github.com/carpedm20/lstm-char-cnn-tensorflow
（3）定义
def linear(input_, output_size, scope=None):
  '''''
  Linear map: output[k] = sum_i(Matrix[k, i] * args[i] ) + Bias[k]
  Args:
      args: a tensor or a list of 2D, batch x n, Tensors.
  output_size: int, second dimension of W[i].
  scope: VariableScope for the created subgraph; defaults to "Linear".
  Returns:
  A 2D Tensor with shape [batch x output_size] equal to
  sum_i(args[i] * W[i]), where W[i]s are newly created matrices.
  Raises:
  ValueError: if some of the arguments has unspecified or wrong shape.
  '''

  shape = input_.get_shape().as_list()
  if len(shape) != 2:
    raise ValueError("Linear is expecting 2D arguments: %s" % str(shape))
  if not shape[1]:
    raise ValueError("Linear expects shape[1] of arguments: %s" % str(shape))
  input_size = shape[1]

  # Now the computation.
  with tf.variable_scope(scope or "SimpleLinear"):
    matrix = tf.get_variable("Matrix", [output_size, input_size], dtype=input_.dtype)
    bias_term = tf.get_variable("Bias", [output_size], dtype=input_.dtype)

  return tf.matmul(input_, tf.transpose(matrix)) + bias_term

# highway layer that borrowed from https://github.com/carpedm20/lstm-char-cnn-tensorflow
def highway(input_, size, layer_size=1, bias=-2, f=tf.nn.relu):
  """Highway Network (cf. http://arxiv.org/abs/1505.00387).
  t = sigmoid(Wy + b)
  z = t * g(Wy + b) + (1 - t) * y
  where g is nonlinearity, t is transform gate, and (1 - t) is carry gate.
  """
  output = input_
  for idx in range(layer_size):
    output = f(
      linear(output, size, scope='output_lin_%d' % idx))  # update
    # add below, and remove scope parameter while calling:
    # from tensorflow.contrib.rnn.python.ops.core_rnn_cell_impl import _linear as linear

    transform_gate = tf.sigmoid(
      linear(input_, size, scope='transform_lin_%d' % idx) + bias)  # update
    carry_gate = 1. - transform_gate

    output = transform_gate * output + carry_gate * input_
  return output
（4）使用
# Add highway
with tf.name_scope("highway"):
  self.h_highway = highway(self.h_pool_flat, self.h_pool_flat.get_shape()[1], 1, 0)

with tf.name_scope("dropout"):
    #没有highway
    # self.h_drop = tf.nn.dropout(self.h_pool_flat, self.dropout_keep_prob)
    #有highway
    self.h_drop = tf.nn.dropout(self.h_highway, self.dropout_keep_prob)
    self.h_drop = tf.reshape(self.h_drop, [tf.shape(self.h_drop)[0] + 0, num_filters_total])