CAFFE源码学习笔记之softmax_layer

一、前言
在全连接层之后，接着就是softmax层。softmax已经介绍过，是表征分类是该类的概率的大小。具体原理见logistic回归与softmax
所以，softmax layer就是将线性预测值转化为类别概率。由于该层没有参数，所以只需要计算向后传播的导数就可以了。而softmax_loss层则是在输出概率的基础上，用负log函数计算其loss。也就是说，caffe将softmax的归一化求概率和目标函数分成了两个部分。

二、源码分析
1、成员变量
为什么设置outer_num_和inner_num_？
inner_num_实际该层输入的图像的尺寸，由于正常网络下，其输入为n*m*1*1，所以inner_num实际就是1 。

  int outer_num_;//外层个数，即batch_size
  int inner_num_;//内层个数，默认为1，应该是输入的图片的尺寸
  int softmax_axis_;//默认都是1
  /// 求和的缓存
  Blob<Dtype> sum_multiplier_;
  /// 输出结果的缓存
  Blob<Dtype> scale_;
};

2、layersetup函数
因为该层没有参数，所以没有自己实现，直接使用基类Layer的实现。
3、reshape函数
经过全连接后，现在输入为input_num*output_num的矩阵

template <typename Dtype>
void SoftmaxLayer<Dtype>::Reshape(const vector<Blob<Dtype>*>& bottom,
      const vector<Blob<Dtype>*>& top) {
  softmax_axis_ =
      bottom[0]->CanonicalAxisIndex(this->layer_param_.softmax_param().axis());//axis：default = 1
  top[0]->ReshapeLike(*bottom[0]);//input_num*output_num
  vector<int> mult_dims(1, bottom[0]->shape(softmax_axis_));//（output_num）
  sum_multiplier_.Reshape(mult_dims);//（output_num）
  Dtype* multiplier_data = sum_multiplier_.mutable_cpu_data();
  caffe_set(sum_multiplier_.count(), Dtype(1), multiplier_data);
  outer_num_ = bottom[0]->count(0, softmax_axis_);//outer_num_ =input_num
  inner_num_ = bottom[0]->count(softmax_axis_ + 1);//inner_num_=count = 1;
  vector<int> scale_dims = bottom[0]->shape();
  scale_dims[softmax_axis_] = 1;
  scale_.Reshape(scale_dims);//最后结果为64*1的向量
}

4、前向计算

前向计算公式：$f(z_k)=\frac{e^{z_k-m}}{\sum_0^{n-1}{e^{z_i-m}}}$
其中$m=max(z_i)$

指数分式中指数内上下同时减m，结果不变。这主要是为了解决数值稳定问题——在反向传播的时候，输出层先求得$\frac{\partial L}{\partial a}=-\frac{loss}{a_y}$,
由于浮点数是有精度限制的，每多一次运算就会多累积一定的误差，如果碰巧这次预测正确的类别所得到的概率非常小（接近零）的话，也就是说$a_y$特别小，得到的$\frac{\partial L}{\partial a}$会特别大，结果会有 overflow 的危险。

template <typename Dtype>
void SoftmaxLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
    const vector<Blob<Dtype>*>& top) {
  const Dtype* bottom_data = bottom[0]->cpu_data();
  Dtype* top_data = top[0]->mutable_cpu_data();
  Dtype* scale_data = scale_.mutable_cpu_data();
  int channels = bottom[0]->shape(softmax_axis_);
  int dim = bottom[0]->count() / outer_num_;
  // 从bottom 复制到 top，以下操作都在top上进行
  caffe_copy(bottom[0]->count(), bottom_data, top_data);
  // We need to subtract the max to avoid numerical issues, compute the exp,
  // and then normalize.
  for (int i = 0; i < outer_num_; ++i) {
    // initialize scale_data to the first plane
    caffe_copy(inner_num_, bottom_data + i * dim, scale_data);//将输入复制到sacle_data中缓存
    for (int j = 0; j < channels; j++) {//针对每一个类别
      for (int k = 0; k < inner_num_; k++) {//针对每一张输入的图片，按照每个像素点进行计算
      //求最大值m=max(z_i)(存放在scale_data)
        scale_data[k] = std::max(scale_data[k],
            bottom_data[i * dim + j * inner_num_ + k]);
      }
    }
    // z_k-m,归一化subtract the max
    caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, channels, inner_num_,
        1, -1., sum_multiplier_.cpu_data(), scale_data, 1., top_data);
    // 求指数
    caffe_exp<Dtype>(dim, top_data, top_data);
    // 向量求和
    caffe_cpu_gemv<Dtype>(CblasTrans, channels, inner_num_, 1.,
        top_data, sum_multiplier_.cpu_data(), 0., scale_data);
    // 做除法，归一化成(0,1)内的概率
    for (int j = 0; j < channels; j++) {
      caffe_div(inner_num_, top_data, scale_data, top_data);
      top_data += inner_num_;//地址移动到下一个图片，此处为+1.
    }
  }
}

5、后向计算

假设其损失函数为$L$

反向传播就是对其求输入的导数：
$\frac{\partial L}{\partial Z}=\frac{\partial L}{\partial a}*\frac{\partial a}{\partial Z}$

其中,$\frac{\partial L}{\partial a}$就是top_diff，实质就是损失，已经算出。$a则是正向得到的f(z)$。z为$bottom\_data$,而a为$top\_data$。

首先，求$\frac{\partial a_i}{\partial z_k}$：

因为前向计算有：

$f(z_i)=\frac{e^{z_i}}{\sum_0^{n-1}{e^{z_j}}}$

假设$n-1=1$，可得：

$a_1=\frac{e^{z_1}}{e^{z_0}+e^{z_1}}$

求导得：

$\frac{\partial a_1}{\partial z_1}=a_1-a_1*a_1;$
$\frac{\partial a_1}{\partial z_0}=-a_1*a_0;$

由归纳法证明：

$\frac{\partial a_i}{\partial z_k}=a_k-a_k*a_k,i=k;$
$\frac{\partial a_i}{\partial z_k}=-a_k*a_i,i \ne k;$

因此$\frac{\partial L}{\partial z_k}=\sum_0^m\frac{\partial L}{\partial a_i}*\frac{\partial a_i}{\partial z_k}=-\frac{\partial L}{\partial a}*a*a_k+\frac{\partial L}{\partial a_k}*a_k$

综合可得：

$\frac{\partial L}{\partial Z}=-\frac{\partial L}{\partial a}*a*a+\frac{\partial L}{\partial a}*a=a*(\frac{\partial L}{\partial a}-\frac{\partial L}{\partial a}*a)=top\_data*(top\_diff-top\_diff*top\_data)$

template <typename Dtype>
void SoftmaxLayer<Dtype>::Backward_cpu(const vector<Blob<Dtype>*>& top,
    const vector<bool>& propagate_down,
    const vector<Blob<Dtype>*>& bottom) {
  const Dtype* top_diff = top[0]->cpu_diff();
  const Dtype* top_data = top[0]->cpu_data();
  Dtype* bottom_diff = bottom[0]->mutable_cpu_diff();
  Dtype* scale_data = scale_.mutable_cpu_data();
  int channels = top[0]->shape(softmax_axis_);
  int dim = top[0]->count() / outer_num_;
  caffe_copy(top[0]->count(), top_diff, bottom_diff);
  for (int i = 0; i < outer_num_; ++i) {
    //  计算top_diff和top_data的点积，然后从bottom_diff中减去该值  top_diff*top_data
    for (int k = 0; k < inner_num_; ++k) {
      scale_data[k] = caffe_cpu_strided_dot<Dtype>(channels,
          bottom_diff + i * dim + k, inner_num_,
          top_data + i * dim + k, inner_num_);
    }
    //  bottom_diff=top_diff-top_diff*top_data
    caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, channels, inner_num_, 1,
        -1., sum_multiplier_.cpu_data(), scale_data, 1., bottom_diff + i * dim);
  }
  // top_data*(top_diff-top_diff*top_data)
  caffe_mul(top[0]->count(), bottom_diff, top_data, bottom_diff);
}

三、softmax_loss_layer
1、成员变量

  /// 将预测映射到概率分布的softmax层指针
  shared_ptr<Layer<Dtype> > softmax_layer_;
  /// 存储输出概率
  Blob<Dtype> prob_;
  /// bottom vector holder used in call to the underlying SoftmaxLayer::Forward
  vector<Blob<Dtype>*> softmax_bottom_vec_;
  /// top vector holder used in call to the underlying SoftmaxLayer::Forward
  vector<Blob<Dtype>*> softmax_top_vec_;
  /// Whether to ignore instances with a certain label.
  bool has_ignore_label_;
  /// The label indicating that an instance should be ignored.
  int ignore_label_;
  /// How to normalize the output loss.
  LossParameter_NormalizationMode normalization_;

  int softmax_axis_, outer_num_, inner_num_;

2、layersetup函数

template <typename Dtype>
void SoftmaxWithLossLayer<Dtype>::LayerSetUp(
    const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
  LossLayer<Dtype>::LayerSetUp(bottom, top);//调用LossLayer的setup初始化层参数
  LayerParameter softmax_param(this->layer_param_);
  softmax_param.set_type("Softmax");//设置层类型
  softmax_layer_ = LayerRegistry<Dtype>::CreateLayer(softmax_param);//注册层
  softmax_bottom_vec_.clear();
  softmax_bottom_vec_.push_back(bottom[0]);//将存放输入的数据结构放入容器
  softmax_top_vec_.clear();
  softmax_top_vec_.push_back(&prob_);//将存放输出概率的数据结构放入容器
  softmax_layer_->SetUp(softmax_bottom_vec_, softmax_top_vec_);//用输入输出初始化softmax层

  has_ignore_label_ =
    this->layer_param_.loss_param().has_ignore_label();
  if (has_ignore_label_) {
    ignore_label_ = this->layer_param_.loss_param().ignore_label();
  }
  if (!this->layer_param_.loss_param().has_normalization() &&
      this->layer_param_.loss_param().has_normalize()) {//normalize有两种：一种是VALID，另一种是BATCH_SIZE
    normalization_ = this->layer_param_.loss_param().normalize() ?
                     LossParameter_NormalizationMode_VALID :
                     LossParameter_NormalizationMode_BATCH_SIZE;
  } else {
    normalization_ = this->layer_param_.loss_param().normalization();
  }
}

3、reshepe函数

template <typename Dtype>
void SoftmaxWithLossLayer<Dtype>::Reshape(
    const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
  LossLayer<Dtype>::Reshape(bottom, top);
  softmax_layer_->Reshape(softmax_bottom_vec_, softmax_top_vec_);
  softmax_axis_ =
      bottom[0]->CanonicalAxisIndex(this->layer_param_.softmax_param().axis());
      //outer_num_ * inner_num_必须和bottom[1]->count()给定的lable数相同
      //softmax前面是卷积层的时候，每个label就不是一个数，而是一个矩阵，对应着每个 feature map中每个像素值的分类
  outer_num_ = bottom[0]->count(0, softmax_axis_);
  inner_num_ = bottom[0]->count(softmax_axis_ + 1);
  CHECK_EQ(outer_num_ * inner_num_, bottom[1]->count())
      << "Number of labels must match number of predictions; "
      << "e.g., if softmax axis == 1 and prediction shape is (N, C, H, W), "
      << "label count (number of labels) must be N*H*W, "
      << "with integer values in {0, 1, ..., C-1}.";
  if (top.size() >= 2) {
    // softmax output
    top[1]->ReshapeLike(*bottom[0]);
  }

4、归一化
归一化主要是为了数值稳定，避免出现overflow等情况。

template <typename Dtype>
Dtype SoftmaxWithLossLayer<Dtype>::get_normalizer(
    LossParameter_NormalizationMode normalization_mode, int valid_count) {
  Dtype normalizer;
  switch (normalization_mode) {
    case LossParameter_NormalizationMode_FULL:
      normalizer = Dtype(outer_num_ * inner_num_);//表示所有的像素点都参与了loss计算
      break;
    case LossParameter_NormalizationMode_VALID:
      if (valid_count == -1) {
        normalizer = Dtype(outer_num_ * inner_num_);
      } else {
        normalizer = Dtype(valid_count);//有多少计算了loss，就归一化除去多少
      }
      break;
    case LossParameter_NormalizationMode_BATCH_SIZE:
      normalizer = Dtype(outer_num_);//表示所有的图像都参与计算loss
      break;
    case LossParameter_NormalizationMode_NONE:
      normalizer = Dtype(1);//不归一化
      break;
    default:
      LOG(FATAL) << "Unknown normalization mode: "
          << LossParameter_NormalizationMode_Name(normalization_mode);
  }
  // Some users will have no labels for some examples in order to 'turn off' a
  // particular loss in a multi-task setup. The max prevents NaNs in that case.
  return std::max(Dtype(1.0), normalizer);
}

5、前向计算

负log函数求损失

template <typename Dtype>
void SoftmaxWithLossLayer<Dtype>::Forward_cpu(
    const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
  // The forward pass computes the softmax prob values.
  softmax_layer_->Forward(softmax_bottom_vec_, softmax_top_vec_);//调用softmaxlayer的前向计算
  const Dtype* prob_data = prob_.cpu_data();
  const Dtype* label = bottom[1]->cpu_data();
  int dim = prob_.count() / outer_num_;
  int count = 0;
  Dtype loss = 0;
  for (int i = 0; i < outer_num_; ++i) {
    for (int j = 0; j < inner_num_; j++) {
      const int label_value = static_cast<int>(label[i * inner_num_ + j]);//从对应的实例中取出相对应的标签
      if (has_ignore_label_ && label_value == ignore_label_) {
        continue;
      }
      DCHECK_GE(label_value, 0);
      DCHECK_LT(label_value, prob_.shape(softmax_axis_));
      loss -= log(std::max(prob_data[i * dim + label_value * inner_num_ + j],
                           Dtype(FLT_MIN)));//计算损失函数
      ++count;
    }
  }
  top[0]->mutable_cpu_data()[0] = loss / get_normalizer(normalization_, count);//归一化，中间有问题的是当有若干的实例不计算loss的时候，最终计算归一化的时候，需要除以真正计算了loss的实例个数
  if (top.size() == 2) {
    top[1]->ShareData(prob_);
  }
}

6、反向计算

经过前向计算：$f(z_k)=\frac{e^{z_k}}{\sum_i^n{e^{z_i}}}$
求其f负log函数：$loss=log\sum_i^n{e^{z_i}}-z_k$
对其输入求导：$\frac{\partial loss}{\partial z_i}= \begin{cases} f(z_k)-1, & \text{if z_i=z_k} \\ f(z_i), & \text{else} \end{cases}$
代码就一句话：

          bottom_diff[i * dim + label_value * inner_num_ + j] -= 1;

template <typename Dtype>
void SoftmaxWithLossLayer<Dtype>::Backward_cpu(const vector<Blob<Dtype>*>& top,
    const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {
  if (propagate_down[1]) {
    LOG(FATAL) << this->type()
               << " Layer cannot backpropagate to label inputs.";
  }
  if (propagate_down[0]) {
    Dtype* bottom_diff = bottom[0]->mutable_cpu_diff();
    const Dtype* prob_data = prob_.cpu_data();
    caffe_copy(prob_.count(), prob_data, bottom_diff);
    const Dtype* label = bottom[1]->cpu_data();
    int dim = prob_.count() / outer_num_;
    int count = 0;
    for (int i = 0; i < outer_num_; ++i) {
      for (int j = 0; j < inner_num_; ++j) {
        const int label_value = static_cast<int>(label[i * inner_num_ + j]);
        if (has_ignore_label_ && label_value == ignore_label_) {
          for (int c = 0; c < bottom[0]->shape(softmax_axis_); ++c) {
            bottom_diff[i * dim + c * inner_num_ + j] = 0;//默认情况下，j=0，inner_num_=1，如果分类有是个，dim=10，
          }
        } else {
          bottom_diff[i * dim + label_value * inner_num_ + j] -= 1;
          ++count;
        }
      }
    }
    // Scale gradient
    Dtype loss_weight = top[0]->cpu_diff()[0] /
                        get_normalizer(normalization_, count);//归一化
    caffe_scal(prob_.count(), loss_weight, bottom_diff);//loss权重
  }
}