Faster RCNN generate.py

该模块的功能主要是生成rpn proposals。
+ cv2.imread读取的图像的存储格式为H W K，且三通道的顺序为BGR
+ PIL.Image.open读取图片的存储格式为：W H (此时的数据为PIL库的某种对象)，且三通道顺序为RGB
但是，当转换为np.array的时候，存储格式将H W K,常用的代码段为：

def load_image(self, idx):
        """
        Load input image and preprocess for Caffe:
        - cast to float
        - switch channels RGB -> BGR
        - subtract mean
        - transpose to channel x height x width order
        """
        im = Image.open('xxx.jpg')
        in_ = np.array(im, dtype=np.float32)
        in_ = in_[:,:,::-1]
        in_ -= self.mean
        in_ = in_.transpose((2,0,1))
        return in_

def imdb_proposals(net, imdb):

def imdb_proposals(net, imdb):
    """Generate RPN proposals on all images in an imdb."""

    _t = Timer()
    imdb_boxes = [[] for _ in xrange(imdb.num_images)]
    for i in xrange(imdb.num_images):
        # cv2.imread读取的图像的存储格式为H W K，且三通道的顺序为BGR
        # PIL.Image.open读取图片的存储格式为：W H K ，且三通道顺序为RGB
        im = cv2.imread(imdb.image_path_at(i))
        _t.tic()
        #调用 im_proposals生成单张图片的rpn proposals， 以及得分
        imdb_boxes[i], scores = im_proposals(net, im)
        _t.toc()
        print 'im_proposals: {:d}/{:d} {:.3f}s' \
              .format(i + 1, imdb.num_images, _t.average_time)
        if 0:
            dets = np.hstack((imdb_boxes[i], scores))
            # from IPython import embed; embed()
            _vis_proposals(im, dets[:3, :], thresh=0.9)
            plt.show()

    return imdb_boxes

---

def im_proposals(net, im):该方法中会调用网络的forwad，从而得到想要的boxes和scores

def im_proposals(net, im):
    """Generate RPN proposals on a single image."""
    blobs = {}
    #调用_get_image_blob函数将图像转换为caffe所支持的输入数据结构即四维blob
    blobs['data'], blobs['im_info'] = _get_image_blob(im)
    # *(blobs['data'].shape 中的 ‘*’涉及到了python中函数的参数收集及逆操作
    net.blobs['data'].reshape(*(blobs['data'].shape))
    net.blobs['im_info'].reshape(*(blobs['im_info'].shape))
    blobs_out = net.forward(
            data=blobs['data'].astype(np.float32, copy=False),
            im_info=blobs['im_info'].astype(np.float32, copy=False))
    #返回im_info这个blob中存储的scale，等价于blobs['im_info'][0][2]
    scale = blobs['im_info'][0, 2]
    # 通过net的前向传播得到boxes, scores，注意，将boxes返回之前，需要将其缩放会原来的size
    # blobs_out['rois'][：]的第一位为类别，后四位才是坐标
    boxes = blobs_out['rois'][:, 1:].copy() / scale
    scores = blobs_out['scores'].copy()
    return boxes, scores

下面重点讲一下blobs_out = net.forward(data=blobs['data'].astype(np.float32, copy=False), im_info=blobs['im_info'].astype(np.float32, copy=False)), 当然纯属个人观点：

def _Net_forward(self, blobs=None, start=None, end=None, **kwargs):
    """
    Forward pass: prepare inputs and run the net forward.

    Parameters
    ----------
    blobs : list of blobs to return in addition to output blobs.
    kwargs : Keys are input blob names and values are blob ndarrays.
             For formatting inputs for Caffe, see Net.preprocess().
             If None, input is taken from data layers.
    start : optional name of layer at which to begin the forward pass
    end : optional name of layer at which to finish the forward pass
          (inclusive)

    Returns
    -------
    outs : {blob name: blob ndarray} dict.
    """
    if blobs is None:
        blobs = []
        # 返回name为start的layer的id，作为start_ind
        start_ind = list(self._layer_names).index(start)
    else:
        start_ind = 0

    if end is not None:
        end_ind = list(self._layer_names).index(end)
        outputs = set([end] + blobs)
    else:
        end_ind = len(self.layers) - 1
        outputs = set(self.outputs + blobs)

    if kwargs:
        if set(kwargs.keys()) != set(self.inputs):
            raise Exception('Input blob arguments do not match net inputs.')
        # Set input according to defined shapes and make arrays single and
        # C-contiguous as Caffe expects.
        # in_为blob name， blob为 blob ndarray
        for in_, blob in kwargs.iteritems():
            if blob.shape[0] != self.blobs[in_].num:
                raise Exception('Input is not batch sized')
            self.blobs[in_].data[...] = blob
    # 对应_caffe.cpp中的.def("_forward", &Net<Dtype>::ForwardFromTo)，可以猜想应该是调用底层的Net<Dtype>::ForwardFromTo方法，进行前向传播
    self._forward(start_ind, end_ind)

    # Unpack blobs to extract
    #rpn_test.pt所定义的网络的output为：rois blob 和 scores blob 两行log可以说明：
    #I0429 03:42:10.559293  9520 net.cpp:270] This network produces output rois
    #I0429 03:42:10.559300  9520 net.cpp:270] This network produces output scores

    #outputs为以列表，其元素为网络所有输出blob的name
    return {out: self.blobs[out].data for out in outputs}

1. net.forwar() 调用的是Pycaffe.py中的_Net_forward函数，代码如上：（关于pycaffe.py —-> Wrap the internal caffe C++ module (_caffe.so) with a clean, Pythonic interface.）
2. 在Pycaffe.py中，方法中带有self参数，个人觉得这应该表示一个Net对象，_Net_forward返回一个字典，{blob name: blob ndarray}，
3. start， end为可选的 layer name，注意是name
4. kwargs : Keys are input *blob names and values are blob ndarrays.*
5. self._layer_names：对应_caffe.cpp中的.add_property("_layer_names", bp::make_function(&Net<Dtype>::layer_names，bp::return_value_policy<bp::copy_const_reference>()))，可以猜想应该是调用底层的Net::layer_names，返回网络中所有层的name
6. self._forward(start_ind, end_ind)：对应_caffe.cpp中的.def("_forward", &Net<Dtype>::ForwardFromTo)，可以猜想应该是调用底层的Net<Dtype>::ForwardFromTo方法，进行前向传播

7. outputs = set(self.outputs + blobs)：对应pycaffe.py中

   @property
   def _Net_outputs(self):
   return [list(self.blobs.keys())[i] for i in self._outputs]
   

   a) self._outputs（_caffe.cpp）调用底层的Net::output_blob_indices方法，返回网络所有输出blob的id ;
   b) self.blobs.keys():
   self.blobs对应pycaffe.py中的

   def _Net_blobs(self):
   """
   An OrderedDict (bottom to top, i.e., input to output) of network
   blobs indexed by name
   """
   return OrderedDict(zip(self._blob_names, self._blobs))

   返回{blob name : blob ndarray} dict
   所以self.outputs返回的应该是网络输出blob的name

总之， _caffe.cpp 和 pycaffe.py这两个文件要好好研究一下

---

def _get_image_blob(im)

def _get_image_blob(im):
    """Converts an image into a network input.也就是将图像转换为caffe所支持的输入数据结构即blob

    Arguments:
        im (ndarray): a color image in BGR order

    Returns:
        blob (ndarray): a data blob holding an image pyramid
        im_scale_factors (list): list of image scales (relative to im) used
            in the image pyramid
    """
    im_orig = im.astype(np.float32, copy=True)
    im_orig -= cfg.PIXEL_MEANS

    im_shape = im_orig.shape
    im_size_min = np.min(im_shape[0:2])
    im_size_max = np.max(im_shape[0:2])

    processed_ims = []

    assert len(cfg.TEST.SCALES) == 1
    target_size = cfg.TEST.SCALES[0]

    im_scale = float(target_size) / float(im_size_min)
    # Prevent the biggest axis from being more than MAX_SIZE
    if np.round(im_scale * im_size_max) > cfg.TEST.MAX_SIZE:
        im_scale = float(cfg.TEST.MAX_SIZE) / float(im_size_max)
    im = cv2.resize(im_orig, None, None, fx=im_scale, fy=im_scale,
                    interpolation=cv2.INTER_LINEAR)

    # im_info，一些图像信息：H W scale， 而且数据结构为[[H, W, scale]]
    #np.newaxis添加了一个新轴，但是，新坐标轴上没有元素
    im_info = np.hstack((im.shape[:2], im_scale))[np.newaxis, :]
    processed_ims.append(im)

    # Create a blob to hold the input images
    #调用blob.py中的im_list_to_blob将图像转换为caffe所支持的数据结构blob，所做的工作就是复制数据，调整通道顺序，im_list_to_blob返回的其实是np.ndarray
    blob = im_list_to_blob(processed_ims)
    #返回的blob, im_info其实都是np.ndarray
    return blob, im_info