Faster RCNN Generate Proposal Labels (#12616)

* Add generate_proposal_labels for Faster-RCNN.

Author: sefira

paddle/fluid/API.spec

@@ -303,6 +303,7 @@ paddle.fluid.layers.ssd_loss ArgSpec(args=['location', 'confidence', 'gt_box', '
 paddle.fluid.layers.detection_map ArgSpec(args=['detect_res', 'label', 'class_num', 'background_label', 'overlap_threshold', 'evaluate_difficult', 'has_state', 'input_states', 'out_states', 'ap_version'], varargs=None, keywords=None, defaults=(0, 0.3, True, None, None, None, 'integral'))
 paddle.fluid.layers.rpn_target_assign ArgSpec(args=['loc', 'scores', 'anchor_box', 'gt_box', 'rpn_batch_size_per_im', 'fg_fraction', 'rpn_positive_overlap', 'rpn_negative_overlap'], varargs=None, keywords=None, defaults=(256, 0.25, 0.7, 0.3))
 paddle.fluid.layers.anchor_generator ArgSpec(args=['input', 'anchor_sizes', 'aspect_ratios', 'variance', 'stride', 'offset', 'name'], varargs=None, keywords=None, defaults=(None, None, [0.1, 0.1, 0.2, 0.2], None, 0.5, None))
+paddle.fluid.layers.generate_proposal_labels ArgSpec(args=['rpn_rois', 'gt_classes', 'gt_boxes', 'im_scales', 'batch_size_per_im', 'fg_fraction', 'fg_thresh', 'bg_thresh_hi', 'bg_thresh_lo', 'bbox_reg_weights', 'class_nums'], varargs=None, keywords=None, defaults=(256, 0.25, 0.25, 0.5, 0.0, [0.1, 0.1, 0.2, 0.2], None))
 paddle.fluid.layers.generate_proposals ArgSpec(args=['scores', 'bbox_deltas', 'im_info', 'anchors', 'variances', 'pre_nms_top_n', 'post_nms_top_n', 'nms_thresh', 'min_size', 'eta', 'name'], varargs=None, keywords=None, defaults=(6000, 1000, 0.5, 0.1, 1.0, None))
 paddle.fluid.layers.iou_similarity ArgSpec(args=[], varargs='args', keywords='kwargs', defaults=None)
 paddle.fluid.layers.box_coder ArgSpec(args=[], varargs='args', keywords='kwargs', defaults=None)

paddle/fluid/operators/detection/CMakeLists.txt

@@ -29,6 +29,7 @@ target_assign_op.cu)
 detection_library(polygon_box_transform_op SRCS polygon_box_transform_op.cc
 polygon_box_transform_op.cu)
 detection_library(rpn_target_assign_op SRCS rpn_target_assign_op.cc)
+detection_library(generate_proposal_labels_op SRCS generate_proposal_labels_op.cc)
 detection_library(generate_proposals_op SRCS generate_proposals_op.cc)
 #Export local libraries to parent
 set(DETECTION_LIBRARY ${LOCAL_DETECTION_LIBS} PARENT_SCOPE)

paddle/fluid/operators/detection/generate_proposal_labels_op.cc

@@ -86,7 +86,7 @@ class RpnTargetAssignKernel : public framework::OpKernel<T> {
                          std::minstd_rand engine,
                          std::vector<int>* inds) const {
     std::uniform_real_distribution<float> uniform(0, 1);
-    const int64_t size = static_cast<int64_t>(inds->size());
+    const int64_t size = static_cast<int64_t>(inds->size() - offset);
     if (size > num) {
       for (int64_t i = num; i < size; ++i) {
         int rng_ind = std::floor(uniform(engine) * i);
@@ -126,7 +126,7 @@ class RpnTargetAssignKernel : public framework::OpKernel<T> {
                 neg_threshold, target_label_data, fg_inds, bg_inds);
     // Reservoir Sampling
     ReservoirSampling(fg_num, fg_offset, engine, fg_inds);
-    int bg_num = rpn_batch_size - fg_inds->size();
+    int bg_num = rpn_batch_size - (fg_inds->size() - fg_offset);
     ReservoirSampling(bg_num, bg_offset, engine, bg_inds);
   }
 

paddle/fluid/operators/detection/rpn_target_assign_op.cc

@@ -101,5 +101,8 @@ namespace ops = paddle::operators;
 REGISTER_OPERATOR(gather, ops::GatherOp, ops::GatherOpMaker,
                   paddle::framework::DefaultGradOpDescMaker<true>);
 REGISTER_OPERATOR(gather_grad, ops::GatherGradOp);
-REGISTER_OP_CPU_KERNEL(gather, ops::GatherOpKernel<float>);
-REGISTER_OP_CPU_KERNEL(gather_grad, ops::GatherGradientOpKernel<float>);
+REGISTER_OP_CPU_KERNEL(gather, ops::GatherOpKernel<float>,
+                       ops::GatherOpKernel<int>, ops::GatherOpKernel<double>);
+REGISTER_OP_CPU_KERNEL(gather_grad, ops::GatherGradientOpKernel<float>,
+                       ops::GatherGradientOpKernel<int>,
+                       ops::GatherGradientOpKernel<double>);

paddle/fluid/operators/gather_op.cc

@@ -39,6 +39,7 @@
     'detection_map',
     'rpn_target_assign',
     'anchor_generator',
+    'generate_proposal_labels',
     'generate_proposals',
 ]
 
@@ -1256,6 +1257,64 @@ def _is_list_or_tuple_(data):
     return anchor, var
 
 
+def generate_proposal_labels(rpn_rois,
+                             gt_classes,
+                             gt_boxes,
+                             im_scales,
+                             batch_size_per_im=256,
+                             fg_fraction=0.25,
+                             fg_thresh=0.25,
+                             bg_thresh_hi=0.5,
+                             bg_thresh_lo=0.0,
+                             bbox_reg_weights=[0.1, 0.1, 0.2, 0.2],
+                             class_nums=None):
+    """
+    ** Generate proposal labels Faster-RCNN **
+    TODO(buxingyuan): Add Document
+    """
+
+    helper = LayerHelper('generate_proposal_labels', **locals())
+
+    rois = helper.create_tmp_variable(dtype=rpn_rois.dtype)
+    labels_int32 = helper.create_tmp_variable(dtype=gt_classes.dtype)
+    bbox_targets = helper.create_tmp_variable(dtype=rpn_rois.dtype)
+    bbox_inside_weights = helper.create_tmp_variable(dtype=rpn_rois.dtype)
+    bbox_outside_weights = helper.create_tmp_variable(dtype=rpn_rois.dtype)
+
+    helper.append_op(
+        type="generate_proposal_labels",
+        inputs={
+            'RpnRois': rpn_rois,
+            'GtClasses': gt_classes,
+            'GtBoxes': gt_boxes,
+            'ImScales': im_scales
+        },
+        outputs={
+            'Rois': rois,
+            'LabelsInt32': labels_int32,
+            'BboxTargets': bbox_targets,
+            'BboxInsideWeights': bbox_inside_weights,
+            'BboxOutsideWeights': bbox_outside_weights
+        },
+        attrs={
+            'batch_size_per_im': batch_size_per_im,
+            'fg_fraction': fg_fraction,
+            'fg_thresh': fg_thresh,
+            'bg_thresh_hi': bg_thresh_hi,
+            'bg_thresh_lo': bg_thresh_lo,
+            'bbox_reg_weights': bbox_reg_weights,
+            'class_nums': class_nums
+        })
+
+    rois.stop_gradient = True
+    labels_int32.stop_gradient = True
+    bbox_targets.stop_gradient = True
+    bbox_inside_weights.stop_gradient = True
+    bbox_outside_weights.stop_gradient = True
+
+    return rois, labels_int32, bbox_targets, bbox_inside_weights, bbox_outside_weights
+
+
 def generate_proposals(scores,
                        bbox_deltas,
                        im_info,

python/paddle/fluid/layers/detection.py

@@ -146,6 +146,55 @@ def test_anchor_generator(self):
         assert anchor.shape[3] == 4
 
 
+class TestGenerateProposalLabels(unittest.TestCase):
+    def test_generate_proposal_labels(self):
+        rpn_rois = layers.data(
+            name='rpn_rois',
+            shape=[4, 4],
+            dtype='float32',
+            lod_level=1,
+            append_batch_size=False)
+        gt_classes = layers.data(
+            name='gt_classes',
+            shape=[6],
+            dtype='int32',
+            lod_level=1,
+            append_batch_size=False)
+        gt_boxes = layers.data(
+            name='gt_boxes',
+            shape=[6, 4],
+            dtype='float32',
+            lod_level=1,
+            append_batch_size=False)
+        im_scales = layers.data(
+            name='im_scales',
+            shape=[1],
+            dtype='float32',
+            lod_level=1,
+            append_batch_size=False)
+        class_nums = 5
+        rois, labels_int32, bbox_targets, bbox_inside_weights, bbox_outside_weights = fluid.layers.generate_proposal_labels(
+            rpn_rois=rpn_rois,
+            gt_classes=gt_classes,
+            gt_boxes=gt_boxes,
+            im_scales=im_scales,
+            batch_size_per_im=2,
+            fg_fraction=0.5,
+            fg_thresh=0.5,
+            bg_thresh_hi=0.5,
+            bg_thresh_lo=0.0,
+            bbox_reg_weights=[0.1, 0.1, 0.2, 0.2],
+            class_nums=class_nums)
+        assert rois.shape[1] == 4
+        assert rois.shape[0] == labels_int32.shape[0]
+        assert rois.shape[0] == bbox_targets.shape[0]
+        assert rois.shape[0] == bbox_inside_weights.shape[0]
+        assert rois.shape[0] == bbox_outside_weights.shape[0]
+        assert bbox_targets.shape[1] == 4 * class_nums
+        assert bbox_inside_weights.shape[1] == 4 * class_nums
+        assert bbox_outside_weights.shape[1] == 4 * class_nums
+
+
 class TestMultiBoxHead(unittest.TestCase):
     def test_multi_box_head(self):
         data_shape = [3, 224, 224]