Skip to content

Commit a4dac31

Browse files
vera121vera121
committed
Merge branch 'development' of https://github.com/foss-for-synopsys-dwc-arc-processors/synopsys-caffe into development
2 parents 6328d7f + 6dde8cf commit a4dac31

File tree

10 files changed

+371
-31
lines changed

10 files changed

+371
-31
lines changed

‎FEATURES.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ mirror_pad_layer
3535
mish_layer
3636
mul_layer
3737
nms_layer
38+
non_max_suppression_layer
3839
one_hot_layer
3940
pad_layer
4041
peephole_lstm_layer

‎include/caffe/layers/nms_layer.hpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@
88
#include "caffe/proto/caffe.pb.h"
99

1010
namespace caffe {
11+
/**
12+
* @brief TensorFlow style implementation of NMS.
13+
*
14+
* ref: https://www.tensorflow.org/api_docs/python/tf/image/non_max_suppression
15+
*/
1116

1217
template <typename Dtype>
1318
class NMSLayer : public Layer<Dtype> {

‎include/caffe/layers/non_max_suppression_layer.hpp‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
#ifndef CAFFE_NON_MAX_SUPPRESSION_LAYER_HPP_
2+
#define CAFFE_NON_MAX_SUPPRESSION_LAYER_HPP_
3+
4+
#include <vector>
5+
6+
#include "caffe/blob.hpp"
7+
#include "caffe/layer.hpp"
8+
#include "caffe/proto/caffe.pb.h"
9+
10+
namespace caffe {
11+
/**
12+
* @brief ONNX style implementation of NonMaxSuppression.
13+
*
14+
* Filter out boxes that have high intersection-over-union (IOU) overlap with previously selected boxes.
15+
* Bounding boxes with score less than score_threshold are removed.
16+
* Bounding box format is indicated by attribute center_point_box.
17+
* ref: https://github.com/onnx/onnx/blob/master/docs/Operators.md#NonMaxSuppression
18+
*/
19+
20+
template <typename Dtype>
21+
class NonMaxSuppressionLayer : public Layer<Dtype> {
22+
public:
23+
24+
explicit NonMaxSuppressionLayer(const LayerParameter& param)
25+
: Layer<Dtype>(param) {}
26+
virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
27+
const vector<Blob<Dtype>*>& top);
28+
virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
29+
const vector<Blob<Dtype>*>& top);
30+
31+
virtual inline const char* type() const { return "NonMaxSuppression"; }
32+
//virtual inline int MinBottomBlobs() const { return 1; }
33+
//virtual inline int MaxBottomBlobs() const { return 2; }
34+
virtual inline int ExactNumBottomBlobs() const { return 2; }
35+
virtual inline int ExactNumTopBlobs() const { return 1; }
36+
37+
protected:
38+
39+
virtual bool SuppressByIOU(const Dtype* boxes_data, int64_t box_index1,
40+
int64_t box_index2, int64_t center_point_box, float iou_threshold);
41+
virtual void MaxMin(float lhs, float rhs, float& min, float& max);
42+
43+
virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
44+
const vector<Blob<Dtype>*>& top);
45+
/// @brief Not implemented (non-differentiable function)
46+
virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
47+
const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {
48+
NOT_IMPLEMENTED;
49+
}
50+
51+
52+
struct ScoreIndexPair {
53+
Dtype score_{};
54+
int64_t index_{};
55+
56+
ScoreIndexPair() = default;
57+
explicit ScoreIndexPair(Dtype score, int64_t idx) : score_(score), index_(idx) {}
58+
59+
bool operator<(const ScoreIndexPair& rhs) const {
60+
return score_ < rhs.score_;
61+
}
62+
};
63+
64+
int max_output_boxes_per_class_;
65+
float iou_threshold_;
66+
float score_threshold_;
67+
int center_point_box_;
68+
69+
int num_batches_;
70+
int num_classes_;
71+
int num_boxes_ ;
72+
};
73+
74+
} // namespace caffe
75+
76+
#endif // CAFFE_NON_MAX_SUPPRESSION_LAYER_HPP_

‎include/caffe/layers/pow_layer.hpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ class PowLayer : public Layer<Dtype> {
2121
public:
2222
explicit PowLayer(const LayerParameter& param)
2323
: Layer<Dtype>(param) {}
24+
virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
25+
const vector<Blob<Dtype>*>& top);
2426
virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
2527
const vector<Blob<Dtype>*>& top);
2628

‎include/caffe/layers/resize_nearest_neighbor_layer.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ class ResizeNearestNeighborLayer : public Layer<Dtype> {
4949
bool align_corners;
5050
string data_format;
5151
bool half_pixel_centers;
52+
bool half_pixel_onnx;
5253
};
5354

5455
} // namespace caffe

‎src/caffe/layers/non_max_suppression_layer.cpp‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
#include <algorithm>
2+
#include <functional>
3+
#include <utility>
4+
#include <vector>
5+
#include <queue>
6+
#define HelperMin(a, b) std::min(a, b)
7+
#define HelperMax(a, b) std::max(a, b)
8+
9+
#include "caffe/layers/non_max_suppression_layer.hpp"
10+
11+
namespace caffe {
12+
13+
template <typename Dtype>
14+
void NonMaxSuppressionLayer<Dtype>::LayerSetUp(const vector<Blob<Dtype>*>& bottom,
15+
const vector<Blob<Dtype>*>& top) {
16+
const NonMaxSuppressionParameter& non_max_suppression_param = this->layer_param_.non_max_suppression_param();
17+
max_output_boxes_per_class_ = non_max_suppression_param.max_output_boxes_per_class();
18+
iou_threshold_ = non_max_suppression_param.iou_threshold();
19+
score_threshold_ = non_max_suppression_param.score_threshold();
20+
center_point_box_ = non_max_suppression_param.center_point_box();
21+
22+
CHECK_EQ(bottom[0]->num_axes(), 3) << "bottom[0] must have 3 axes.";
23+
CHECK_EQ(bottom[1]->num_axes(), 3) << "bottom[1] must have 3 axes.";
24+
25+
CHECK_EQ(bottom[0]->shape(0), bottom[1]->shape(0)) << "The boxes and scores should have same num_batches.";
26+
CHECK_EQ(bottom[0]->shape(1), bottom[1]->shape(2)) << "The boxes and scores should have same spatial_dimension.";
27+
CHECK_EQ(bottom[0]->shape(2), 4) << "This coordinates axis of boxes must have shape 4.";
28+
num_batches_ = bottom[0]->shape(0);
29+
num_classes_ = bottom[1]->shape(1);
30+
num_boxes_ = bottom[0]->shape(1);
31+
32+
CHECK_GE(iou_threshold_, 0) << "The iou_threshold must not be less than 0.";
33+
CHECK_LE(iou_threshold_, 1) << "The iou_threshold must not be greater than 1.";
34+
35+
}
36+
37+
template <typename Dtype>
38+
void NonMaxSuppressionLayer<Dtype>::Reshape(const vector<Blob<Dtype>*>& bottom,
39+
const vector<Blob<Dtype>*>& top) {
40+
std::vector<int> shape(2);
41+
shape[0] = max_output_boxes_per_class_ * num_classes_ * num_batches_;
42+
shape[1] = 3;
43+
top[0]->Reshape(shape);
44+
}
45+
46+
47+
template <typename Dtype>
48+
void NonMaxSuppressionLayer<Dtype>::MaxMin(float lhs, float rhs, float& min, float& max) {
49+
if (lhs >= rhs) {
50+
min = rhs;
51+
max = lhs;
52+
} else {
53+
min = lhs;
54+
max = rhs;
55+
}
56+
}
57+
58+
59+
template <typename Dtype>
60+
bool NonMaxSuppressionLayer<Dtype>::SuppressByIOU(const Dtype* boxes_data, int64_t box_index1, int64_t box_index2,
61+
int64_t center_point_box, float iou_threshold) {
62+
float x1_min{};
63+
float y1_min{};
64+
float x1_max{};
65+
float y1_max{};
66+
float x2_min{};
67+
float y2_min{};
68+
float x2_max{};
69+
float y2_max{};
70+
71+
const Dtype* box1 = boxes_data + 4 * box_index1;
72+
const Dtype* box2 = boxes_data + 4 * box_index2;
73+
// center_point_box_ only support 0 or 1
74+
if (0 == center_point_box) {
75+
// boxes data format [y1, x1, y2, x2],
76+
MaxMin(box1[1], box1[3], x1_min, x1_max);
77+
MaxMin(box1[0], box1[2], y1_min, y1_max);
78+
MaxMin(box2[1], box2[3], x2_min, x2_max);
79+
MaxMin(box2[0], box2[2], y2_min, y2_max);
80+
} else {
81+
// 1 == center_point_box_ => boxes data format [x_center, y_center, width, height]
82+
float box1_width_half = box1[2] / 2;
83+
float box1_height_half = box1[3] / 2;
84+
float box2_width_half = box2[2] / 2;
85+
float box2_height_half = box2[3] / 2;
86+
87+
x1_min = box1[0] - box1_width_half;
88+
x1_max = box1[0] + box1_width_half;
89+
y1_min = box1[1] - box1_height_half;
90+
y1_max = box1[1] + box1_height_half;
91+
92+
x2_min = box2[0] - box2_width_half;
93+
x2_max = box2[0] + box2_width_half;
94+
y2_min = box2[1] - box2_height_half;
95+
y2_max = box2[1] + box2_height_half;
96+
}
97+
98+
const float intersection_x_min = HelperMax(x1_min, x2_min);
99+
const float intersection_y_min = HelperMax(y1_min, y2_min);
100+
const float intersection_x_max = HelperMin(x1_max, x2_max);
101+
const float intersection_y_max = HelperMin(y1_max, y2_max);
102+
103+
const float intersection_area = HelperMax(intersection_x_max - intersection_x_min, .0f) *
104+
HelperMax(intersection_y_max - intersection_y_min, .0f);
105+
106+
if (intersection_area <= .0f) {
107+
return false;
108+
}
109+
110+
const float area1 = (x1_max - x1_min) * (y1_max - y1_min);
111+
const float area2 = (x2_max - x2_min) * (y2_max - y2_min);
112+
const float union_area = area1 + area2 - intersection_area;
113+
114+
if (area1 <= .0f || area2 <= .0f || union_area <= .0f) {
115+
return false;
116+
}
117+
118+
const float intersection_over_union = intersection_area / union_area;
119+
120+
return intersection_over_union > iou_threshold;
121+
}
122+
123+
template <typename Dtype>
124+
void NonMaxSuppressionLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
125+
const vector<Blob<Dtype>*>& top) {
126+
const Dtype* boxes_data = bottom[0]->cpu_data();
127+
const Dtype* scores_data = bottom[1]->cpu_data();
128+
129+
std::vector<int64_t> selected_indices;
130+
for (int64_t batch_index = 0; batch_index < num_batches_; ++batch_index) {
131+
for (int64_t class_index = 0; class_index < num_classes_; ++class_index) {
132+
int64_t box_score_offset = (batch_index * num_classes_ + class_index) * num_boxes_;
133+
int64_t box_offset = batch_index * num_boxes_ * 4;
134+
135+
// Filter by score_threshold_
136+
std::priority_queue<ScoreIndexPair, std::deque<ScoreIndexPair>> sorted_scores_with_index;
137+
const Dtype* class_scores = scores_data + box_score_offset;
138+
for (int64_t box_index = 0; box_index < num_boxes_; ++box_index, ++class_scores) {
139+
if (*class_scores > score_threshold_) {
140+
sorted_scores_with_index.push(ScoreIndexPair(*class_scores, box_index));
141+
}
142+
}
143+
144+
ScoreIndexPair next_top_score;
145+
std::vector<int64_t> selected_indices_inside_class;
146+
// Get the next box with top score, filter by iou_threshold
147+
while (!sorted_scores_with_index.empty()) {
148+
next_top_score = sorted_scores_with_index.top();
149+
sorted_scores_with_index.pop();
150+
151+
bool selected = true;
152+
// Check with existing selected boxes for this class, suppress if exceed the IOU (Intersection Over Union) threshold
153+
for (int64_t selected_index : selected_indices_inside_class) {
154+
if (SuppressByIOU(boxes_data + box_offset, selected_index, next_top_score.index_,
155+
center_point_box_, iou_threshold_)) {
156+
selected = false;
157+
break;
158+
}
159+
}
160+
161+
if (selected) {
162+
if (max_output_boxes_per_class_ > 0 &&
163+
static_cast<int64_t>(selected_indices_inside_class.size()) >= max_output_boxes_per_class_) {
164+
break;
165+
}
166+
selected_indices_inside_class.push_back(next_top_score.index_);
167+
//selected_indices.emplace_back(batch_index, class_index, next_top_score.index_);
168+
selected_indices.emplace_back(batch_index);
169+
selected_indices.emplace_back(class_index);
170+
selected_indices.emplace_back(next_top_score.index_);
171+
}
172+
} //while
173+
174+
} //for class_index
175+
} //for batch_index
176+
177+
Dtype* top_data = top[0]->mutable_cpu_data();
178+
const int64_t num_selected = selected_indices.size();
179+
for(int i=0; i<num_selected; i++)
180+
top_data[i] = selected_indices[i];
181+
// Note: fixed output shape might be larger than the count of selected indices,
182+
// the following parts will be filled by 0.
183+
}
184+
185+
INSTANTIATE_CLASS(NonMaxSuppressionLayer);
186+
REGISTER_LAYER_CLASS(NonMaxSuppression);
187+
188+
} // namespace caffe

‎src/caffe/layers/pow_layer.cpp‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,28 @@
66
#include "caffe/layers/pow_layer.hpp"
77

88
namespace caffe {
9+
template <typename Dtype>
10+
void PowLayer<Dtype>::LayerSetUp(const vector<Blob<Dtype>*>& bottom,
11+
const vector<Blob<Dtype>*>& top) {
12+
// Add the fake initialization to avoid special case in evconvert for the moment when
13+
// the prototxt is generated, the 2nd input is parameter to be fed into caffemodel, but the caffemodel is not generated yet
14+
if (bottom.size() == 1 && this->blobs_.size() == 0)
15+
{
16+
LOG(WARNING) << "Note: Make fake parameter initialization to avoid segment fault!";
17+
// initialize the 2nd input to avoid error in reshape stage
18+
this->blobs_.resize(1);
19+
const vector<int>::const_iterator& shape_start =
20+
bottom[0]->shape().begin();
21+
const vector<int>::const_iterator& shape_end = bottom[0]->shape().end();
22+
vector<int> second_shape(shape_start, shape_end);
23+
this->blobs_[0].reset(new Blob<Dtype>(second_shape));
24+
// Note: seems not to support broadcasting data value initialize when
25+
// input1 shape < input2 shape in this special situation
26+
27+
this->param_propagate_down_.resize(this->blobs_.size(), true);
28+
}
29+
}
30+
931
template <typename Dtype>
1032
void PowLayer<Dtype>::Reshape(const vector<Blob<Dtype>*>& bottom,
1133
const vector<Blob<Dtype>*>& top) {

0 commit comments

Comments
 (0)