Skip to content

Commit de19551

Browse files
panyx0718panyx0718
authored
Merge pull request #13621 from seiriosPlus/release/1.0.0
CherryPick BatchAUC/Distributed UT from develop
2 parents 501b58b + a8d2dd9 commit de19551

26 files changed

+1293
-208
lines changed

‎paddle/fluid/API.spec

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,7 @@ paddle.fluid.layers.iou_similarity ArgSpec(args=['x', 'y', 'name'], varargs=None
269269
paddle.fluid.layers.box_coder ArgSpec(args=['prior_box', 'prior_box_var', 'target_box', 'code_type', 'box_normalized', 'name'], varargs=None, keywords=None, defaults=('encode_center_size', True, None))
270270
paddle.fluid.layers.polygon_box_transform ArgSpec(args=['input', 'name'], varargs=None, keywords=None, defaults=(None,))
271271
paddle.fluid.layers.accuracy ArgSpec(args=['input', 'label', 'k', 'correct', 'total'], varargs=None, keywords=None, defaults=(1, None, None))
272-
paddle.fluid.layers.auc ArgSpec(args=['input', 'label', 'curve', 'num_thresholds', 'topk'], varargs=None, keywords=None, defaults=('ROC', 4095, 1))
272+
paddle.fluid.layers.auc ArgSpec(args=['input', 'label', 'curve', 'num_thresholds', 'topk', 'slide_steps'], varargs=None, keywords=None, defaults=('ROC', 4095, 1, 1))
273273
paddle.fluid.layers.exponential_decay ArgSpec(args=['learning_rate', 'decay_steps', 'decay_rate', 'staircase'], varargs=None, keywords=None, defaults=(False,))
274274
paddle.fluid.layers.natural_exp_decay ArgSpec(args=['learning_rate', 'decay_steps', 'decay_rate', 'staircase'], varargs=None, keywords=None, defaults=(False,))
275275
paddle.fluid.layers.inverse_time_decay ArgSpec(args=['learning_rate', 'decay_steps', 'decay_rate', 'staircase'], varargs=None, keywords=None, defaults=(False,))

‎paddle/fluid/framework/selected_rows_test.cc

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,11 @@ class SelectedRowsTester : public ::testing::Test {
2727
selected_rows_.reset(new SelectedRows(rows, height));
2828

2929
Tensor* value = selected_rows_->mutable_value();
30-
value->mutable_data<float>(
30+
auto* data = value->mutable_data<float>(
3131
make_ddim({static_cast<int64_t>(rows.size()), row_numel}), place_);
32+
for (int64_t i = 0; i < value->numel(); ++i) {
33+
data[i] = static_cast<float>(i);
34+
}
3235
}
3336

3437
protected:
@@ -60,6 +63,10 @@ TEST_F(SelectedRowsTester, SerializeAndDeseralize) {
6063
ASSERT_EQ(selected_rows_->height(), dst_tensor.height());
6164
ASSERT_EQ(selected_rows_->value().dims(), dst_tensor.value().dims());
6265
ASSERT_EQ(selected_rows_->GetCompleteDims(), dst_tensor.GetCompleteDims());
66+
auto* dst_data = dst_tensor.value().data<float>();
67+
for (int64_t i = 0; i < dst_tensor.value().numel(); ++i) {
68+
ASSERT_EQ(dst_data[i], static_cast<float>(i));
69+
}
6370
}
6471

6572
TEST(SelectedRows, SparseTable) {

‎paddle/fluid/operators/auc_op.cc

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,16 @@ class AucOp : public framework::OperatorWithKernel {
3636
"Out and Label should have same height.");
3737

3838
int num_pred_buckets = ctx->Attrs().Get<int>("num_thresholds") + 1;
39+
int slide_steps = ctx->Attrs().Get<int>("slide_steps");
40+
41+
PADDLE_ENFORCE_GE(num_pred_buckets, 1, "num_thresholds must larger than 1");
42+
PADDLE_ENFORCE_GE(slide_steps, 0, "slide_steps must be natural number");
3943

4044
ctx->SetOutputDim("AUC", {1});
41-
ctx->SetOutputDim("BatchAUC", {1});
42-
ctx->SetOutputDim("StatPosOut", {num_pred_buckets});
43-
ctx->SetOutputDim("StatNegOut", {num_pred_buckets});
45+
46+
slide_steps = slide_steps == 0 ? 1 : slide_steps;
47+
ctx->SetOutputDim("StatPosOut", {slide_steps, num_pred_buckets});
48+
ctx->SetOutputDim("StatNegOut", {slide_steps, num_pred_buckets});
4449
}
4550

4651
protected:
@@ -62,25 +67,27 @@ class AucOpMaker : public framework::OpProtoAndCheckerMaker {
6267
AddInput("Label",
6368
"A 2D int tensor indicating the label of the training data. "
6469
"shape: [batch_size, 1]");
70+
6571
// TODO(typhoonzero): support weight input
6672
AddInput("StatPos", "Statistic value when label = 1");
6773
AddInput("StatNeg", "Statistic value when label = 0");
6874

6975
AddOutput("AUC",
7076
"A scalar representing the "
7177
"current area-under-the-curve.");
72-
AddOutput("BatchAUC", "The AUC for current batch");
78+
7379
AddOutput("StatPosOut", "Statistic value when label = 1");
7480
AddOutput("StatNegOut", "Statistic value when label = 0");
7581

7682
AddAttr<std::string>("curve", "Curve type, can be 'ROC' or 'PR'.")
7783
.SetDefault("ROC");
7884

79-
AddAttr<int>("num_thresholds",
80-
"The number of thresholds to use when discretizing the"
81-
" roc curve.")
85+
AddAttr<int>(
86+
"num_thresholds",
87+
"The number of thresholds to use when discretizing the roc curve.")
8288
.SetDefault((2 << 12) - 1);
83-
89+
AddAttr<int>("slide_steps", "Use slide steps to calc batch auc.")
90+
.SetDefault(1);
8491
AddComment(R"DOC(
8592
Area Under The Curve (AUC) Operator.
8693

‎paddle/fluid/operators/auc_op.h

Lines changed: 70 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -32,24 +32,29 @@ class AucKernel : public framework::OpKernel<T> {
3232

3333
std::string curve = ctx.Attr<std::string>("curve");
3434
int num_thresholds = ctx.Attr<int>("num_thresholds");
35+
// buckets contain numbers from 0 to num_thresholds
3536
int num_pred_buckets = num_thresholds + 1;
37+
int slide_steps = ctx.Attr<int>("slide_steps");
3638

3739
// Only use output var for now, make sure it's persistable and
3840
// not cleaned up for each batch.
3941
auto *auc = ctx.Output<Tensor>("AUC");
4042
auto *stat_pos = ctx.Output<Tensor>("StatPosOut");
4143
auto *stat_neg = ctx.Output<Tensor>("StatNegOut");
4244

43-
auto *stat_pos_data = stat_pos->mutable_data<int64_t>(ctx.GetPlace());
44-
auto *stat_neg_data = stat_neg->mutable_data<int64_t>(ctx.GetPlace());
45-
calcAuc(ctx, label, predict, stat_pos_data, stat_neg_data, num_thresholds,
46-
auc);
45+
auto *origin_stat_pos = stat_pos->mutable_data<int64_t>(ctx.GetPlace());
46+
auto *origin_stat_neg = stat_neg->mutable_data<int64_t>(ctx.GetPlace());
4747

48-
auto *batch_auc = ctx.Output<Tensor>("BatchAUC");
49-
std::vector<int64_t> stat_pos_batch(num_pred_buckets, 0);
50-
std::vector<int64_t> stat_neg_batch(num_pred_buckets, 0);
51-
calcAuc(ctx, label, predict, stat_pos_batch.data(), stat_neg_batch.data(),
52-
num_thresholds, batch_auc);
48+
std::vector<int64_t> stat_pos_data(num_pred_buckets, 0);
49+
std::vector<int64_t> stat_neg_data(num_pred_buckets, 0);
50+
51+
auto stat_pos_calc = stat_pos_data.data();
52+
auto stat_neg_calc = stat_neg_data.data();
53+
54+
statAuc(label, predict, num_pred_buckets, num_thresholds, slide_steps,
55+
origin_stat_pos, origin_stat_neg, &stat_pos_calc, &stat_neg_calc);
56+
57+
calcAuc(ctx, stat_pos_calc, stat_neg_calc, num_thresholds, auc);
5358
}
5459

5560
private:
@@ -58,29 +63,76 @@ class AucKernel : public framework::OpKernel<T> {
5863
return (X1 > X2 ? (X1 - X2) : (X2 - X1)) * (Y1 + Y2) / 2.0;
5964
}
6065

61-
inline static void calcAuc(const framework::ExecutionContext &ctx,
62-
const framework::Tensor *label,
66+
inline static void statAuc(const framework::Tensor *label,
6367
const framework::Tensor *predict,
64-
int64_t *stat_pos, int64_t *stat_neg,
65-
int num_thresholds,
66-
framework::Tensor *auc_tensor) {
68+
const int num_pred_buckets,
69+
const int num_thresholds, const int slide_steps,
70+
int64_t *origin_stat_pos, int64_t *origin_stat_neg,
71+
int64_t **stat_pos, int64_t **stat_neg) {
6772
size_t batch_size = predict->dims()[0];
6873
size_t inference_width = predict->dims()[1];
6974
const T *inference_data = predict->data<T>();
7075
const auto *label_data = label->data<int64_t>();
7176

72-
auto *auc = auc_tensor->mutable_data<double>(ctx.GetPlace());
73-
7477
for (size_t i = 0; i < batch_size; i++) {
7578
uint32_t binIdx = static_cast<uint32_t>(
7679
inference_data[i * inference_width + 1] * num_thresholds);
7780
if (label_data[i]) {
78-
stat_pos[binIdx] += 1.0;
81+
(*stat_pos)[binIdx] += 1.0;
7982
} else {
80-
stat_neg[binIdx] += 1.0;
83+
(*stat_neg)[binIdx] += 1.0;
8184
}
8285
}
8386

87+
int bucket_length = num_pred_buckets * sizeof(int64_t);
88+
89+
// will stat auc unlimited.
90+
if (slide_steps == 0) {
91+
for (int slide = 0; slide < num_pred_buckets; ++slide) {
92+
origin_stat_pos[slide] += (*stat_pos)[slide];
93+
origin_stat_neg[slide] += (*stat_neg)[slide];
94+
}
95+
96+
*stat_pos = origin_stat_pos;
97+
*stat_neg = origin_stat_neg;
98+
99+
} else {
100+
for (int slide = 1; slide < slide_steps; ++slide) {
101+
int dst_idx = (slide - 1) * num_pred_buckets;
102+
int src_inx = slide * num_pred_buckets;
103+
std::memcpy(origin_stat_pos + dst_idx, origin_stat_pos + src_inx,
104+
bucket_length);
105+
std::memcpy(origin_stat_neg + dst_idx, origin_stat_neg + src_inx,
106+
bucket_length);
107+
}
108+
109+
std::memcpy(origin_stat_pos + (slide_steps - 1) * num_pred_buckets,
110+
*stat_pos, bucket_length);
111+
std::memcpy(origin_stat_neg + (slide_steps - 1) * num_pred_buckets,
112+
*stat_neg, bucket_length);
113+
114+
std::memset(*stat_pos, 0, bucket_length);
115+
std::memset(*stat_neg, 0, bucket_length);
116+
117+
for (int slide = 0; slide < num_pred_buckets; ++slide) {
118+
int stat_pos_steps = 0;
119+
int stat_neg_steps = 0;
120+
for (int step = 0; step < slide_steps; ++step) {
121+
stat_pos_steps += origin_stat_pos[slide + step * num_pred_buckets];
122+
stat_neg_steps += origin_stat_neg[slide + step * num_pred_buckets];
123+
}
124+
(*stat_pos)[slide] += stat_pos_steps;
125+
(*stat_neg)[slide] += stat_neg_steps;
126+
}
127+
}
128+
}
129+
130+
inline static void calcAuc(const framework::ExecutionContext &ctx,
131+
int64_t *stat_pos, int64_t *stat_neg,
132+
int num_thresholds,
133+
framework::Tensor *auc_tensor) {
134+
auto *auc = auc_tensor->mutable_data<double>(ctx.GetPlace());
135+
84136
*auc = 0.0f;
85137

86138
double totPos = 0.0;
@@ -96,7 +148,6 @@ class AucKernel : public framework::OpKernel<T> {
96148
totPos += stat_pos[idx];
97149
totNeg += stat_neg[idx];
98150
*auc += trapezoidArea(totNeg, totNegPrev, totPos, totPosPrev);
99-
100151
--idx;
101152
}
102153

‎paddle/fluid/operators/scale_op.cc

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,10 @@ class ScaleOpVarTypeInference : public framework::VarTypeInference {
7777
auto out_var_name = op_desc.Output("Out").front();
7878
auto *out_var = block->FindVarRecursive(out_var_name);
7979

80-
out_var->SetType(in_var.GetType());
81-
out_var->SetDataType(in_var.GetDataType());
80+
if (in_var_name != out_var_name) {
81+
out_var->SetType(in_var.GetType());
82+
out_var->SetDataType(in_var.GetDataType());
83+
}
8284
}
8385
};
8486

‎paddle/fluid/operators/sum_op.h

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ class SumKernel : public framework::OpKernel<T> {
3232
public:
3333
void Compute(const framework::ExecutionContext &context) const override {
3434
auto in_vars = context.MultiInputVar("X");
35-
int N = in_vars.size();
35+
size_t in_num = in_vars.size();
3636
auto out_var = context.OutputVar("Out");
3737

3838
bool in_place = out_var == in_vars[0];
@@ -53,7 +53,7 @@ class SumKernel : public framework::OpKernel<T> {
5353
auto &place =
5454
*context.template device_context<DeviceContext>().eigen_device();
5555
// If in_place, just skip the first tensor
56-
for (int i = in_place ? 1 : 0; i < N; i++) {
56+
for (size_t i = in_place ? 1 : 0; i < in_num; i++) {
5757
if (in_vars[i]->IsType<framework::LoDTensor>()) {
5858
auto &in_t = in_vars[i]->Get<framework::LoDTensor>();
5959
if (in_t.numel() == 0) {
@@ -101,13 +101,13 @@ class SumKernel : public framework::OpKernel<T> {
101101

102102
// Runtime InferShape
103103
size_t first_dim = 0;
104-
for (int i = 0; i < N; i++) {
104+
for (size_t i = 0; i < in_num; i++) {
105105
auto &sel_row = get_selected_row(i);
106106
first_dim += sel_row.rows().size();
107107
}
108108

109109
std::vector<int64_t> in_dim;
110-
for (int i = 0; i < N; i++) {
110+
for (size_t i = 0; i < in_num; i++) {
111111
auto &sel_row = get_selected_row(i);
112112
if (sel_row.rows().size() > 0) {
113113
in_dim = framework::vectorize(sel_row.value().dims());
@@ -116,7 +116,8 @@ class SumKernel : public framework::OpKernel<T> {
116116
}
117117
if (in_dim.empty()) {
118118
VLOG(3) << "WARNING: all the inputs are empty";
119-
in_dim = framework::vectorize(get_selected_row(N - 1).value().dims());
119+
in_dim =
120+
framework::vectorize(get_selected_row(in_num - 1).value().dims());
120121
} else {
121122
in_dim[0] = static_cast<int64_t>(first_dim);
122123
}
@@ -133,7 +134,7 @@ class SumKernel : public framework::OpKernel<T> {
133134
math::SelectedRowsAddTo<DeviceContext, T> functor;
134135

135136
int64_t offset = 0;
136-
for (int i = 0; i < N; i++) {
137+
for (size_t i = 0; i < in_num; i++) {
137138
auto &sel_row = get_selected_row(i);
138139
if (sel_row.rows().size() == 0) {
139140
continue;

‎python/paddle/dataset/common.py

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -77,13 +77,14 @@ def download(url, module_name, md5sum, save_name=None):
7777
retry_limit = 3
7878
while not (os.path.exists(filename) and md5file(filename) == md5sum):
7979
if os.path.exists(filename):
80-
print("file md5", md5file(filename), md5sum)
80+
sys.stderr.write("file %s md5 %s" % (md5file(filename), md5sum))
8181
if retry < retry_limit:
8282
retry += 1
8383
else:
8484
raise RuntimeError("Cannot download {0} within retry limit {1}".
8585
format(url, retry_limit))
86-
print("Cache file %s not found, downloading %s" % (filename, url))
86+
sys.stderr.write("Cache file %s not found, downloading %s" %
87+
(filename, url))
8788
r = requests.get(url, stream=True)
8889
total_length = r.headers.get('content-length')
8990

@@ -100,10 +101,11 @@ def download(url, module_name, md5sum, save_name=None):
100101
dl += len(data)
101102
f.write(data)
102103
done = int(50 * dl / total_length)
103-
sys.stdout.write("\r[%s%s]" % ('=' * done,
104+
sys.stderr.write("\r[%s%s]" % ('=' * done,
104105
' ' * (50 - done)))
105106
sys.stdout.flush()
106-
107+
sys.stderr.write("\n")
108+
sys.stdout.flush()
107109
return filename
108110

109111

0 commit comments

Comments
 (0)