Skip to content

Commit 8277347

Browse files
author
chengduo
authored
Add selu (#14415)
* add selu * use for range test=develop * add API test=develop * follow comment test=develop * update API.spec test=develop
1 parent 9d29ebc commit 8277347

File tree

6 files changed

+395
-0
lines changed

6 files changed

+395
-0
lines changed

‎paddle/fluid/API.spec

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@ paddle.fluid.layers.sequence_scatter ArgSpec(args=['input', 'index', 'updates',
128128
paddle.fluid.layers.random_crop ArgSpec(args=['x', 'shape', 'seed'], varargs=None, keywords=None, defaults=(None,))
129129
paddle.fluid.layers.mean_iou ArgSpec(args=['input', 'label', 'num_classes'], varargs=None, keywords=None, defaults=None)
130130
paddle.fluid.layers.relu ArgSpec(args=['x', 'name'], varargs=None, keywords=None, defaults=(None,))
131+
paddle.fluid.layers.selu ArgSpec(args=['x', 'scale', 'alpha', 'name'], varargs=None, keywords=None, defaults=(None, None, None))
131132
paddle.fluid.layers.log ArgSpec(args=['x', 'name'], varargs=None, keywords=None, defaults=(None,))
132133
paddle.fluid.layers.crop ArgSpec(args=['x', 'shape', 'offsets', 'name'], varargs=None, keywords=None, defaults=(None, None, None))
133134
paddle.fluid.layers.rank_loss ArgSpec(args=['label', 'left', 'right', 'name'], varargs=None, keywords=None, defaults=(None,))

‎paddle/fluid/operators/selu_op.cc

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. */
14+
15+
#include "paddle/fluid/operators/selu_op.h"
16+
#include <string>
17+
18+
namespace paddle {
19+
namespace operators {
20+
21+
class SeluOp : public framework::OperatorWithKernel {
22+
public:
23+
SeluOp(const std::string &type, const framework::VariableNameMap &inputs,
24+
const framework::VariableNameMap &outputs,
25+
const framework::AttributeMap &attrs)
26+
: OperatorWithKernel(type, inputs, outputs, attrs) {}
27+
28+
void InferShape(framework::InferShapeContext *ctx) const override {
29+
PADDLE_ENFORCE(ctx->HasInput("X"),
30+
"Input(X) of SeluOp should not be null.");
31+
PADDLE_ENFORCE(ctx->HasOutput("Out"),
32+
"Output(Out) of SeluOp should not be null.");
33+
34+
ctx->ShareDim("X", /*->*/ "Out");
35+
ctx->ShareLoD("X", /*->*/ "Out");
36+
}
37+
38+
protected:
39+
framework::OpKernelType GetExpectedKernelType(
40+
const framework::ExecutionContext &ctx) const override {
41+
return framework::OpKernelType(
42+
framework::GetDataTypeOfVar(ctx.InputVar("X")), ctx.GetPlace());
43+
}
44+
};
45+
46+
class SeluOpInferVarType : public framework::PassInDtypeAndVarTypeToOutput {
47+
protected:
48+
std::unordered_map<std::string, std::string> GetInputOutputWithSameType()
49+
const override {
50+
return std::unordered_map<std::string, std::string>{{"X", /*->*/ "Out"}};
51+
}
52+
};
53+
54+
class SeluOpMaker : public framework::OpProtoAndCheckerMaker {
55+
public:
56+
void Make() override {
57+
AddInput("X", "The input tensor of selu operator.");
58+
AddOutput("Out", "The output tensor of selu operator.");
59+
AddAttr<float>("scale",
60+
"(float) the default value is 1.0507~. For more "
61+
"information about this value, please refer to:"
62+
"https://arxiv.org/abs/1706.02515.")
63+
.SetDefault(1.0507009873554804934193349852946);
64+
AddAttr<float>("alpha",
65+
"(float) the default value is 1.6732~. For more "
66+
"information about this value, please refer to:"
67+
"https://arxiv.org/abs/1706.02515.")
68+
.SetDefault(1.6732632423543772848170429916717);
69+
AddComment(R"DOC(
70+
Selu Operator.
71+
72+
The equation is:
73+
$$
74+
f(x) =\lambda*
75+
\begin{cases}
76+
\quad \quad x, \quad \quad \quad \text{if} \ x > 0 \\
77+
\alpha * e^x - \alpha, \qquad \text{if} \ x <= 0
78+
\end{cases}
79+
$$
80+
81+
The input `X` can carry the LoD (Level of Details) information,
82+
or not. And the output shares the LoD information with input `X`.
83+
)DOC");
84+
}
85+
};
86+
87+
class SeluGradMaker : public framework::SingleGradOpDescMaker {
88+
public:
89+
using framework::SingleGradOpDescMaker::SingleGradOpDescMaker;
90+
91+
std::unique_ptr<framework::OpDesc> Apply() const override {
92+
auto *grad_op = new framework::OpDesc();
93+
grad_op->SetType("selu_grad");
94+
grad_op->SetInput("Out", Output("Out"));
95+
grad_op->SetInput(framework::GradVarName("Out"), OutputGrad("Out"));
96+
grad_op->SetOutput(framework::GradVarName("X"), InputGrad("X"));
97+
grad_op->SetAttrMap(this->Attrs());
98+
return std::unique_ptr<framework::OpDesc>(grad_op);
99+
}
100+
};
101+
102+
class SeluGradOp : public framework::OperatorWithKernel {
103+
public:
104+
using framework::OperatorWithKernel::OperatorWithKernel;
105+
106+
void InferShape(framework::InferShapeContext *ctx) const override {
107+
PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")),
108+
"Input(Out@GRAD) should not be null");
109+
PADDLE_ENFORCE(ctx->HasInput("Out"), "Input(Out) should not be null");
110+
auto x_grad_name = framework::GradVarName("X");
111+
ctx->SetOutputDim(x_grad_name, ctx->GetInputDim("Out"));
112+
}
113+
114+
protected:
115+
framework::OpKernelType GetExpectedKernelType(
116+
const framework::ExecutionContext &ctx) const override {
117+
return framework::OpKernelType(
118+
framework::GetDataTypeOfVar(ctx.InputVar("Out")), ctx.GetPlace());
119+
}
120+
};
121+
122+
} // namespace operators
123+
} // namespace paddle
124+
125+
namespace ops = paddle::operators;
126+
127+
REGISTER_OPERATOR(selu, ops::SeluOp, ops::SeluOpMaker, ops::SeluOpInferVarType,
128+
ops::SeluGradMaker);
129+
REGISTER_OPERATOR(selu_grad, ops::SeluGradOp);
130+
REGISTER_OP_CPU_KERNEL(
131+
selu, ops::SeluKernel<paddle::platform::CPUDeviceContext, float>,
132+
ops::SeluKernel<paddle::platform::CPUDeviceContext, double>);
133+
REGISTER_OP_CPU_KERNEL(
134+
selu_grad, ops::SeluGradKernel<paddle::platform::CPUDeviceContext, float>,
135+
ops::SeluGradKernel<paddle::platform::CPUDeviceContext, double>);

‎paddle/fluid/operators/selu_op.cu

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. */
14+
#include "paddle/fluid/operators/selu_op.h"
15+
16+
namespace ops = paddle::operators;
17+
REGISTER_OP_CUDA_KERNEL(
18+
selu, ops::SeluKernel<paddle::platform::CUDADeviceContext, float>,
19+
ops::SeluKernel<paddle::platform::CUDADeviceContext, double>);
20+
REGISTER_OP_CUDA_KERNEL(
21+
selu_grad, ops::SeluGradKernel<paddle::platform::CUDADeviceContext, float>,
22+
ops::SeluGradKernel<paddle::platform::CUDADeviceContext, double>);

‎paddle/fluid/operators/selu_op.h

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. */
14+
15+
#pragma once
16+
#include <string>
17+
#include "paddle/fluid/framework/op_registry.h"
18+
#include "paddle/fluid/platform/for_range.h"
19+
namespace paddle {
20+
namespace operators {
21+
22+
static HOSTDEVICE float real_exp(float x) { return expf(x); }
23+
static HOSTDEVICE float real_exp(double x) { return exp(x); }
24+
25+
template <typename T>
26+
struct SeluFunctor {
27+
SeluFunctor(const T* x_data_ptr, float alpha, float scale, T* y_data_ptr)
28+
: x_data_ptr_(x_data_ptr),
29+
alpha_(alpha),
30+
scale_(scale),
31+
y_data_ptr_(y_data_ptr) {}
32+
33+
HOSTDEVICE void operator()(size_t idx) const {
34+
T x_ele = x_data_ptr_[idx];
35+
if (x_ele <= 0) {
36+
x_ele = alpha_ * real_exp(x_ele) - alpha_;
37+
}
38+
y_data_ptr_[idx] = scale_ * x_ele;
39+
}
40+
const T* x_data_ptr_;
41+
const float alpha_;
42+
const float scale_;
43+
T* y_data_ptr_;
44+
};
45+
46+
template <typename T>
47+
struct SeluGradFunctor {
48+
SeluGradFunctor(const T* y_data_ptr, const T* dy_data_ptr, float alpha,
49+
float scale, T* dx_data_ptr)
50+
: y_data_ptr_(y_data_ptr),
51+
dy_data_ptr_(dy_data_ptr),
52+
alpha_(alpha),
53+
scale_(scale),
54+
la_(alpha * scale),
55+
dx_data_ptr_(dx_data_ptr) {}
56+
57+
HOSTDEVICE void operator()(size_t idx) const {
58+
T y_ele = y_data_ptr_[idx];
59+
T dy_ele = dy_data_ptr_[idx];
60+
61+
float tmp = scale_;
62+
if (y_ele <= 0) {
63+
tmp = y_ele + la_;
64+
}
65+
dx_data_ptr_[idx] = dy_ele * tmp;
66+
}
67+
const T* y_data_ptr_;
68+
const T* dy_data_ptr_;
69+
const float alpha_;
70+
const float scale_;
71+
const float la_;
72+
T* dx_data_ptr_;
73+
};
74+
75+
template <typename DeviceContext, typename T>
76+
class SeluKernel : public framework::OpKernel<T> {
77+
public:
78+
void Compute(const framework::ExecutionContext& context) const override {
79+
using Tensor = framework::Tensor;
80+
81+
auto* x = context.Input<Tensor>("X");
82+
auto* out = context.Output<Tensor>("Out");
83+
84+
float alpha = context.Attr<float>("alpha");
85+
float scale = context.Attr<float>("scale");
86+
87+
auto out_ptr = out->mutable_data<T>(context.GetPlace());
88+
89+
SeluFunctor<T> functor(x->data<T>(), alpha, scale, out_ptr);
90+
91+
auto& dev_ctx = context.template device_context<DeviceContext>();
92+
size_t limit = static_cast<size_t>(x->numel());
93+
platform::ForRange<DeviceContext> for_range(dev_ctx, limit);
94+
for_range(functor);
95+
}
96+
};
97+
98+
template <typename DeviceContext, typename T>
99+
class SeluGradKernel : public framework::OpKernel<T> {
100+
public:
101+
void Compute(const framework::ExecutionContext& context) const override {
102+
using Tensor = framework::Tensor;
103+
104+
auto* out = context.Input<Tensor>("Out");
105+
auto* dout = context.Input<Tensor>(framework::GradVarName("Out"));
106+
auto* dx = context.Output<Tensor>(framework::GradVarName("X"));
107+
108+
float alpha = context.Attr<float>("alpha");
109+
float scale = context.Attr<float>("scale");
110+
111+
auto dx_ptr = dx->mutable_data<T>(context.GetPlace());
112+
113+
SeluGradFunctor<T> functor(out->data<T>(), dout->data<T>(), alpha, scale,
114+
dx_ptr);
115+
116+
auto& dev_ctx = context.template device_context<DeviceContext>();
117+
size_t limit = static_cast<size_t>(out->numel());
118+
platform::ForRange<DeviceContext> for_range(dev_ctx, limit);
119+
for_range(functor);
120+
}
121+
};
122+
123+
} // namespace operators
124+
} // namespace paddle

‎python/paddle/fluid/layers/nn.py

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@
110110
'random_crop',
111111
'mean_iou',
112112
'relu',
113+
'selu',
113114
'log',
114115
'crop',
115116
'rank_loss',
@@ -6182,6 +6183,47 @@ def relu(x, name=None):
61826183
return out
61836184

61846185

6186+
@templatedoc()
6187+
def selu(x, scale=None, alpha=None, name=None):
6188+
"""
6189+
${comment}
6190+
6191+
Args:
6192+
x (Variable): The input tensor.
6193+
scale(float, None): If the scale is not set,
6194+
the default value is 1.0507009873554804934193349852946.
6195+
For more information about this value, please refer
6196+
to: https://arxiv.org/abs/1706.02515.
6197+
alpha(float, None): If the alpha is not set,
6198+
the default value is 1.6732632423543772848170429916717.
6199+
For more information about this value, please refer
6200+
to: https://arxiv.org/abs/1706.02515.
6201+
name (str|None, default None): A name for this layer If set None,
6202+
the layer will be named automatically.
6203+
6204+
Returns:
6205+
Variable: The output tensor with the same shape as input.
6206+
6207+
Examples:
6208+
6209+
.. code-block:: python
6210+
6211+
output = fluid.layers.selu(x)
6212+
"""
6213+
helper = LayerHelper('selu', **locals())
6214+
dtype = helper.input_dtype(input_param_name='x')
6215+
out = helper.create_variable_for_type_inference(dtype)
6216+
attrs = {}
6217+
if scale is not None:
6218+
attrs["scale"] = scale
6219+
if alpha is not None:
6220+
attrs["alpha"] = alpha
6221+
6222+
helper.append_op(
6223+
type="selu", inputs={"X": x}, outputs={"Out": out}, attrs=attrs)
6224+
return out
6225+
6226+
61856227
def mean_iou(input, label, num_classes):
61866228
"""
61876229
Mean Intersection-Over-Union is a common evaluation metric for

0 commit comments

Comments
 (0)