Skip to content

Commit 62401d8

Browse files
committed
Proper formula and reference for xIELU in the comments.
1 parent 57e2263 commit 62401d8

File tree

1 file changed

+2
-1
lines changed

1 file changed

+2
-1
lines changed

ggml/include/ggml.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1150,7 +1150,8 @@ extern "C" {
11501150
struct ggml_tensor * a);
11511151

11521152
// xIELU activation function
1153-
// x = x * (alpha_n + alpha_p * sigmoid(beta * x)) + eps * (x > 0)
1153+
// https://arxiv.org/abs/2411.13010
1154+
// x = (x > 0) ? a_p * x^2 + b * x : (exmp1(min(x, eps)) - x) * a_n + b * x
11541155
GGML_API struct ggml_tensor * ggml_xielu(
11551156
struct ggml_context * ctx,
11561157
struct ggml_tensor * a,

0 commit comments

Comments
 (0)