Skip to content

ES|QL: Add unsigned long and mixed numeric types support to Decay function #134440

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
Open

ES|QL: Add unsigned long and mixed numeric types support to Decay function #134440

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
d57a8e9
Add unsigned long support to Decay function
timgrein Sep 10, 2025
58a230c
Separate unsigned long and long evaluators
timgrein Sep 16, 2025
408d4c9
Use actual unsigned long in unsigned long integration test case
timgrein Sep 16, 2025
c86e56c
spotlessApply
timgrein Sep 16, 2025
f43b505
Support mixed numeric types for numeric decay
timgrein Sep 17, 2025
1319903
Add mixed numeric types test cases
timgrein Sep 17, 2025
eee150f
Use EsqlDataTypeConverter for most cases
timgrein Sep 17, 2025
13f103a
Add Evaluator
timgrein Sep 22, 2025
e35b86f
Merge branch 'main' into esql-decay-function-unsigned-long-support
timgrein Sep 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...reference/query-languages/esql/_snippets/functions/functionNamedParams/decay.md
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions docs/reference/query-languages/esql/_snippets/functions/types/decay.md
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

31 changes: 31 additions & 0 deletions docs/reference/query-languages/esql/kibana/definition/functions/decay.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -623,6 +623,10 @@ public static boolean isSortable(DataType t) {
return false == (t == SOURCE || isCounter(t) || isSpatialOrGrid(t) || t == AGGREGATE_METRIC_DOUBLE);
}

public static boolean isUnsignedLong(DataType t) {
return t == UNSIGNED_LONG;
}

public String nameUpper() {
return name;
}
Expand Down
11 changes: 11 additions & 0 deletions x-pack/plugin/esql/qa/testFixtures/src/main/resources/decay.csv-spec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,17 @@ decay_result:double
1.0
;

unsignedLongLinear
required_capability: decay_function

ROW value = 15::unsigned_long
| EVAL decay_result = decay(value, 10::unsigned_long, 10::unsigned_long, {"offset": 18446744073709551615, "decay": 0.5, "type": "linear"})
| KEEP decay_result;

decay_result:double
1.0
;

cartesianPointLinear1
required_capability: decay_function

Expand Down
43 changes: 37 additions & 6 deletions ...ql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/score/Decay.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.core.util.NumericUtils;
import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes;
import org.elasticsearch.xpack.esql.expression.function.Example;
import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesTo;
Expand Down Expand Up @@ -63,11 +64,13 @@
import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION;
import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
import static org.elasticsearch.xpack.esql.core.type.DataType.isDateNanos;
import static org.elasticsearch.xpack.esql.core.type.DataType.isGeoPoint;
import static org.elasticsearch.xpack.esql.core.type.DataType.isMillisOrNanos;
import static org.elasticsearch.xpack.esql.core.type.DataType.isSpatialPoint;
import static org.elasticsearch.xpack.esql.core.type.DataType.isTimeDuration;
import static org.elasticsearch.xpack.esql.core.type.DataType.isUnsignedLong;

/**
* Decay a numeric, spatial or date type value based on the distance of it to an origin.
Expand All @@ -91,7 +94,7 @@ public class Decay extends EsqlScalarFunction implements OptionalArgument, PostO

private static final Map<String, Collection<DataType>> ALLOWED_OPTIONS = Map.of(
OFFSET,
Set.of(TIME_DURATION, INTEGER, LONG, DOUBLE, KEYWORD, TEXT),
Set.of(TIME_DURATION, INTEGER, LONG, UNSIGNED_LONG, DOUBLE, KEYWORD, TEXT),
DECAY,
Set.of(DOUBLE),
TYPE,
Expand Down Expand Up @@ -140,25 +143,25 @@ public Decay(
Source source,
@Param(
name = "value",
type = { "double", "integer", "long", "date", "date_nanos", "geo_point", "cartesian_point" },
type = { "double", "integer", "long", "unsigned_long", "date", "date_nanos", "geo_point", "cartesian_point" },
description = "The input value to apply decay scoring to."
) Expression value,
@Param(
name = ORIGIN,
type = { "double", "integer", "long", "date", "date_nanos", "geo_point", "cartesian_point" },
type = { "double", "integer", "long", "unsigned_long", "date", "date_nanos", "geo_point", "cartesian_point" },
description = "Central point from which the distances are calculated."
) Expression origin,
@Param(
name = SCALE,
type = { "double", "integer", "long", "time_duration", "keyword", "text" },
type = { "double", "integer", "long", "unsigned_long", "time_duration", "keyword", "text" },
description = "Distance from the origin where the function returns the decay value."
) Expression scale,
@MapParam(
name = "options",
params = {
@MapParam.MapParamEntry(
name = OFFSET,
type = { "double", "integer", "long", "time_duration", "keyword", "text" },
type = { "double", "integer", "long", "unsigned_long", "time_duration", "keyword", "text" },
description = "Distance from the origin where no decay occurs."
),
@MapParam.MapParamEntry(
Expand Down Expand Up @@ -234,6 +237,8 @@ private TypeResolution validateOriginAndScale(DataType valueType) {
);
} else if (isMillisOrNanos(valueType)) {
return validateOriginAndScale(DataType::isMillisOrNanos, "datetime or date_nanos", DataType::isTimeDuration, "time_duration");
} else if (isUnsignedLong(valueType)) {
return validateOriginAndScale(DataType::isUnsignedLong, "unsigned long", DataType::isUnsignedLong, "unsigned_long");
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if it's enough to check if origin and scale are numeric? It should be possible to mix and match numeric types, i.e. use an unsigned long origin but an int scale etc

Copy link
Contributor Author

@timgrein timgrein Sep 17, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's a good point, I wanted to restrict it first to make it a bit simpler, but giving it a second thought it definitely makes sense to support mixed numeric types.

Adjusted with:

Full disclosure: we need to the same for offset. I'll add it to #134789, which I've opened yesterday and address it in a separate PR.

} else {
return validateOriginAndScale(DataType::isNumeric, "numeric", DataType::isNumeric, "numeric");
}
Expand Down Expand Up @@ -319,6 +324,15 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua
decayFolded,
decayFunction
);
case UNSIGNED_LONG -> new DecayUnsignedLongEvaluator.Factory(
source(),
valueFactory,
(Long) originFolded,
(Long) scaleFolded,
(Long) offsetFolded,
decayFolded,
decayFunction
);
case GEO_POINT -> new DecayGeoPointEvaluator.Factory(
source(),
valueFactory,
Expand Down Expand Up @@ -403,7 +417,24 @@ static double process(
@Fixed DecayFunction decayFunction
) {
return decayFunction.numericDecay(value, origin, scale, offset, decay);
}

@Evaluator(extraName = "UnsignedLong")
static double processUnsignedLong(
long value,
@Fixed long origin,
@Fixed long scale,
@Fixed long offset,
@Fixed double decay,
@Fixed DecayFunction decayFunction
) {
return decayFunction.numericDecay(
NumericUtils.unsignedLongToDouble(value),
NumericUtils.unsignedLongToDouble(origin),
NumericUtils.unsignedLongToDouble(scale),
NumericUtils.unsignedLongToDouble(offset),
decay
);
}

@Evaluator(extraName = "GeoPoint")
Expand Down Expand Up @@ -634,7 +665,7 @@ private Long getTemporalOffsetAsNanos(FoldContext foldCtx, Expression offset) {
private Object getDefaultOffset(DataType valueDataType) {
return switch (valueDataType) {
case INTEGER -> DEFAULT_INTEGER_OFFSET;
case LONG -> DEFAULT_LONG_OFFSET;
case LONG, UNSIGNED_LONG -> DEFAULT_LONG_OFFSET;
case DOUBLE -> DEFAULT_DOUBLE_OFFSET;
case GEO_POINT -> DEFAULT_GEO_POINT_OFFSET;
case CARTESIAN_POINT -> DEFAULT_CARTESIAN_POINT_OFFSET;
Expand Down
76 changes: 76 additions & 0 deletions ...c/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/score/DecayTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@
import org.elasticsearch.xpack.esql.core.expression.MapExpression;
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.core.util.NumericUtils;
import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;

import java.math.BigInteger;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
Expand Down Expand Up @@ -102,6 +104,9 @@ public static Iterable<Object[]> parameters() {
// Long random
testCaseSuppliers.addAll(longRandomTestCases());

// Unsigned Long random
testCaseSuppliers.addAll(unsignedLongRandomTestCases());

// Double Linear
testCaseSuppliers.addAll(doubleTestCase(0.0, 10.0, 10000000.0, 200.0, 0.25, "linear", 1.0));
testCaseSuppliers.addAll(doubleTestCase(10.0, 10.0, 10000000.0, 200.0, 0.25, "linear", 1.0));
Expand Down Expand Up @@ -747,6 +752,77 @@ private static double longDecayWithScoreScript(long value, long origin, long sca
};
}

private static List<TestCaseSupplier> unsignedLongRandomTestCases() {
return List.of(
new TestCaseSupplier(List.of(DataType.UNSIGNED_LONG, DataType.UNSIGNED_LONG, DataType.UNSIGNED_LONG, DataType.SOURCE), () -> {
BigInteger randomValueBig = randomUnsignedLongBetween(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX);
BigInteger randomOriginBig = randomUnsignedLongBetween(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX);
BigInteger randomScaleBig = randomUnsignedLongBetween(BigInteger.ONE, NumericUtils.UNSIGNED_LONG_MAX);
BigInteger randomOffsetBig = randomUnsignedLongBetween(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX);

// Convert to the signed long representation used internally
long randomValue = NumericUtils.asLongUnsigned(randomValueBig);
long randomOrigin = NumericUtils.asLongUnsigned(randomOriginBig);
long randomScale = NumericUtils.asLongUnsigned(randomScaleBig);
long randomOffset = NumericUtils.asLongUnsigned(randomOffsetBig);

double randomDecay = randomDouble();
String randomType = randomFrom("linear", "gauss", "exp");

double scoreScriptNumericResult = unsignedLongDecayWithScoreScript(
randomValue,
randomOrigin,
randomScale,
randomOffset,
randomDecay,
randomType
);

return new TestCaseSupplier.TestCase(
List.of(
new TestCaseSupplier.TypedData(randomValue, DataType.UNSIGNED_LONG, "value"),
new TestCaseSupplier.TypedData(randomOrigin, DataType.UNSIGNED_LONG, "origin").forceLiteral(),
new TestCaseSupplier.TypedData(randomScale, DataType.UNSIGNED_LONG, "scale").forceLiteral(),
new TestCaseSupplier.TypedData(createOptionsMap(randomOffset, randomDecay, randomType), DataType.SOURCE, "options")
.forceLiteral()
),
startsWith("DecayUnsignedLongEvaluator["),
DataType.DOUBLE,
equalTo(scoreScriptNumericResult)
);
})
);
}

private static double unsignedLongDecayWithScoreScript(long value, long origin, long scale, long offset, double decay, String type) {
var valueUnsignedLongAsDouble = NumericUtils.unsignedLongToDouble(value);
var originUnsignedLongAsDouble = NumericUtils.unsignedLongToDouble(origin);
var scaleUnsignedLongAsDouble = NumericUtils.unsignedLongToDouble(scale);
var offsetUnsignedLongAsDouble = NumericUtils.unsignedLongToDouble(offset);

return switch (type) {
case "linear" -> new ScoreScriptUtils.DecayNumericLinear(
originUnsignedLongAsDouble,
scaleUnsignedLongAsDouble,
offsetUnsignedLongAsDouble,
decay
).decayNumericLinear(valueUnsignedLongAsDouble);
case "gauss" -> new ScoreScriptUtils.DecayNumericGauss(
originUnsignedLongAsDouble,
scaleUnsignedLongAsDouble,
offsetUnsignedLongAsDouble,
decay
).decayNumericGauss(valueUnsignedLongAsDouble);
case "exp" -> new ScoreScriptUtils.DecayNumericExp(
originUnsignedLongAsDouble,
scaleUnsignedLongAsDouble,
offsetUnsignedLongAsDouble,
decay
).decayNumericExp(valueUnsignedLongAsDouble);
default -> throw new IllegalArgumentException("Unknown decay function type [" + type + "]");
};
}

private static List<TestCaseSupplier> doubleTestCase(
double value,
double origin,
Expand Down