From b94d9d8929ca0acdb2a34063feb15189ec939b88 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 14 Aug 2025 08:53:47 -0400 Subject: [PATCH 1/4] ESQL: Mark new signatures in MIN and MAX The `MIN` and `MAX` functions support unsigned longs in 9.2.0. The docs just show it's that unsigned long is supported but you can't tell for what version. This fixes that, applying the version. The version table is generated from the tests. So we have to annotate some test cases with `applies_to` and pass it through the test infrastructure. --- .../esql/_snippets/functions/types/max.md | 2 +- .../functions/types/max_over_time.md | 2 +- .../esql/_snippets/functions/types/min.md | 2 +- .../functions/types/min_over_time.md | 2 +- .../xpack/esql/action/LookupJoinTypesIT.java | 9 ++- .../function/AbstractFunctionTestCase.java | 28 +++---- .../expression/function/DocsV3Support.java | 67 +++++++++------- .../function/DocsV3SupportTests.java | 4 +- .../function/MultiRowTestCaseSupplier.java | 11 ++- .../expression/function/TestCaseSupplier.java | 80 ++++++++++++++++--- .../aggregate/CountDistinctTests.java | 9 ++- .../aggregate/FirstOverTimeTests.java | 9 ++- .../function/aggregate/LastOverTimeTests.java | 9 ++- .../function/aggregate/MaxTests.java | 20 ++++- .../function/aggregate/MinTests.java | 20 ++++- .../function/aggregate/RateTests.java | 9 ++- .../predicate/operator/CastOperatorTests.java | 2 +- 17 files changed, 205 insertions(+), 80 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/max.md b/docs/reference/query-languages/esql/_snippets/functions/types/max.md index e930fc056c8d8..8368c8fdd4919 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/max.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/max.md @@ -13,6 +13,6 @@ | keyword | keyword | | long | long | | text | keyword | -| unsigned_long | unsigned_long | +| unsigned_long{applies_to}`stack: ga 9.2.0` | unsigned_long | | version | version | diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/max_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/types/max_over_time.md index e930fc056c8d8..8368c8fdd4919 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/max_over_time.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/max_over_time.md @@ -13,6 +13,6 @@ | keyword | keyword | | long | long | | text | keyword | -| unsigned_long | unsigned_long | +| unsigned_long{applies_to}`stack: ga 9.2.0` | unsigned_long | | version | version | diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/min.md b/docs/reference/query-languages/esql/_snippets/functions/types/min.md index e930fc056c8d8..8368c8fdd4919 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/min.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/min.md @@ -13,6 +13,6 @@ | keyword | keyword | | long | long | | text | keyword | -| unsigned_long | unsigned_long | +| unsigned_long{applies_to}`stack: ga 9.2.0` | unsigned_long | | version | version | diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/min_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/types/min_over_time.md index e930fc056c8d8..8368c8fdd4919 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/min_over_time.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/min_over_time.md @@ -13,6 +13,6 @@ | keyword | keyword | | long | long | | text | keyword | -| unsigned_long | unsigned_long | +| unsigned_long{applies_to}`stack: ga 9.2.0` | unsigned_long | | version | version | diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java index 0a1bc37094762..06645ca3af517 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java @@ -270,14 +270,17 @@ private static boolean existingIndex(Collection existing, DataType /** This test generates documentation for the supported output types of the lookup join. */ public void testOutputSupportedTypes() throws Exception { - Map, DataType> signatures = new LinkedHashMap<>(); + Map, DataType> signatures = new LinkedHashMap<>(); for (TestConfigs configs : testConfigurations.values()) { if (configs.group.equals("unsupported") || configs.group.equals("union-types")) { continue; } for (TestConfig config : configs.configs.values()) { if (config instanceof TestConfigPasses) { - signatures.put(List.of(config.mainType(), config.lookupType()), null); + signatures.put( + List.of(new DocsV3Support.Param(config.mainType(), List.of()), new DocsV3Support.Param(config.lookupType(), null)), + null + ); } } } @@ -767,7 +770,7 @@ private boolean isValidDataType(DataType dataType) { return UNDER_CONSTRUCTION.get(dataType) == null || UNDER_CONSTRUCTION.get(dataType).isEnabled(); } - private static void saveJoinTypes(Supplier, DataType>> signatures) throws Exception { + private static void saveJoinTypes(Supplier, DataType>> signatures) throws Exception { if (System.getProperty("generateDocs") == null) { return; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index 00f20b9376a6f..62c3cd389401a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -758,10 +758,10 @@ public static void testFunctionInfo() { for (int i = 0; i < args.size(); i++) { typesFromSignature.add(new HashSet<>()); } - for (Map.Entry, DataType> entry : signatures(testClass).entrySet()) { - List types = entry.getKey(); + for (Map.Entry, DataType> entry : signatures(testClass).entrySet()) { + List types = entry.getKey(); for (int i = 0; i < args.size() && i < types.size(); i++) { - typesFromSignature.get(i).add(types.get(i).esNameIfPossible()); + typesFromSignature.get(i).add(types.get(i).dataType().esNameIfPossible()); } if (DataType.UNDER_CONSTRUCTION.containsKey(entry.getValue()) == false) { returnFromSignature.add(entry.getValue().esNameIfPossible()); @@ -840,14 +840,14 @@ public static void testFunctionLicenseChecks() throws Exception { // Go through all signatures and assert that the license is as expected signatures(testClass).forEach((signature, returnType) -> { try { - License.OperationMode license = licenseChecker.invoke(signature); + License.OperationMode license = licenseChecker.invoke(signature.stream().map(DocsV3Support.Param::dataType).toList()); assertNotNull("License should not be null", license); // Construct an instance of the class and then call it's licenseCheck method, and compare the results Object[] args = new Object[ctor.getParameterCount()]; args[0] = Source.EMPTY; for (int i = 0; i < signature.size(); i++) { - args[i + 1] = new Literal(Source.EMPTY, null, signature.get(i)); + args[i + 1] = new Literal(Source.EMPTY, null, signature.get(i).dataType()); } Object instance = ctor.newInstance(args); // Check that object implements the LicenseAware interface @@ -874,7 +874,7 @@ private static class TestCheckLicense { private void assertLicenseCheck( LicenseAware licenseAware, - List signature, + List signature, boolean allowsBasic, boolean allowsPlatinum, boolean allowsEnterprise @@ -933,9 +933,9 @@ protected final void assertTypeResolutionFailure(Expression expression) { /** * Unique signatures in this test’s parameters. */ - private static Map, DataType> signatures; + private static Map, DataType> signatures; - public static Map, DataType> signatures(Class testClass) { + public static Map, DataType> signatures(Class testClass) { if (signatures != null && classGeneratingSignatures == testClass) { return signatures; } @@ -959,17 +959,17 @@ public static Map, DataType> signatures(Class testClass) { if (tc.getData().stream().anyMatch(t -> t.type() == DataType.NULL)) { continue; } - List types = tc.getData().stream().map(TestCaseSupplier.TypedData::type).toList(); - signatures.putIfAbsent(signatureTypes(testClass, types), tc.expectedType()); + List sig = tc.getData().stream().map(d -> new DocsV3Support.Param(d.type(), d.appliesTo())).toList(); + signatures.putIfAbsent(signatureTypes(testClass, sig), tc.expectedType()); } return signatures; } @SuppressWarnings("unchecked") - private static List signatureTypes(Class testClass, List types) { + private static List signatureTypes(Class testClass, List types) { try { Method method = testClass.getMethod("signatureTypes", List.class); - return (List) method.invoke(null, types); + return (List) method.invoke(null, types); } catch (NoSuchMethodException ingored) { return types; } catch (Exception e) { @@ -1053,9 +1053,9 @@ private static boolean isAggregation() { /** * Should this particular signature be hidden from the docs even though we test it? */ - static boolean shouldHideSignature(List argTypes, DataType returnType) { + static boolean shouldHideSignature(List argTypes, DataType returnType) { for (DataType dt : DataType.UNDER_CONSTRUCTION.keySet()) { - if (returnType == dt || argTypes.contains(dt)) { + if (returnType == dt || argTypes.stream().anyMatch(p -> p.dataType() == dt)) { return true; } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java index 826d8534d7587..182895be8bce6 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java @@ -99,6 +99,8 @@ * and partially re-written to satisfy the above requirements. */ public abstract class DocsV3Support { + public record Param(DataType dataType, List appliesTo) {} + private static final Logger logger = LogManager.getLogger(DocsV3Support.class); private static final String DOCS_WARNING_JSON = @@ -372,7 +374,7 @@ public License.OperationMode invoke(List fieldTypes) throws Exception protected final String category; protected final String name; protected final FunctionDefinition definition; - protected final Supplier, DataType>> signatures; + protected final Supplier, DataType>> signatures; protected final Callbacks callbacks; private final LicenseRequirementChecker licenseChecker; @@ -380,7 +382,7 @@ protected DocsV3Support( String category, String name, Class testClass, - Supplier, DataType>> signatures, + Supplier, DataType>> signatures, Callbacks callbacks ) { this(category, name, null, testClass, signatures, callbacks); @@ -391,7 +393,7 @@ private DocsV3Support( String name, FunctionDefinition definition, Class testClass, - Supplier, DataType>> signatures, + Supplier, DataType>> signatures, Callbacks callbacks ) { this.category = category; @@ -571,7 +573,7 @@ private FunctionDocsSupport(String name, Class testClass, Callbacks callbacks String name, Class testClass, FunctionDefinition definition, - Supplier, DataType>> signatures, + Supplier, DataType>> signatures, Callbacks callbacks ) { super("functions", name, definition, testClass, signatures, callbacks); @@ -662,10 +664,10 @@ private void renderFunctionNamedParams(EsqlFunctionRegistry.MapArgSignature mapA writeToTempSnippetsDir("functionNamedParams", rendered.toString()); } - private String makeAppliesToText(FunctionAppliesTo[] functionAppliesTos, boolean preview) { + private static String makeAppliesToText(List functionAppliesTos, boolean preview, boolean oneLine) { StringBuilder appliesToText = new StringBuilder(); - if (functionAppliesTos.length > 0) { - appliesToText.append("```{applies_to}\n"); + if (false == functionAppliesTos.isEmpty()) { + appliesToText.append(oneLine ? "{applies_to}`" : "```{applies_to}\n"); StringBuilder stackEntries = new StringBuilder(); for (FunctionAppliesTo appliesTo : functionAppliesTos) { @@ -680,15 +682,21 @@ private String makeAppliesToText(FunctionAppliesTo[] functionAppliesTos, boolean // Add the stack entries if (stackEntries.isEmpty() == false) { - appliesToText.append("stack: ").append(stackEntries).append("\n"); + appliesToText.append("stack: ").append(stackEntries); + if (false == oneLine) { + appliesToText.append('\n'); + } } // Only specify serverless if it's preview, using the preview boolean (GA is the default) if (preview) { - appliesToText.append("serverless: preview\n"); + appliesToText.append("serverless: preview"); + if (false == oneLine) { + appliesToText.append('\n'); + } } - appliesToText.append("```\n"); + appliesToText.append(oneLine ? "`" : "```\n"); } return appliesToText.toString(); } @@ -711,7 +719,7 @@ private void renderFullLayout(FunctionInfo info, boolean hasExamples, boolean ha .replace("$NAME$", name) .replace("$CATEGORY$", category) .replace("$UPPER_NAME$", name.toUpperCase(Locale.ROOT)) - .replace("$APPLIES_TO$", makeAppliesToText(info.appliesTo(), info.preview())) + .replace("$APPLIES_TO$", makeAppliesToText(Arrays.asList(info.appliesTo()), info.preview(), false)) ); for (String section : new String[] { "parameters", "description", "types" }) { rendered.append(addInclude(section)); @@ -755,7 +763,7 @@ public OperatorsDocsSupport( String name, Class testClass, OperatorConfig op, - Supplier, DataType>> signatures, + Supplier, DataType>> signatures, Callbacks callbacks ) { super("operators", name, testClass, signatures, callbacks); @@ -888,7 +896,9 @@ void renderDocsForOperators( if (mapParamInfo != null) { args.add(mapParam(mapParamInfo)); } else { - Param paramInfo = params[i].getAnnotation(Param.class); + org.elasticsearch.xpack.esql.expression.function.Param paramInfo = params[i].getAnnotation( + org.elasticsearch.xpack.esql.expression.function.Param.class + ); args.add(paramInfo != null ? param(paramInfo, false) : paramWithoutAnnotation(params[i].getName())); } } @@ -956,7 +966,7 @@ public CommandsDocsSupport( Class testClass, LogicalPlan command, List args, - Supplier, DataType>> signatures, + Supplier, DataType>> signatures, Callbacks callbacks ) { super("commands", name, testClass, signatures, callbacks); @@ -1016,12 +1026,12 @@ void renderTypes(String name, List args) thro } Map> compactedTable = new TreeMap<>(); - for (Map.Entry, DataType> sig : this.signatures.get().entrySet()) { + for (Map.Entry, DataType> sig : this.signatures.get().entrySet()) { if (shouldHideSignature(sig.getKey(), sig.getValue())) { continue; } - String mainType = sig.getKey().getFirst().esNameIfPossible(); - String secondaryType = sig.getKey().get(1).esNameIfPossible(); + String mainType = sig.getKey().getFirst().dataType().esNameIfPossible(); + String secondaryType = sig.getKey().get(1).dataType().esNameIfPossible(); List secondaryTypes = compactedTable.computeIfAbsent(mainType, (k) -> new ArrayList<>()); secondaryTypes.add(secondaryType); } @@ -1079,7 +1089,7 @@ void renderTypes(String name, List args) thro } List table = new ArrayList<>(); - for (Map.Entry, DataType> sig : this.signatures.get().entrySet()) { // TODO flip to using sortedSignatures + for (Map.Entry, DataType> sig : this.signatures.get().entrySet()) { // TODO flip to using sortedSignatures if (shouldHideSignature(sig.getKey(), sig.getValue())) { continue; } @@ -1104,18 +1114,21 @@ void renderTypes(String name, List args) thro private static String getTypeRow( List args, - Map.Entry, DataType> sig, + Map.Entry, DataType> sig, List argNames, boolean showResultColumn ) { StringBuilder b = new StringBuilder("| "); for (int i = 0; i < sig.getKey().size(); i++) { - DataType argType = sig.getKey().get(i); + Param param = sig.getKey().get(i); EsqlFunctionRegistry.ArgSignature argSignature = args.get(i); if (argSignature.mapArg()) { b.append("named parameters"); } else { - b.append(argType.esNameIfPossible()); + b.append(param.dataType().esNameIfPossible()); + if (param.appliesTo() != null) { + b.append(FunctionDocsSupport.makeAppliesToText(param.appliesTo(), false, true)); + } } b.append(" | "); } @@ -1274,7 +1287,7 @@ void renderKibanaFunctionDefinition( builder.endObject(); } else { int minArgCount = (int) args.stream().filter(a -> false == a.optional()).count(); - for (Map.Entry, DataType> sig : sortedSignatures()) { + for (Map.Entry, DataType> sig : sortedSignatures()) { if (variadic && sig.getKey().size() > args.size()) { // For variadic functions we test much longer signatures, let’s just stop at the last one continue; @@ -1302,7 +1315,7 @@ void renderKibanaFunctionDefinition( .collect(Collectors.joining(", ")) ); } else { - builder.field("type", sig.getKey().get(i).esNameIfPossible()); + builder.field("type", sig.getKey().get(i).dataType().esNameIfPossible()); } builder.field("optional", arg.optional()); String cleanedParamDesc = removeAppliesToBlocks(arg.description()); @@ -1310,7 +1323,7 @@ void renderKibanaFunctionDefinition( builder.endObject(); } builder.endArray(); - license = licenseChecker.invoke(sig.getKey()); + license = licenseChecker.invoke(sig.getKey().stream().map(Param::dataType).toList()); if (license != null && license != License.OperationMode.BASIC) { builder.field("license", license.toString()); } @@ -1358,8 +1371,8 @@ private static String removeAppliesToBlocks(String content) { return content.replaceAll("\\s*\\{applies_to\\}`[^`]*`\\s*", ""); } - private List, DataType>> sortedSignatures() { - List, DataType>> sortedSignatures = new ArrayList<>(signatures.get().entrySet()); + private List, DataType>> sortedSignatures() { + List, DataType>> sortedSignatures = new ArrayList<>(signatures.get().entrySet()); sortedSignatures.sort((lhs, rhs) -> { int maxlen = Math.max(lhs.getKey().size(), rhs.getKey().size()); for (int i = 0; i < maxlen; i++) { @@ -1369,7 +1382,7 @@ private List, DataType>> sortedSignatures() { if (rhs.getKey().size() <= i) { return 1; } - int c = lhs.getKey().get(i).esNameIfPossible().compareTo(rhs.getKey().get(i).esNameIfPossible()); + int c = lhs.getKey().get(i).dataType().esNameIfPossible().compareTo(rhs.getKey().get(i).dataType().esNameIfPossible()); if (c != 0) { return c; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3SupportTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3SupportTests.java index bcb2a52336f78..4ad83f3495294 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3SupportTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3SupportTests.java @@ -422,8 +422,8 @@ public TestClass(Source source, @Param(name = "str", type = { "keyword", "text" super(source, List.of(field)); } - public static Map, DataType> signatures() { - return Map.of(List.of(DataType.KEYWORD), DataType.LONG); + public static Map, DataType> signatures() { + return Map.of(List.of(new DocsV3Support.Param(DataType.KEYWORD, List.of())), DataType.LONG); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java index 3ffe2619526bb..1e86516133f76 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java @@ -506,12 +506,19 @@ private static void addSuppliers( Supplier valueSupplier ) { if (minRows <= 1 && maxRows >= 1) { - cases.add(new TypedDataSupplier("", () -> randomList(1, 1, valueSupplier), type, false, true)); + cases.add(new TypedDataSupplier("", () -> randomList(1, 1, valueSupplier), type, false, true, List.of())); } if (maxRows > 1) { cases.add( - new TypedDataSupplier("<" + name + "s>", () -> randomList(Math.max(2, minRows), maxRows, valueSupplier), type, false, true) + new TypedDataSupplier( + "<" + name + "s>", + () -> randomList(Math.max(2, minRows), maxRows, valueSupplier), + type, + false, + true, + List.of() + ) ); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java index 242c2bb6c0182..7a1600db82c58 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java @@ -9,6 +9,7 @@ import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.time.DateUtils; @@ -29,6 +30,7 @@ import org.elasticsearch.xpack.versionfield.Version; import org.hamcrest.Matcher; +import java.lang.annotation.Annotation; import java.math.BigInteger; import java.time.Duration; import java.time.Instant; @@ -1755,18 +1757,31 @@ public Matcher evaluatorToString() { * exists because we can't generate random values from the test parameter generation functions, and instead need to return * suppliers which generate the random values at test execution time. */ - public record TypedDataSupplier(String name, Supplier supplier, DataType type, boolean forceLiteral, boolean multiRow) { - + public record TypedDataSupplier( + String name, + Supplier supplier, + DataType type, + boolean forceLiteral, + boolean multiRow, + List appliesTo + ) { public TypedDataSupplier(String name, Supplier supplier, DataType type, boolean forceLiteral) { - this(name, supplier, type, forceLiteral, false); + this(name, supplier, type, forceLiteral, false, List.of()); } public TypedDataSupplier(String name, Supplier supplier, DataType type) { - this(name, supplier, type, false, false); + this(name, supplier, type, false, false, List.of()); + } + + /** + * Marks the version of Elasticsearch in which this signature was first supported. + */ + public TypedDataSupplier withAppliesTo(FunctionAppliesTo appliesTo) { + return new TypedDataSupplier(name, supplier, type, forceLiteral, multiRow, appendAppliesTo(this.appliesTo, appliesTo)); } public TypedData get() { - return new TypedData(supplier.get(), type, name, forceLiteral, multiRow); + return new TypedData(supplier.get(), type, name, forceLiteral, multiRow, appliesTo); } } @@ -1783,6 +1798,7 @@ public static class TypedData { private final boolean forceLiteral; private final boolean multiRow; private final boolean mapExpression; + private final List appliesTo; /** * @param data value to test against @@ -1791,7 +1807,14 @@ public static class TypedData { * @param forceLiteral should this data always be converted to a literal and never to a field reference? * @param multiRow if true, data is expected to be a List of values, one per row */ - private TypedData(Object data, DataType type, String name, boolean forceLiteral, boolean multiRow) { + private TypedData( + Object data, + DataType type, + String name, + boolean forceLiteral, + boolean multiRow, + List appliesTo + ) { assert multiRow == false || data instanceof List : "multiRow data must be a List"; assert multiRow == false || forceLiteral == false : "multiRow data can't be converted to a literal"; @@ -1805,6 +1828,7 @@ private TypedData(Object data, DataType type, String name, boolean forceLiteral, this.forceLiteral = forceLiteral; this.multiRow = multiRow; this.mapExpression = data instanceof MapExpression; + this.appliesTo = appliesTo; } /** @@ -1813,7 +1837,7 @@ private TypedData(Object data, DataType type, String name, boolean forceLiteral, * @param name a name for the value, used for generating test case names */ public TypedData(Object data, DataType type, String name) { - this(data, type, name, false, false); + this(data, type, name, false, false, List.of()); } /** @@ -1834,7 +1858,7 @@ public TypedData(Object data, String name) { * @param name a name for the value, used for generating test case names */ public static TypedData multiRow(List data, DataType type, String name) { - return new TypedData(data, type, name, false, true); + return new TypedData(data, type, name, false, true, List.of()); } /** @@ -1843,7 +1867,7 @@ public static TypedData multiRow(List data, DataType type, String name) { * must be constants. */ public TypedData forceLiteral() { - return new TypedData(data, type, name, true, multiRow); + return new TypedData(data, type, name, true, multiRow, appliesTo); } /** @@ -1860,13 +1884,24 @@ public boolean isMultiRow() { return multiRow; } + public List appliesTo() { + return appliesTo; + } + /** * Return a {@link TypedData} with the new data. * * @param data The new data for the {@link TypedData}. */ public TypedData withData(Object data) { - return new TypedData(data, type, name, forceLiteral, multiRow); + return new TypedData(data, type, name, forceLiteral, multiRow, appliesTo); + } + + /** + * Marks the version of Elasticsearch in which this signature was first supported. + */ + public TypedData withAppliesTo(FunctionAppliesTo appliesTo) { + return new TypedData(data, type, name, forceLiteral, multiRow, appendAppliesTo(this.appliesTo, appliesTo)); } @Override @@ -1977,4 +2012,29 @@ public String name() { return name; } } + + /** + * Builds a version of Elasticsearch for use with {@link TypedDataSupplier#withAppliesTo(FunctionAppliesTo)}. + */ + public static FunctionAppliesTo appliesTo( + FunctionAppliesToLifecycle lifeCycle, + String version, + String description, + boolean serverless + ) { + return new AppliesTo(lifeCycle, version, description, serverless); + } + + private record AppliesTo(FunctionAppliesToLifecycle lifeCycle, String version, String description, boolean serverless) + implements + FunctionAppliesTo { + @Override + public Class annotationType() { + return FunctionAppliesTo.class; + } + } + + static List appendAppliesTo(List current, FunctionAppliesTo next) { + return Iterators.toList(Iterators.concat(current.iterator(), Iterators.single(next))); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java index b3d21a8aab8fc..33319f158daea 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java @@ -80,7 +80,14 @@ public static Iterable parameters() { DataType.KEYWORD, DataType.TEXT )) { - var emptyFieldSupplier = new TestCaseSupplier.TypedDataSupplier("No rows (" + dataType + ")", List::of, dataType, false, true); + var emptyFieldSupplier = new TestCaseSupplier.TypedDataSupplier( + "No rows (" + dataType + ")", + List::of, + dataType, + false, + true, + List.of() + ); // With precision for (var precisionCaseSupplier : precisionSuppliers) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstOverTimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstOverTimeTests.java index dc64f35a0fe79..065f3fd35ebb1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstOverTimeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstOverTimeTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase; +import org.elasticsearch.xpack.esql.expression.function.DocsV3Support; import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; @@ -91,9 +92,9 @@ private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier }); } - public static List signatureTypes(List testCaseTypes) { - assertThat(testCaseTypes, hasSize(2)); - assertThat(testCaseTypes.get(1), equalTo(DataType.DATETIME)); - return List.of(testCaseTypes.get(0)); + public static List signatureTypes(List params) { + assertThat(params, hasSize(2)); + assertThat(params.get(1).dataType(), equalTo(DataType.DATETIME)); + return List.of(params.get(0)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java index cf9e8d414590d..0b2d2c8b53df6 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase; +import org.elasticsearch.xpack.esql.expression.function.DocsV3Support; import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; @@ -91,9 +92,9 @@ private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier }); } - public static List signatureTypes(List testCaseTypes) { - assertThat(testCaseTypes, hasSize(2)); - assertThat(testCaseTypes.get(1), equalTo(DataType.DATETIME)); - return List.of(testCaseTypes.get(0)); + public static List signatureTypes(List params) { + assertThat(params, hasSize(2)); + assertThat(params.get(1).dataType(), equalTo(DataType.DATETIME)); + return List.of(params.get(0)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java index 6bc93fbea598f..bc181df790d6e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java @@ -17,6 +17,8 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase; +import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesTo; +import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesToLifecycle; import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.versionfield.Version; @@ -29,6 +31,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier.appliesTo; import static org.hamcrest.Matchers.equalTo; public class MaxTests extends AbstractAggregationTestCase { @@ -43,7 +46,6 @@ public static Iterable parameters() { Stream.of( MultiRowTestCaseSupplier.intCases(1, 1000, Integer.MIN_VALUE, Integer.MAX_VALUE, true), MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true), - MultiRowTestCaseSupplier.ulongCases(1, 1000, BigInteger.ZERO, UNSIGNED_LONG_MAX, true), MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true), MultiRowTestCaseSupplier.dateCases(1, 1000), MultiRowTestCaseSupplier.booleanCases(1, 1000), @@ -53,6 +55,17 @@ public static Iterable parameters() { MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT) ).flatMap(List::stream).map(MaxTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers)); + FunctionAppliesTo unsignedLongAppliesTo = appliesTo(FunctionAppliesToLifecycle.GA, "9.2.0", "", true); + for (TestCaseSupplier.TypedDataSupplier supplier : MultiRowTestCaseSupplier.ulongCases( + 1, + 1000, + BigInteger.ZERO, + UNSIGNED_LONG_MAX, + true + )) { + suppliers.add(makeSupplier(supplier.withAppliesTo(unsignedLongAppliesTo))); + } + suppliers.addAll( List.of( // Folding @@ -77,7 +90,10 @@ public static Iterable parameters() { new TestCaseSupplier( List.of(DataType.UNSIGNED_LONG), () -> new TestCaseSupplier.TestCase( - List.of(TestCaseSupplier.TypedData.multiRow(List.of(new BigInteger("200")), DataType.UNSIGNED_LONG, "field")), + List.of( + TestCaseSupplier.TypedData.multiRow(List.of(new BigInteger("200")), DataType.UNSIGNED_LONG, "field") + .withAppliesTo(unsignedLongAppliesTo) + ), "Max[field=Attribute[channel=0]]", DataType.UNSIGNED_LONG, equalTo(new BigInteger("200")) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java index 391d130350a90..872a794c7be8e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java @@ -17,6 +17,8 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase; +import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesTo; +import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesToLifecycle; import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.versionfield.Version; @@ -29,6 +31,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier.appliesTo; import static org.hamcrest.Matchers.equalTo; public class MinTests extends AbstractAggregationTestCase { @@ -43,7 +46,6 @@ public static Iterable parameters() { Stream.of( MultiRowTestCaseSupplier.intCases(1, 1000, Integer.MIN_VALUE, Integer.MAX_VALUE, true), MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true), - MultiRowTestCaseSupplier.ulongCases(1, 1000, BigInteger.ZERO, UNSIGNED_LONG_MAX, true), MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true), MultiRowTestCaseSupplier.dateCases(1, 1000), MultiRowTestCaseSupplier.booleanCases(1, 1000), @@ -53,6 +55,17 @@ public static Iterable parameters() { MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT) ).flatMap(List::stream).map(MinTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers)); + FunctionAppliesTo unsignedLongAppliesTo = appliesTo(FunctionAppliesToLifecycle.GA, "9.2.0", "", true); + for (TestCaseSupplier.TypedDataSupplier supplier : MultiRowTestCaseSupplier.ulongCases( + 1, + 1000, + BigInteger.ZERO, + UNSIGNED_LONG_MAX, + true + )) { + suppliers.add(makeSupplier(supplier.withAppliesTo(unsignedLongAppliesTo))); + } + suppliers.addAll( List.of( // Folding @@ -77,7 +90,10 @@ public static Iterable parameters() { new TestCaseSupplier( List.of(DataType.UNSIGNED_LONG), () -> new TestCaseSupplier.TestCase( - List.of(TestCaseSupplier.TypedData.multiRow(List.of(new BigInteger("200")), DataType.UNSIGNED_LONG, "field")), + List.of( + TestCaseSupplier.TypedData.multiRow(List.of(new BigInteger("200")), DataType.UNSIGNED_LONG, "field") + .withAppliesTo(unsignedLongAppliesTo) + ), "Max[field=Attribute[channel=0]]", DataType.UNSIGNED_LONG, equalTo(new BigInteger("200")) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/RateTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/RateTests.java index 7da479ea28dba..e5a58bc29831e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/RateTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/RateTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase; +import org.elasticsearch.xpack.esql.expression.function.DocsV3Support; import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.hamcrest.Matcher; @@ -121,9 +122,9 @@ private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier }); } - public static List signatureTypes(List testCaseTypes) { - assertThat(testCaseTypes, hasSize(2)); - assertThat(testCaseTypes.get(1), equalTo(DataType.DATETIME)); - return List.of(testCaseTypes.get(0)); + public static List signatureTypes(List params) { + assertThat(params, hasSize(2)); + assertThat(params.get(1).dataType(), equalTo(DataType.DATETIME)); + return List.of(params.get(0)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/CastOperatorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/CastOperatorTests.java index 72c5a346b74f3..af53a1c666859 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/CastOperatorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/CastOperatorTests.java @@ -46,7 +46,7 @@ public static void renderDocs() throws Exception { docs.renderDocs(); } - public static Map, DataType> signatures() { + public static Map, DataType> signatures() { // The cast operator cannot produce sensible signatures unless we consider the type as an extra parameter return Map.of(); } From 050059b3f5c9ceb00ac2ec306d00453e17011f7a Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 15 Aug 2025 08:47:29 -0400 Subject: [PATCH 2/4] Javadoc --- .../xpack/esql/expression/function/DocsV3Support.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java index 182895be8bce6..254e3e9fbb5de 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java @@ -664,6 +664,17 @@ private void renderFunctionNamedParams(EsqlFunctionRegistry.MapArgSignature mapA writeToTempSnippetsDir("functionNamedParams", rendered.toString()); } + /** + * Build the {@code {applies_to}} annotation for the docs to tell users which version of + * Elasticsearch first supported this function/operator/signature. + * @param functionAppliesTos The version information for stateful Elasticsearch + * @param preview Is this tech preview? Effectively just generates the + * {@code serverless: preview} annotation if true and nothing if false. + * @param oneLine Should we generate a single line variant of the {@code {applies_to}} + * annotation compatible with tables (true) or the more readable + * multi-line variant (false)? + * @return Text of the {@code {applies_to}} annotation + */ private static String makeAppliesToText(List functionAppliesTos, boolean preview, boolean oneLine) { StringBuilder appliesToText = new StringBuilder(); if (false == functionAppliesTos.isEmpty()) { From 60149908aa79839a884c5de40896bb4c50f364c7 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 18 Aug 2025 13:25:18 -0400 Subject: [PATCH 3/4] Bogdan likes this It is nice --- .../xpack/esql/expression/function/DocsV3Support.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java index 254e3e9fbb5de..9d7b1720acfb5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java @@ -678,7 +678,13 @@ private void renderFunctionNamedParams(EsqlFunctionRegistry.MapArgSignature mapA private static String makeAppliesToText(List functionAppliesTos, boolean preview, boolean oneLine) { StringBuilder appliesToText = new StringBuilder(); if (false == functionAppliesTos.isEmpty()) { - appliesToText.append(oneLine ? "{applies_to}`" : "```{applies_to}\n"); + if (oneLine) { + appliesToText.append(" {applies_to}"); + appliesToText.append("`"); + } else { + appliesToText.append("```"); + appliesToText.append("{applies_to}\n"); + } StringBuilder stackEntries = new StringBuilder(); for (FunctionAppliesTo appliesTo : functionAppliesTos) { From 65e5efdff247ed0d0fcfa463d43efd9de43095a2 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 18 Aug 2025 14:02:35 -0400 Subject: [PATCH 4/4] Regen --- .../query-languages/esql/_snippets/functions/types/max.md | 2 +- .../esql/_snippets/functions/types/max_over_time.md | 2 +- .../query-languages/esql/_snippets/functions/types/min.md | 2 +- .../esql/_snippets/functions/types/min_over_time.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/max.md b/docs/reference/query-languages/esql/_snippets/functions/types/max.md index 8368c8fdd4919..3e79cc241a965 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/max.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/max.md @@ -13,6 +13,6 @@ | keyword | keyword | | long | long | | text | keyword | -| unsigned_long{applies_to}`stack: ga 9.2.0` | unsigned_long | +| unsigned_long {applies_to}`stack: ga 9.2.0` | unsigned_long | | version | version | diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/max_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/types/max_over_time.md index 8368c8fdd4919..3e79cc241a965 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/max_over_time.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/max_over_time.md @@ -13,6 +13,6 @@ | keyword | keyword | | long | long | | text | keyword | -| unsigned_long{applies_to}`stack: ga 9.2.0` | unsigned_long | +| unsigned_long {applies_to}`stack: ga 9.2.0` | unsigned_long | | version | version | diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/min.md b/docs/reference/query-languages/esql/_snippets/functions/types/min.md index 8368c8fdd4919..3e79cc241a965 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/min.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/min.md @@ -13,6 +13,6 @@ | keyword | keyword | | long | long | | text | keyword | -| unsigned_long{applies_to}`stack: ga 9.2.0` | unsigned_long | +| unsigned_long {applies_to}`stack: ga 9.2.0` | unsigned_long | | version | version | diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/min_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/types/min_over_time.md index 8368c8fdd4919..3e79cc241a965 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/min_over_time.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/min_over_time.md @@ -13,6 +13,6 @@ | keyword | keyword | | long | long | | text | keyword | -| unsigned_long{applies_to}`stack: ga 9.2.0` | unsigned_long | +| unsigned_long {applies_to}`stack: ga 9.2.0` | unsigned_long | | version | version |