From 854d292891275111aa0c46c122356d6b47a16b15 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Tue, 1 Jul 2025 22:33:05 +0200 Subject: [PATCH 1/8] Support types table in lookup join docs --- .../_snippets/commands/layout/lookup-join.md | 3 + .../_snippets/commands/types/lookup-join.md | 77 +++++++++++++++++++ .../query-languages/esql/esql-lookup-join.md | 10 ++- x-pack/plugin/esql/build.gradle | 39 ++++++++++ .../xpack/esql/action/LookupJoinTypesIT.java | 34 ++++++++ .../expression/function/DocsV3Support.java | 26 ++++++- 6 files changed, 186 insertions(+), 3 deletions(-) create mode 100644 docs/reference/query-languages/esql/_snippets/commands/types/lookup-join.md diff --git a/docs/reference/query-languages/esql/_snippets/commands/layout/lookup-join.md b/docs/reference/query-languages/esql/_snippets/commands/layout/lookup-join.md index da99cc69c031a..294bf00522eb0 100644 --- a/docs/reference/query-languages/esql/_snippets/commands/layout/lookup-join.md +++ b/docs/reference/query-languages/esql/_snippets/commands/layout/lookup-join.md @@ -81,3 +81,6 @@ before the lookup when possible. :::{include} ../examples/lookup-join.csv-spec/filterOnRightSide.md ::: + +:::{include} ../types/lookup-join.md +::: diff --git a/docs/reference/query-languages/esql/_snippets/commands/types/lookup-join.md b/docs/reference/query-languages/esql/_snippets/commands/types/lookup-join.md new file mode 100644 index 0000000000000..a5760f1225e28 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/commands/types/lookup-join.md @@ -0,0 +1,77 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| main | join | result | +| --- | --- | --- | +| boolean | boolean | boolean | +| byte | byte | byte | +| byte | double | double | +| byte | float | float | +| byte | half_float | half_float | +| byte | integer | integer | +| byte | long | long | +| byte | scaled_float | scaled_float | +| byte | short | short | +| date | date | date | +| date_nanos | date_nanos | date_nanos | +| double | byte | double | +| double | double | double | +| double | float | double | +| double | half_float | double | +| double | integer | double | +| double | long | double | +| double | scaled_float | scaled_float | +| double | short | double | +| float | byte | float | +| float | double | double | +| float | float | float | +| float | half_float | half_float | +| float | integer | float | +| float | long | float | +| float | scaled_float | scaled_float | +| float | short | float | +| half_float | byte | half_float | +| half_float | double | double | +| half_float | float | float | +| half_float | half_float | half_float | +| half_float | integer | half_float | +| half_float | long | half_float | +| half_float | scaled_float | scaled_float | +| half_float | short | half_float | +| integer | byte | integer | +| integer | double | double | +| integer | float | float | +| integer | half_float | half_float | +| integer | integer | integer | +| integer | long | long | +| integer | scaled_float | scaled_float | +| integer | short | integer | +| ip | ip | ip | +| keyword | keyword | keyword | +| long | byte | long | +| long | double | double | +| long | float | float | +| long | half_float | half_float | +| long | integer | long | +| long | long | long | +| long | scaled_float | scaled_float | +| long | short | long | +| scaled_float | byte | scaled_float | +| scaled_float | double | double | +| scaled_float | float | scaled_float | +| scaled_float | half_float | scaled_float | +| scaled_float | integer | scaled_float | +| scaled_float | long | scaled_float | +| scaled_float | scaled_float | scaled_float | +| scaled_float | short | scaled_float | +| short | byte | short | +| short | double | double | +| short | float | float | +| short | half_float | half_float | +| short | integer | integer | +| short | long | long | +| short | scaled_float | scaled_float | +| short | short | short | +| text | keyword | keyword | + diff --git a/docs/reference/query-languages/esql/esql-lookup-join.md b/docs/reference/query-languages/esql/esql-lookup-join.md index d57437833c1b2..5684a2234322e 100644 --- a/docs/reference/query-languages/esql/esql-lookup-join.md +++ b/docs/reference/query-languages/esql/esql-lookup-join.md @@ -153,8 +153,16 @@ To use `LOOKUP JOIN`, the following requirements must be met: * For text fields: You can only use text fields as the join key on the left-hand side of the join and only if they have a `.keyword` subfield To obtain a join key with a compatible type, use a [conversion function](/reference/query-languages/esql/functions-operators/type-conversion-functions.md) if needed. +* Both `KEYWORD` and `TEXT` are supported in the main index, but can only join against `KEYWORD` in the join index. +* All numerical types are supported as join keys, using the same rules as applicable to the `==` operator when performing the join. FOr example, a `double` can be joined with an `integer`. +* `DATE` and `DATE_NONES` can only be joined against the exact same type. +* `IP` is supported, but `VERSION` is not. -For a complete list of supported data types and their internal representations, see the [Supported Field Types documentation](/reference/query-languages/esql/limitations.md#_supported_types). +The list of unsupported fields includes all types not supported by {{esql}} as described in the [Unsupported Field Types documentation](/reference/query-languages/esql/limitations.md#_unsupported_types). +as well as the following: `VERSION`, `UNSIGNED_LONG`, all spatial types like `GEO_POINT`, `GEO_SHAPE`, and all +temporal periods like `DURATION` and `PERIOD`. + +For a complete list of all types supported in `LOOKUP JOIN`, refer to the [`LOOKUP JOIN` supported types table](/reference/query-languages/esql/commands/processing-commands#esql-lookup-join). ## Usage notes diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle index 0db0e558cd147..3c83be5fe1856 100644 --- a/x-pack/plugin/esql/build.gradle +++ b/x-pack/plugin/esql/build.gradle @@ -226,6 +226,45 @@ tasks.named("test").configure { } } +tasks.named("internalClusterTest").configure { + if (buildParams.ci == false) { + systemProperty 'generateDocs', true + def injected = project.objects.newInstance(Injected) + // Define the folder to delete and recreate + def tempDir = file("build/testrun/internalClusterTest/temp/esql") + doFirst { + injected.fs.delete { + it.delete(tempDir) + } + // Re-create this folder so we can save a table of generated examples to extract from csv-spec tests + tempDir.mkdirs() // Recreate the folder + } + File snippetsFolder = file("build/testrun/internalClusterTest/temp/esql/_snippets") + def snippetsDocFolder = file("${rootDir}/docs/reference/query-languages/esql/_snippets") + def snippetsTree = fileTree(snippetsFolder).matching { + include "**/types/*.md" // Recursively include all types/*.md files (effectively counting functions and operators) + } + + doLast { + def snippets = snippetsTree.files.collect { it.name } + int countSnippets = snippets.size() + if (countSnippets == 0) { + logger.quiet("ESQL Docs: No function/operator snippets created. Skipping sync.") + } else { + logger.quiet("ESQL Docs: Found $countSnippets generated function/operator snippets to patch into docs") + injected.fs.sync { + from snippetsFolder + into snippetsDocFolder + include '**/*.md' + preserve { + include '**/*.md' + } + } + } + } + } +} + /**************************************************************** * Enable QA/rest integration tests for snapshot builds only * * TODO: Enable for all builds upon this feature release * diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java index df021f27a31fe..238b12ad4ec8a 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java @@ -19,8 +19,11 @@ import org.elasticsearch.xpack.core.esql.action.ColumnInfo; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.DocsV3Support; +import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.plan.logical.join.Join; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; +import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter; import org.elasticsearch.xpack.spatial.SpatialPlugin; import org.elasticsearch.xpack.unsignedlong.UnsignedLongMapperPlugin; import org.elasticsearch.xpack.versionfield.VersionFieldPlugin; @@ -36,6 +39,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Consumer; +import java.util.function.Supplier; import java.util.stream.Collectors; import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; @@ -265,6 +269,22 @@ private static boolean existingIndex(Collection existing, DataType return existing.stream().anyMatch(c -> c.exists(indexName)); } + public void testOutputSupportedTypes() throws Exception { + Map, DataType> signatures = new LinkedHashMap<>(); + for (TestConfigs configs : testConfigurations.values()) { + if (configs.group.equals("unsupported") || configs.group.equals("union-types")) { + continue; + } + for (TestConfig config : configs.configs.values()) { + if (config instanceof TestConfigPasses) { + DataType commonType = EsqlDataTypeConverter.commonType(config.mainType(), config.lookupType()); + signatures.put(List.of(config.mainType(), config.lookupType()), commonType); + } + } + } + saveJoinTypes(() -> signatures); + } + public void testLookupJoinStrings() { testLookupJoinTypes("strings"); } @@ -747,4 +767,18 @@ public void doTest() { private boolean isValidDataType(DataType dataType) { return UNDER_CONSTRUCTION.get(dataType) == null || UNDER_CONSTRUCTION.get(dataType).isEnabled(); } + + private static void saveJoinTypes(Supplier, DataType>> signatures) throws Exception { + ArrayList args = new ArrayList<>(); + args.add(new EsqlFunctionRegistry.ArgSignature("main", null, "Field from the main index", false, false)); + args.add(new EsqlFunctionRegistry.ArgSignature("join", null, "Field from the join index", false, false)); + DocsV3Support.CommandsDocsSupport docs = new DocsV3Support.CommandsDocsSupport( + "lookup-join", + LookupJoinTypesIT.class, + null, + args, + signatures + ); + docs.renderDocs(); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java index 585f507e1e039..2eb9b3e3e067f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java @@ -836,14 +836,30 @@ void renderDetailedDescription(String detailedDescription, String note) throws I /** Command specific docs generating, currently very empty since we only render kibana definition files */ public static class CommandsDocsSupport extends DocsV3Support { private final LogicalPlan command; + private List args; private final XPackLicenseState licenseState; + /** Used in CommandLicenseTests to generate Kibana docs with licensing information for commands */ public CommandsDocsSupport(String name, Class testClass, LogicalPlan command, XPackLicenseState licenseState) { super("commands", name, testClass, Map::of); this.command = command; this.licenseState = licenseState; } + /** Used in LookupJoinTypesIT to generate table of supported types for join field */ + public CommandsDocsSupport( + String name, + Class testClass, + LogicalPlan command, + List args, + Supplier, DataType>> signatures + ) { + super("commands", name, testClass, signatures); + this.command = command; + this.args = args; + this.licenseState = null; + } + @Override public void renderSignature() throws IOException { // Unimplemented until we make command docs dynamically generated @@ -851,8 +867,14 @@ public void renderSignature() throws IOException { @Override public void renderDocs() throws Exception { - // Currently we only render kibana definition files, but we could expand to rendering much more if we decide to - renderKibanaCommandDefinition(); + // Currently we only render either signatures or kibana definition files, + // but we could expand to rendering much more if we decide to + if (args != null) { + renderTypes(name, args); + } + if (licenseState != null) { + renderKibanaCommandDefinition(); + } } void renderKibanaCommandDefinition() throws Exception { From bad33c1840a97f80284af9cf7aae326b10b1b678 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Tue, 1 Jul 2025 22:52:00 +0200 Subject: [PATCH 2/8] Don't show a results column in the join types --- .../_snippets/commands/layout/lookup-join.md | 6 +- .../_snippets/commands/types/lookup-join.md | 144 +++++++++--------- .../query-languages/esql/esql-lookup-join.md | 5 +- .../xpack/esql/action/LookupJoinTypesIT.java | 4 +- .../expression/function/DocsV3Support.java | 19 ++- 5 files changed, 90 insertions(+), 88 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/commands/layout/lookup-join.md b/docs/reference/query-languages/esql/_snippets/commands/layout/lookup-join.md index 294bf00522eb0..7b4f2d794ac22 100644 --- a/docs/reference/query-languages/esql/_snippets/commands/layout/lookup-join.md +++ b/docs/reference/query-languages/esql/_snippets/commands/layout/lookup-join.md @@ -42,6 +42,9 @@ results, the output will contain one row for each matching combination. For important information about using `LOOKUP JOIN`, refer to [Usage notes](../../../../esql/esql-lookup-join.md#usage-notes). :::: +:::{include} ../types/lookup-join.md +::: + **Examples** **IP Threat correlation**: This query would allow you to see if any source @@ -81,6 +84,3 @@ before the lookup when possible. :::{include} ../examples/lookup-join.csv-spec/filterOnRightSide.md ::: - -:::{include} ../types/lookup-join.md -::: diff --git a/docs/reference/query-languages/esql/_snippets/commands/types/lookup-join.md b/docs/reference/query-languages/esql/_snippets/commands/types/lookup-join.md index a5760f1225e28..04f62e3f59d47 100644 --- a/docs/reference/query-languages/esql/_snippets/commands/types/lookup-join.md +++ b/docs/reference/query-languages/esql/_snippets/commands/types/lookup-join.md @@ -2,76 +2,76 @@ **Supported types** -| main | join | result | -| --- | --- | --- | -| boolean | boolean | boolean | -| byte | byte | byte | -| byte | double | double | -| byte | float | float | -| byte | half_float | half_float | -| byte | integer | integer | -| byte | long | long | -| byte | scaled_float | scaled_float | -| byte | short | short | -| date | date | date | -| date_nanos | date_nanos | date_nanos | -| double | byte | double | -| double | double | double | -| double | float | double | -| double | half_float | double | -| double | integer | double | -| double | long | double | -| double | scaled_float | scaled_float | -| double | short | double | -| float | byte | float | -| float | double | double | -| float | float | float | -| float | half_float | half_float | -| float | integer | float | -| float | long | float | -| float | scaled_float | scaled_float | -| float | short | float | -| half_float | byte | half_float | -| half_float | double | double | -| half_float | float | float | -| half_float | half_float | half_float | -| half_float | integer | half_float | -| half_float | long | half_float | -| half_float | scaled_float | scaled_float | -| half_float | short | half_float | -| integer | byte | integer | -| integer | double | double | -| integer | float | float | -| integer | half_float | half_float | -| integer | integer | integer | -| integer | long | long | -| integer | scaled_float | scaled_float | -| integer | short | integer | -| ip | ip | ip | -| keyword | keyword | keyword | -| long | byte | long | -| long | double | double | -| long | float | float | -| long | half_float | half_float | -| long | integer | long | -| long | long | long | -| long | scaled_float | scaled_float | -| long | short | long | -| scaled_float | byte | scaled_float | -| scaled_float | double | double | -| scaled_float | float | scaled_float | -| scaled_float | half_float | scaled_float | -| scaled_float | integer | scaled_float | -| scaled_float | long | scaled_float | -| scaled_float | scaled_float | scaled_float | -| scaled_float | short | scaled_float | -| short | byte | short | -| short | double | double | -| short | float | float | -| short | half_float | half_float | -| short | integer | integer | -| short | long | long | -| short | scaled_float | scaled_float | -| short | short | short | -| text | keyword | keyword | +| main | join | +| --- | --- | +| boolean | boolean | +| byte | byte | +| byte | double | +| byte | float | +| byte | half_float | +| byte | integer | +| byte | long | +| byte | scaled_float | +| byte | short | +| date | date | +| date_nanos | date_nanos | +| double | byte | +| double | double | +| double | float | +| double | half_float | +| double | integer | +| double | long | +| double | scaled_float | +| double | short | +| float | byte | +| float | double | +| float | float | +| float | half_float | +| float | integer | +| float | long | +| float | scaled_float | +| float | short | +| half_float | byte | +| half_float | double | +| half_float | float | +| half_float | half_float | +| half_float | integer | +| half_float | long | +| half_float | scaled_float | +| half_float | short | +| integer | byte | +| integer | double | +| integer | float | +| integer | half_float | +| integer | integer | +| integer | long | +| integer | scaled_float | +| integer | short | +| ip | ip | +| keyword | keyword | +| long | byte | +| long | double | +| long | float | +| long | half_float | +| long | integer | +| long | long | +| long | scaled_float | +| long | short | +| scaled_float | byte | +| scaled_float | double | +| scaled_float | float | +| scaled_float | half_float | +| scaled_float | integer | +| scaled_float | long | +| scaled_float | scaled_float | +| scaled_float | short | +| short | byte | +| short | double | +| short | float | +| short | half_float | +| short | integer | +| short | long | +| short | scaled_float | +| short | short | +| text | keyword | diff --git a/docs/reference/query-languages/esql/esql-lookup-join.md b/docs/reference/query-languages/esql/esql-lookup-join.md index 5684a2234322e..1247723ca34de 100644 --- a/docs/reference/query-languages/esql/esql-lookup-join.md +++ b/docs/reference/query-languages/esql/esql-lookup-join.md @@ -151,12 +151,9 @@ To use `LOOKUP JOIN`, the following requirements must be met: * `short` and `byte` are compatible with `integer` (all represented as `int`) * `float`, `half_float`, and `scaled_float` are compatible with `double` (all represented as `double`) * For text fields: You can only use text fields as the join key on the left-hand side of the join and only if they have a `.keyword` subfield + * `DATE` and `DATE_NANOS` can only be joined against the exact same type. To obtain a join key with a compatible type, use a [conversion function](/reference/query-languages/esql/functions-operators/type-conversion-functions.md) if needed. -* Both `KEYWORD` and `TEXT` are supported in the main index, but can only join against `KEYWORD` in the join index. -* All numerical types are supported as join keys, using the same rules as applicable to the `==` operator when performing the join. FOr example, a `double` can be joined with an `integer`. -* `DATE` and `DATE_NONES` can only be joined against the exact same type. -* `IP` is supported, but `VERSION` is not. The list of unsupported fields includes all types not supported by {{esql}} as described in the [Unsupported Field Types documentation](/reference/query-languages/esql/limitations.md#_unsupported_types). as well as the following: `VERSION`, `UNSIGNED_LONG`, all spatial types like `GEO_POINT`, `GEO_SHAPE`, and all diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java index 238b12ad4ec8a..44b465270579d 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java @@ -23,7 +23,6 @@ import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.plan.logical.join.Join; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; -import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter; import org.elasticsearch.xpack.spatial.SpatialPlugin; import org.elasticsearch.xpack.unsignedlong.UnsignedLongMapperPlugin; import org.elasticsearch.xpack.versionfield.VersionFieldPlugin; @@ -277,8 +276,7 @@ public void testOutputSupportedTypes() throws Exception { } for (TestConfig config : configs.configs.values()) { if (config instanceof TestConfigPasses) { - DataType commonType = EsqlDataTypeConverter.commonType(config.mainType(), config.lookupType()); - signatures.put(List.of(config.mainType(), config.lookupType()), commonType); + signatures.put(List.of(config.mainType(), config.lookupType()), null); } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java index 2eb9b3e3e067f..4ba8e19327d89 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java @@ -60,6 +60,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.function.Function; import java.util.function.Supplier; @@ -915,6 +916,7 @@ void renderParametersList(List argNames, List argDescriptions) t } void renderTypes(String name, List args) throws IOException { + boolean showResultColumn = signatures.get().values().stream().anyMatch(Objects::nonNull); StringBuilder header = new StringBuilder("| "); StringBuilder separator = new StringBuilder("| "); List argNames = args.stream().map(EsqlFunctionRegistry.ArgSignature::name).toList(); @@ -922,8 +924,10 @@ void renderTypes(String name, List args) thro header.append(arg).append(" | "); separator.append("---").append(" | "); } - header.append("result |"); - separator.append("--- |"); + if (showResultColumn) { + header.append("result |"); + separator.append("--- |"); + } List table = new ArrayList<>(); for (Map.Entry, DataType> sig : this.signatures.get().entrySet()) { // TODO flip to using sortedSignatures @@ -933,7 +937,7 @@ void renderTypes(String name, List args) thro if (sig.getKey().size() > argNames.size()) { // skip variadic [test] cases (but not those with optional parameters) continue; } - table.add(getTypeRow(args, sig, argNames)); + table.add(getTypeRow(args, sig, argNames, showResultColumn)); } Collections.sort(table); if (table.isEmpty()) { @@ -952,7 +956,8 @@ void renderTypes(String name, List args) thro private static String getTypeRow( List args, Map.Entry, DataType> sig, - List argNames + List argNames, + boolean showResultColumn ) { StringBuilder b = new StringBuilder("| "); for (int i = 0; i < sig.getKey().size(); i++) { @@ -966,8 +971,10 @@ private static String getTypeRow( b.append(" | "); } b.append("| ".repeat(argNames.size() - sig.getKey().size())); - b.append(sig.getValue().esNameIfPossible()); - b.append(" |"); + if (showResultColumn) { + b.append(sig.getValue().esNameIfPossible()); + b.append(" |"); + } return b.toString(); } From edf5047039a9b6af2cd7e04018c0b3f70696b499 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 2 Jul 2025 10:11:12 +0200 Subject: [PATCH 3/8] Newline changes after running tests --- .../esql/kibana/definition/functions/st_geohash.json | 2 +- .../esql/kibana/definition/functions/st_geohex.json | 2 +- .../esql/kibana/definition/functions/st_geohex_to_long.json | 2 +- .../esql/kibana/definition/functions/st_geohex_to_string.json | 2 +- .../esql/kibana/definition/functions/st_geotile.json | 2 +- .../esql/kibana/definition/functions/st_geotile_to_long.json | 2 +- .../esql/kibana/definition/functions/st_geotile_to_string.json | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/st_geohash.json b/docs/reference/query-languages/esql/kibana/definition/functions/st_geohash.json index d2fc83008c150..43633b336453a 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/st_geohash.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/st_geohash.json @@ -52,4 +52,4 @@ ], "preview" : true, "snapshot_only" : true -} \ No newline at end of file +} diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex.json b/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex.json index 9a3a04cb0a7f8..f29db14ed50e7 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex.json @@ -55,4 +55,4 @@ ], "preview" : true, "snapshot_only" : true -} \ No newline at end of file +} diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex_to_long.json b/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex_to_long.json index 52c7918a0c3ad..d582739620024 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex_to_long.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex_to_long.json @@ -34,4 +34,4 @@ ], "preview" : true, "snapshot_only" : true -} \ No newline at end of file +} diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex_to_string.json b/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex_to_string.json index 612b13691d40c..a1abce7c75adb 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex_to_string.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/st_geohex_to_string.json @@ -34,4 +34,4 @@ ], "preview" : true, "snapshot_only" : true -} \ No newline at end of file +} diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile.json b/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile.json index 06df5e3076fea..d728f186fc5ae 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile.json @@ -52,4 +52,4 @@ ], "preview" : true, "snapshot_only" : true -} \ No newline at end of file +} diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile_to_long.json b/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile_to_long.json index 2eb49b5c320f9..b2c7c01aea606 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile_to_long.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile_to_long.json @@ -34,4 +34,4 @@ ], "preview" : true, "snapshot_only" : true -} \ No newline at end of file +} diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile_to_string.json b/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile_to_string.json index df8e91514dc7b..5a327c2c50976 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile_to_string.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/st_geotile_to_string.json @@ -34,4 +34,4 @@ ], "preview" : true, "snapshot_only" : true -} \ No newline at end of file +} From 8e470d76a17697bf4dbdce8b6ea5595cea88d8ee Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 2 Jul 2025 10:16:20 +0200 Subject: [PATCH 4/8] Fix compile error after merging main --- .../xpack/esql/expression/function/DocsV3Support.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java index 8e2c4c19bb942..77a78241fb863 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java @@ -890,6 +890,7 @@ public CommandsDocsSupport( this.command = command; this.args = args; this.licenseState = null; + this.observabilityTier = null; } @Override From f9282758f1e1df538e4cf324b5c1e2d7e2fe1785 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 2 Jul 2025 11:32:16 +0200 Subject: [PATCH 5/8] Update docs/reference/query-languages/esql/esql-lookup-join.md Co-authored-by: Liam Thompson <32779855+leemthompo@users.noreply.github.com> --- docs/reference/query-languages/esql/esql-lookup-join.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/query-languages/esql/esql-lookup-join.md b/docs/reference/query-languages/esql/esql-lookup-join.md index 1247723ca34de..d954be2ed1cb0 100644 --- a/docs/reference/query-languages/esql/esql-lookup-join.md +++ b/docs/reference/query-languages/esql/esql-lookup-join.md @@ -159,7 +159,7 @@ The list of unsupported fields includes all types not supported by {{esql}} as d as well as the following: `VERSION`, `UNSIGNED_LONG`, all spatial types like `GEO_POINT`, `GEO_SHAPE`, and all temporal periods like `DURATION` and `PERIOD`. -For a complete list of all types supported in `LOOKUP JOIN`, refer to the [`LOOKUP JOIN` supported types table](/reference/query-languages/esql/commands/processing-commands#esql-lookup-join). +For a complete list of all types supported in `LOOKUP JOIN`, refer to the [`LOOKUP JOIN` supported types table](/reference/query-languages/esql/commands/processing-commands.md#esql-lookup-join). ## Usage notes From 78e10d8123eba4b1943b6fda61764b1cc17813a8 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 2 Jul 2025 17:45:11 +0200 Subject: [PATCH 6/8] Make LOOKUP JOIN types table more compact --- .../_snippets/commands/types/lookup-join.md | 86 ++++--------------- .../xpack/esql/action/LookupJoinTypesIT.java | 4 +- .../expression/function/DocsV3Support.java | 44 +++++++++- 3 files changed, 60 insertions(+), 74 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/commands/types/lookup-join.md b/docs/reference/query-languages/esql/_snippets/commands/types/lookup-join.md index 04f62e3f59d47..3e54f0ad66277 100644 --- a/docs/reference/query-languages/esql/_snippets/commands/types/lookup-join.md +++ b/docs/reference/query-languages/esql/_snippets/commands/types/lookup-join.md @@ -2,76 +2,20 @@ **Supported types** -| main | join | +| field from the left index | field from the lookup index | | --- | --- | -| boolean | boolean | -| byte | byte | -| byte | double | -| byte | float | -| byte | half_float | -| byte | integer | -| byte | long | -| byte | scaled_float | -| byte | short | -| date | date | -| date_nanos | date_nanos | -| double | byte | -| double | double | -| double | float | -| double | half_float | -| double | integer | -| double | long | -| double | scaled_float | -| double | short | -| float | byte | -| float | double | -| float | float | -| float | half_float | -| float | integer | -| float | long | -| float | scaled_float | -| float | short | -| half_float | byte | -| half_float | double | -| half_float | float | -| half_float | half_float | -| half_float | integer | -| half_float | long | -| half_float | scaled_float | -| half_float | short | -| integer | byte | -| integer | double | -| integer | float | -| integer | half_float | -| integer | integer | -| integer | long | -| integer | scaled_float | -| integer | short | -| ip | ip | -| keyword | keyword | -| long | byte | -| long | double | -| long | float | -| long | half_float | -| long | integer | -| long | long | -| long | scaled_float | -| long | short | -| scaled_float | byte | -| scaled_float | double | -| scaled_float | float | -| scaled_float | half_float | -| scaled_float | integer | -| scaled_float | long | -| scaled_float | scaled_float | -| scaled_float | short | -| short | byte | -| short | double | -| short | float | -| short | half_float | -| short | integer | -| short | long | -| short | scaled_float | -| short | short | -| text | keyword | +| boolean | boolean | +| byte | half_float, float, double, scaled_float, byte, short, integer, long | +| date | date | +| date_nanos | date_nanos | +| double | half_float, float, double, scaled_float, byte, short, integer, long | +| float | half_float, float, double, scaled_float, byte, short, integer, long | +| half_float | half_float, float, double, scaled_float, byte, short, integer, long | +| integer | half_float, float, double, scaled_float, byte, short, integer, long | +| ip | ip | +| keyword | keyword | +| long | half_float, float, double, scaled_float, byte, short, integer, long | +| scaled_float | half_float, float, double, scaled_float, byte, short, integer, long | +| short | half_float, float, double, scaled_float, byte, short, integer, long | +| text | keyword | diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java index 44b465270579d..91a72d96ec4bf 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java @@ -768,8 +768,8 @@ private boolean isValidDataType(DataType dataType) { private static void saveJoinTypes(Supplier, DataType>> signatures) throws Exception { ArrayList args = new ArrayList<>(); - args.add(new EsqlFunctionRegistry.ArgSignature("main", null, "Field from the main index", false, false)); - args.add(new EsqlFunctionRegistry.ArgSignature("join", null, "Field from the join index", false, false)); + args.add(new EsqlFunctionRegistry.ArgSignature("field from the left index", null, null, false, false)); + args.add(new EsqlFunctionRegistry.ArgSignature("field from the lookup index", null, null, false, false)); DocsV3Support.CommandsDocsSupport docs = new DocsV3Support.CommandsDocsSupport( "lookup-join", LookupJoinTypesIT.class, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java index 77a78241fb863..d52032684a25a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java @@ -64,6 +64,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.TreeMap; import java.util.function.Function; import java.util.function.Supplier; import java.util.regex.Matcher; @@ -311,7 +312,7 @@ public License.OperationMode invoke(List fieldTypes) throws Exception protected final String name; protected final FunctionDefinition definition; protected final Logger logger; - private final Supplier, DataType>> signatures; + protected final Supplier, DataType>> signatures; private TempFileWriter tempFileWriter; private final LicenseRequirementChecker licenseChecker; @@ -930,6 +931,47 @@ void renderKibanaCommandDefinition() throws Exception { writeToTempKibanaDir("definition", "json", rendered); } } + + @Override + void renderTypes(String name, List args) throws IOException { + assert args.size() == 2; + StringBuilder header = new StringBuilder("| "); + StringBuilder separator = new StringBuilder("| "); + List argNames = args.stream().map(EsqlFunctionRegistry.ArgSignature::name).toList(); + for (String arg : argNames) { + header.append(arg).append(" | "); + separator.append("---").append(" | "); + } + + Map> compactedTable = new TreeMap<>(); + for (Map.Entry, DataType> sig : this.signatures.get().entrySet()) { + if (shouldHideSignature(sig.getKey(), sig.getValue())) { + continue; + } + String mainType = sig.getKey().getFirst().esNameIfPossible(); + String secondaryType = sig.getKey().get(1).esNameIfPossible(); + List secondaryTypes = compactedTable.computeIfAbsent(mainType, (k) -> new ArrayList<>()); + secondaryTypes.add(secondaryType); + } + + List table = new ArrayList<>(); + for (Map.Entry> sig : compactedTable.entrySet()) { + String row = "| " + sig.getKey() + " | " + String.join(", ", sig.getValue()) + " |"; + table.add(row); + } + Collections.sort(table); + if (table.isEmpty()) { + logger.info("Warning: No table of types generated for [{}]", name); + return; + } + + String rendered = DOCS_WARNING + """ + **Supported types** + + """ + header + "\n" + separator + "\n" + String.join("\n", table) + "\n\n"; + logger.info("Writing function types for [{}]:\n{}", name, rendered); + writeToTempSnippetsDir("types", rendered); + } } protected String buildFunctionSignatureSvg() throws IOException { From 2092a23170b4405762a55d619b3e3e5437d4c6d1 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 2 Jul 2025 17:48:50 +0200 Subject: [PATCH 7/8] Update docs/reference/query-languages/esql/esql-lookup-join.md Co-authored-by: Alexander Spies --- docs/reference/query-languages/esql/esql-lookup-join.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/query-languages/esql/esql-lookup-join.md b/docs/reference/query-languages/esql/esql-lookup-join.md index d954be2ed1cb0..2211f87a907d4 100644 --- a/docs/reference/query-languages/esql/esql-lookup-join.md +++ b/docs/reference/query-languages/esql/esql-lookup-join.md @@ -155,7 +155,7 @@ To use `LOOKUP JOIN`, the following requirements must be met: To obtain a join key with a compatible type, use a [conversion function](/reference/query-languages/esql/functions-operators/type-conversion-functions.md) if needed. -The list of unsupported fields includes all types not supported by {{esql}} as described in the [Unsupported Field Types documentation](/reference/query-languages/esql/limitations.md#_unsupported_types). +The list of unsupported field types includes all types not supported by {{esql}} as described in the [Unsupported Field Types documentation](/reference/query-languages/esql/limitations.md#_unsupported_types). as well as the following: `VERSION`, `UNSIGNED_LONG`, all spatial types like `GEO_POINT`, `GEO_SHAPE`, and all temporal periods like `DURATION` and `PERIOD`. From 724d3d4c007dc441d54c29afe4113e6c93482abb Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 2 Jul 2025 18:04:53 +0200 Subject: [PATCH 8/8] Code review suggestions --- .../query-languages/esql/esql-lookup-join.md | 38 +++++++++++++------ x-pack/plugin/esql/build.gradle | 2 + .../xpack/esql/action/LookupJoinTypesIT.java | 1 + 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/docs/reference/query-languages/esql/esql-lookup-join.md b/docs/reference/query-languages/esql/esql-lookup-join.md index d954be2ed1cb0..0b6764834e1d6 100644 --- a/docs/reference/query-languages/esql/esql-lookup-join.md +++ b/docs/reference/query-languages/esql/esql-lookup-join.md @@ -142,24 +142,38 @@ Refer to the examples section of the [`LOOKUP JOIN`](/reference/query-languages/ ## Prerequisites [esql-lookup-join-prereqs] -To use `LOOKUP JOIN`, the following requirements must be met: +### Index configuration -* Indices used for lookups must be configured with the [`lookup` index mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting) -* **Compatible data types**: The join key and join field in the lookup index must have compatible data types. This means: - * The data types must either be identical or be internally represented as the same type in {{esql}} - * Numeric types follow these compatibility rules: - * `short` and `byte` are compatible with `integer` (all represented as `int`) - * `float`, `half_float`, and `scaled_float` are compatible with `double` (all represented as `double`) - * For text fields: You can only use text fields as the join key on the left-hand side of the join and only if they have a `.keyword` subfield - * `DATE` and `DATE_NANOS` can only be joined against the exact same type. +Indices used for lookups must be configured with the [`lookup` index mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting). +### Data type compatibility + +Join keys must have compatible data types between the source and lookup indices. Types within the same compatibility group can be joined together: + +| Compatibility group | Types | Notes | +|------------------------|-------------------------------------------------------------------------------------|----------------------------------------------------------------------------------| +| **Numeric family** | `byte`, `short`, `integer`, `long`, `half_float`, `float`, `scaled_float`, `double` | All compatible | +| **Keyword family** | `keyword`, `text.keyword` | Text fields only as join key on left-hand side and must have `.keyword` subfield | +| **Date (Exact)** | `date` | Must match exactly | +| **Date Nanos (Exact)** | `date_nanos` | Must match exactly | +| **Boolean** | `boolean` | Must match exactly | + +```{tip} To obtain a join key with a compatible type, use a [conversion function](/reference/query-languages/esql/functions-operators/type-conversion-functions.md) if needed. +``` -The list of unsupported fields includes all types not supported by {{esql}} as described in the [Unsupported Field Types documentation](/reference/query-languages/esql/limitations.md#_unsupported_types). -as well as the following: `VERSION`, `UNSIGNED_LONG`, all spatial types like `GEO_POINT`, `GEO_SHAPE`, and all -temporal periods like `DURATION` and `PERIOD`. +### Unsupported Types +In addition to the [{{esql}} unsupported field types](/reference/query-languages/esql/limitations.md#_unsupported_types), `LOOKUP JOIN` does not support: + +* `VERSION` +* `UNSIGNED_LONG` +* Spatial types like `GEO_POINT`, `GEO_SHAPE` +* Temporal intervals like `DURATION`, `PERIOD` + +```{note} For a complete list of all types supported in `LOOKUP JOIN`, refer to the [`LOOKUP JOIN` supported types table](/reference/query-languages/esql/commands/processing-commands.md#esql-lookup-join). +``` ## Usage notes diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle index 3c83be5fe1856..447ceb53489f2 100644 --- a/x-pack/plugin/esql/build.gradle +++ b/x-pack/plugin/esql/build.gradle @@ -226,6 +226,8 @@ tasks.named("test").configure { } } +// This is similar to the test task above, but needed for the LookupJoinTypesIT which runs in the internalClusterTest task +// and generates a types table for the LOOKUP JOIN command. It is possible in future we might have move tests that do this. tasks.named("internalClusterTest").configure { if (buildParams.ci == false) { systemProperty 'generateDocs', true diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java index 91a72d96ec4bf..731b2976f88d8 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java @@ -268,6 +268,7 @@ private static boolean existingIndex(Collection existing, DataType return existing.stream().anyMatch(c -> c.exists(indexName)); } + /** This test generates documentation for the supported output types of the lookup join. */ public void testOutputSupportedTypes() throws Exception { Map, DataType> signatures = new LinkedHashMap<>(); for (TestConfigs configs : testConfigurations.values()) {