From 66d8530e0066a8844df18c57fa166c819e780295 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Thu, 3 Jul 2025 13:08:55 +0200 Subject: [PATCH] ES|QL: add tests and fix docs for LOOKUP JOIN with index datemath (#130535) --- .../query-languages/esql/esql-lookup-join.md | 3 +- .../xpack/esql/qa/single_node/RestEsqlIT.java | 114 ++++++++++++++++++ 2 files changed, 116 insertions(+), 1 deletion(-) diff --git a/docs/reference/query-languages/esql/esql-lookup-join.md b/docs/reference/query-languages/esql/esql-lookup-join.md index 0b6764834e1d6..826de488e5897 100644 --- a/docs/reference/query-languages/esql/esql-lookup-join.md +++ b/docs/reference/query-languages/esql/esql-lookup-join.md @@ -200,6 +200,7 @@ The following are the current limitations with `LOOKUP JOIN`: * Indices in [`lookup` mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting) are always single-sharded. * Cross cluster search is unsupported initially. Both source and lookup indices must be local. * Currently, only matching on equality is supported. -* `LOOKUP JOIN` can only use a single match field and a single index. Wildcards, aliases, datemath, and datastreams are not supported. +* `LOOKUP JOIN` can only use a single match field and a single index. Wildcards are not supported. + * Aliases, datemath, and datastreams are supported, as long as the index pattern matches a single concrete index {applies_to}`stack: ga 9.1.0`. * The name of the match field in `LOOKUP JOIN lu_idx ON match_field` must match an existing field in the query. This may require `RENAME`s or `EVAL`s to achieve. * The query will circuit break if there are too many matching documents in the lookup index, or if the documents are too large. More precisely, `LOOKUP JOIN` works in batches of, normally, about 10,000 rows; a large amount of heap space is needed if the matching documents from the lookup index for a batch are multiple megabytes or larger. This is roughly the same as for `ENRICH`. diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index 1ef49652c3afc..6dca5cf4efaf5 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -40,6 +40,9 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -786,6 +789,117 @@ && isMillisOrNanos(listOfTypes.get(j))) { } } + public void testDateMathIndexPattern() throws IOException { + ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); + + String[] indices = { + "test-index-" + DateTimeFormatter.ofPattern("yyyy", Locale.ROOT).format(now), + "test-index-" + DateTimeFormatter.ofPattern("yyyy", Locale.ROOT).format(now.minusYears(1)), + "test-index-" + DateTimeFormatter.ofPattern("yyyy", Locale.ROOT).format(now.minusYears(2)) }; + + int idx = 0; + for (String index : indices) { + createIndex(index); + for (int i = 0; i < 10; i++) { + Request request = new Request("POST", "/" + index + "/_doc/"); + request.addParameter("refresh", "true"); + request.setJsonEntity("{\"f\":" + idx++ + "}"); + assertOK(client().performRequest(request)); + } + } + + String query = """ + { + "query": "from | sort f asc | limit 1 | keep f" + } + """; + Request request = new Request("POST", "/_query"); + request.setJsonEntity(query); + Response resp = client().performRequest(request); + Map results = entityAsMap(resp); + List values = (List) results.get("values"); + assertThat(values.size(), is(1)); + List row = (List) values.get(0); + assertThat(row.get(0), is(0)); + + query = """ + { + "query": "from | sort f asc | limit 1 | keep f" + } + """; + request = new Request("POST", "/_query"); + request.setJsonEntity(query); + resp = client().performRequest(request); + results = entityAsMap(resp); + values = (List) results.get("values"); + assertThat(values.size(), is(1)); + row = (List) values.get(0); + assertThat(row.get(0), is(10)); + + for (String index : indices) { + assertThat(deleteIndex(index).isAcknowledged(), is(true)); // clean up + } + } + + public void testDateMathInJoin() throws IOException { + ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); + + createIndex("idx", Settings.EMPTY, """ + { + "properties": { + "key": { + "type": "keyword" + } + } + } + """); + + Request request = new Request("POST", "/idx/_doc/"); + request.addParameter("refresh", "true"); + request.setJsonEntity("{\"key\":\"foo\"}"); + assertOK(client().performRequest(request)); + + String[] lookupIndices = { + "lookup-index-" + DateTimeFormatter.ofPattern("yyyy", Locale.ROOT).format(now), + "lookup-index-" + DateTimeFormatter.ofPattern("yyyy", Locale.ROOT).format(now.minusYears(1)) }; + + for (String index : lookupIndices) { + createIndex(index, Settings.builder().put("mode", "lookup").build(), """ + { + "properties": { + "key": { + "type": "keyword" + } + } + } + """); + request = new Request("POST", "/" + index + "/_doc/"); + request.addParameter("refresh", "true"); + request.setJsonEntity("{\"key\":\"foo\", \"value\": \"" + index + "\"}"); + assertOK(client().performRequest(request)); + } + + String[] queries = { + "from idx | lookup join on key | limit 1", + "from idx | lookup join on key | limit 1" }; + for (int i = 0; i < queries.length; i++) { + String queryPayload = "{\"query\": \"" + queries[i] + "\"}"; + request = new Request("POST", "/_query"); + request.setJsonEntity(queryPayload); + Response resp = client().performRequest(request); + Map results = entityAsMap(resp); + List values = (List) results.get("values"); + assertThat(values.size(), is(1)); + List row = (List) values.get(0); + assertThat(row.get(1), is(lookupIndices[i])); + } + + assertThat(deleteIndex("idx").isAcknowledged(), is(true)); // clean up + for (String index : lookupIndices) { + assertThat(deleteIndex(index).isAcknowledged(), is(true)); // clean up + } + } + static MapMatcher commonProfile() { return matchesMap() // .entry("description", any(String.class))