From fb0c33ebb3bfe2fff39bf34b2a71228ce280bfb1 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Thu, 3 Jul 2025 11:09:12 +0200 Subject: [PATCH 1/3] ES|QL: add tests and fix docs for LOOKUP JOIN with index datemath --- .../query-languages/esql/esql-lookup-join.md | 2 +- .../xpack/esql/qa/single_node/RestEsqlIT.java | 114 ++++++++++++++++++ 2 files changed, 115 insertions(+), 1 deletion(-) diff --git a/docs/reference/query-languages/esql/esql-lookup-join.md b/docs/reference/query-languages/esql/esql-lookup-join.md index d57437833c1b2..293c5bcfeeea7 100644 --- a/docs/reference/query-languages/esql/esql-lookup-join.md +++ b/docs/reference/query-languages/esql/esql-lookup-join.md @@ -181,6 +181,6 @@ The following are the current limitations with `LOOKUP JOIN`: * Indices in [`lookup` mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting) are always single-sharded. * Cross cluster search is unsupported initially. Both source and lookup indices must be local. * Currently, only matching on equality is supported. -* `LOOKUP JOIN` can only use a single match field and a single index. Wildcards, aliases, datemath, and datastreams are not supported. +* `LOOKUP JOIN` can only use a single match field and a single index. Wildcards, aliases, datemath, and datastreams are supported, as long as the index pattern matches a single concrete index {applies_to}`stack: ga 9.1.0`. * The name of the match field in `LOOKUP JOIN lu_idx ON match_field` must match an existing field in the query. This may require `RENAME`s or `EVAL`s to achieve. * The query will circuit break if there are too many matching documents in the lookup index, or if the documents are too large. More precisely, `LOOKUP JOIN` works in batches of, normally, about 10,000 rows; a large amount of heap space is needed if the matching documents from the lookup index for a batch are multiple megabytes or larger. This is roughly the same as for `ENRICH`. diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index 1ef49652c3afc..d606987a1ee41 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -40,6 +40,9 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -786,6 +789,117 @@ && isMillisOrNanos(listOfTypes.get(j))) { } } + public void testDateMathIndexPattern() throws IOException { + ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); + + String[] indices = { + "test-index-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now), + "test-index-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now.minusDays(1)), + "test-index-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now.minusDays(2)) }; + + int idx = 0; + for (String index : indices) { + createIndex(index); + for (int i = 0; i < 10; i++) { + Request request = new Request("POST", "/" + index + "/_doc/"); + request.addParameter("refresh", "true"); + request.setJsonEntity("{\"f\":" + idx++ + "}"); + assertOK(client().performRequest(request)); + } + } + + String query = """ + { + "query": "from | sort f asc | limit 1 | keep f" + } + """; + Request request = new Request("POST", "/_query"); + request.setJsonEntity(query); + Response resp = client().performRequest(request); + Map results = entityAsMap(resp); + List values = (List) results.get("values"); + assertThat(values.size(), is(1)); + List row = (List) values.get(0); + assertThat(row.get(0), is(0)); + + query = """ + { + "query": "from | sort f asc | limit 1 | keep f" + } + """; + request = new Request("POST", "/_query"); + request.setJsonEntity(query); + resp = client().performRequest(request); + results = entityAsMap(resp); + values = (List) results.get("values"); + assertThat(values.size(), is(1)); + row = (List) values.get(0); + assertThat(row.get(0), is(10)); + + for (String index : indices) { + assertThat(deleteIndex(index).isAcknowledged(), is(true)); // clean up + } + } + + public void testDateMathInJoin() throws IOException { + ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); + + createIndex("idx", Settings.EMPTY, """ + { + "properties": { + "key": { + "type": "keyword" + } + } + } + """); + + Request request = new Request("POST", "/idx/_doc/"); + request.addParameter("refresh", "true"); + request.setJsonEntity("{\"key\":\"foo\"}"); + assertOK(client().performRequest(request)); + + String[] lookupIndices = { + "lookup-index-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now), + "lookup-index-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now.minusDays(1)) }; + + for (String index : lookupIndices) { + createIndex(index, Settings.builder().put("mode", "lookup").build(), """ + { + "properties": { + "key": { + "type": "keyword" + } + } + } + """); + request = new Request("POST", "/" + index + "/_doc/"); + request.addParameter("refresh", "true"); + request.setJsonEntity("{\"key\":\"foo\", \"value\": \"" + index + "\"}"); + assertOK(client().performRequest(request)); + } + + String[] queries = { + "from idx | lookup join on key | limit 1", + "from idx | lookup join on key | limit 1" }; + for (int i = 0; i < queries.length; i++) { + String queryPayload = "{\"query\": \"" + queries[i] + "\"}"; + request = new Request("POST", "/_query"); + request.setJsonEntity(queryPayload); + Response resp = client().performRequest(request); + Map results = entityAsMap(resp); + List values = (List) results.get("values"); + assertThat(values.size(), is(1)); + List row = (List) values.get(0); + assertThat(row.get(1), is(lookupIndices[i])); + } + + assertThat(deleteIndex("idx").isAcknowledged(), is(true)); // clean up + for (String index : lookupIndices) { + assertThat(deleteIndex(index).isAcknowledged(), is(true)); // clean up + } + } + static MapMatcher commonProfile() { return matchesMap() // .entry("description", any(String.class)) From 5855380edd27a8e4e61648f9b42c053f6e927380 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Thu, 3 Jul 2025 11:48:15 +0200 Subject: [PATCH 2/3] Fix docs --- docs/reference/query-languages/esql/esql-lookup-join.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/reference/query-languages/esql/esql-lookup-join.md b/docs/reference/query-languages/esql/esql-lookup-join.md index 293c5bcfeeea7..5882cdca645bc 100644 --- a/docs/reference/query-languages/esql/esql-lookup-join.md +++ b/docs/reference/query-languages/esql/esql-lookup-join.md @@ -181,6 +181,7 @@ The following are the current limitations with `LOOKUP JOIN`: * Indices in [`lookup` mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting) are always single-sharded. * Cross cluster search is unsupported initially. Both source and lookup indices must be local. * Currently, only matching on equality is supported. -* `LOOKUP JOIN` can only use a single match field and a single index. Wildcards, aliases, datemath, and datastreams are supported, as long as the index pattern matches a single concrete index {applies_to}`stack: ga 9.1.0`. +* `LOOKUP JOIN` can only use a single match field and a single index. Wildcards are not supported. + * Aliases, datemath, and datastreams are supported, as long as the index pattern matches a single concrete index {applies_to}`stack: ga 9.1.0`. * The name of the match field in `LOOKUP JOIN lu_idx ON match_field` must match an existing field in the query. This may require `RENAME`s or `EVAL`s to achieve. * The query will circuit break if there are too many matching documents in the lookup index, or if the documents are too large. More precisely, `LOOKUP JOIN` works in batches of, normally, about 10,000 rows; a large amount of heap space is needed if the matching documents from the lookup index for a batch are multiple megabytes or larger. This is roughly the same as for `ENRICH`. From 2c57f5f25dbf52d96e5ed123703712a86601c9f6 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Thu, 3 Jul 2025 12:07:50 +0200 Subject: [PATCH 3/3] Make tests more deterministic --- .../xpack/esql/qa/single_node/RestEsqlIT.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index d606987a1ee41..6dca5cf4efaf5 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -793,9 +793,9 @@ public void testDateMathIndexPattern() throws IOException { ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); String[] indices = { - "test-index-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now), - "test-index-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now.minusDays(1)), - "test-index-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now.minusDays(2)) }; + "test-index-" + DateTimeFormatter.ofPattern("yyyy", Locale.ROOT).format(now), + "test-index-" + DateTimeFormatter.ofPattern("yyyy", Locale.ROOT).format(now.minusYears(1)), + "test-index-" + DateTimeFormatter.ofPattern("yyyy", Locale.ROOT).format(now.minusYears(2)) }; int idx = 0; for (String index : indices) { @@ -810,7 +810,7 @@ public void testDateMathIndexPattern() throws IOException { String query = """ { - "query": "from | sort f asc | limit 1 | keep f" + "query": "from | sort f asc | limit 1 | keep f" } """; Request request = new Request("POST", "/_query"); @@ -824,7 +824,7 @@ public void testDateMathIndexPattern() throws IOException { query = """ { - "query": "from | sort f asc | limit 1 | keep f" + "query": "from | sort f asc | limit 1 | keep f" } """; request = new Request("POST", "/_query"); @@ -860,8 +860,8 @@ public void testDateMathInJoin() throws IOException { assertOK(client().performRequest(request)); String[] lookupIndices = { - "lookup-index-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now), - "lookup-index-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now.minusDays(1)) }; + "lookup-index-" + DateTimeFormatter.ofPattern("yyyy", Locale.ROOT).format(now), + "lookup-index-" + DateTimeFormatter.ofPattern("yyyy", Locale.ROOT).format(now.minusYears(1)) }; for (String index : lookupIndices) { createIndex(index, Settings.builder().put("mode", "lookup").build(), """ @@ -880,8 +880,8 @@ public void testDateMathInJoin() throws IOException { } String[] queries = { - "from idx | lookup join on key | limit 1", - "from idx | lookup join on key | limit 1" }; + "from idx | lookup join on key | limit 1", + "from idx | lookup join on key | limit 1" }; for (int i = 0; i < queries.length; i++) { String queryPayload = "{\"query\": \"" + queries[i] + "\"}"; request = new Request("POST", "/_query");