Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion docs/reference/query-languages/esql/esql-lookup-join.md
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,7 @@ The following are the current limitations with `LOOKUP JOIN`:
* Indices in [`lookup` mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting) are always single-sharded.
* Cross cluster search is unsupported initially. Both source and lookup indices must be local.
* Currently, only matching on equality is supported.
* `LOOKUP JOIN` can only use a single match field and a single index. Wildcards, aliases, datemath, and datastreams are not supported.
* `LOOKUP JOIN` can only use a single match field and a single index. Wildcards are not supported.
* Aliases, datemath, and datastreams are supported, as long as the index pattern matches a single concrete index {applies_to}`stack: ga 9.1.0`.
* The name of the match field in `LOOKUP JOIN lu_idx ON match_field` must match an existing field in the query. This may require `RENAME`s or `EVAL`s to achieve.
* The query will circuit break if there are too many matching documents in the lookup index, or if the documents are too large. More precisely, `LOOKUP JOIN` works in batches of, normally, about 10,000 rows; a large amount of heap space is needed if the matching documents from the lookup index for a batch are multiple megabytes or larger. This is roughly the same as for `ENRICH`.
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
Expand Down Expand Up @@ -786,6 +789,117 @@ && isMillisOrNanos(listOfTypes.get(j))) {
}
}

public void testDateMathIndexPattern() throws IOException {
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Using now may lead to flakiness when the tests are executed around midnight, no? Maybe a fixed date would be better?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought about it, but date math is all about dynamic patterns using now. I'm not sure a test with a fixed date will be realistic.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One thing I can do is to use the year only, that should drastically reduce the risk of flakiness

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How long does this test runs?
I have seen some sql/eql examples that add assumption they are executed at least 10-15 minutes before midnight.


String[] indices = {
"test-index-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now),
"test-index-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now.minusDays(1)),
"test-index-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now.minusDays(2)) };

int idx = 0;
for (String index : indices) {
createIndex(index);
for (int i = 0; i < 10; i++) {
Request request = new Request("POST", "/" + index + "/_doc/");
request.addParameter("refresh", "true");
request.setJsonEntity("{\"f\":" + idx++ + "}");
assertOK(client().performRequest(request));
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this might need a refresh.
I wonder if we have anything similar to indexRandom for the RestTestCases?

}

String query = """
{
"query": "from <test-index-{now/d}> | sort f asc | limit 1 | keep f"
}
""";
Request request = new Request("POST", "/_query");
request.setJsonEntity(query);
Response resp = client().performRequest(request);
Map<String, Object> results = entityAsMap(resp);
List<?> values = (List<?>) results.get("values");
assertThat(values.size(), is(1));
List<?> row = (List<?>) values.get(0);
assertThat(row.get(0), is(0));

query = """
{
"query": "from <test-index-{now/d-1d}> | sort f asc | limit 1 | keep f"
}
""";
request = new Request("POST", "/_query");
request.setJsonEntity(query);
resp = client().performRequest(request);
results = entityAsMap(resp);
values = (List<?>) results.get("values");
assertThat(values.size(), is(1));
row = (List<?>) values.get(0);
assertThat(row.get(0), is(10));

for (String index : indices) {
assertThat(deleteIndex(index).isAcknowledged(), is(true)); // clean up
}
}

public void testDateMathInJoin() throws IOException {
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);

createIndex("idx", Settings.EMPTY, """
{
"properties": {
"key": {
"type": "keyword"
}
}
}
""");

Request request = new Request("POST", "/idx/_doc/");
request.addParameter("refresh", "true");
request.setJsonEntity("{\"key\":\"foo\"}");
assertOK(client().performRequest(request));

String[] lookupIndices = {
"lookup-index-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now),
"lookup-index-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now.minusDays(1)) };

for (String index : lookupIndices) {
createIndex(index, Settings.builder().put("mode", "lookup").build(), """
{
"properties": {
"key": {
"type": "keyword"
}
}
}
""");
request = new Request("POST", "/" + index + "/_doc/");
request.addParameter("refresh", "true");
request.setJsonEntity("{\"key\":\"foo\", \"value\": \"" + index + "\"}");
assertOK(client().performRequest(request));
}

String[] queries = {
"from idx | lookup join <lookup-index-{now/d}> on key | limit 1",
"from idx | lookup join <lookup-index-{now/d-1d}> on key | limit 1" };
for (int i = 0; i < queries.length; i++) {
String queryPayload = "{\"query\": \"" + queries[i] + "\"}";
request = new Request("POST", "/_query");
request.setJsonEntity(queryPayload);
Response resp = client().performRequest(request);
Map<String, Object> results = entityAsMap(resp);
List<?> values = (List<?>) results.get("values");
assertThat(values.size(), is(1));
List<?> row = (List<?>) values.get(0);
assertThat(row.get(1), is(lookupIndices[i]));
}

assertThat(deleteIndex("idx").isAcknowledged(), is(true)); // clean up
for (String index : lookupIndices) {
assertThat(deleteIndex(index).isAcknowledged(), is(true)); // clean up
}
}

static MapMatcher commonProfile() {
return matchesMap() //
.entry("description", any(String.class))
Expand Down
Loading