Skip to content

Update grammar to rely on indexPattern instead of identifier in join target #120494

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 27 commits into from
Jan 24, 2025
Merged

Update grammar to rely on indexPattern instead of identifier in join target #120494

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
fa962c4
Update grammar to rely on indexPattern instead of identifier in join …
idegtiarenko Jan 20, 2025
6049006
Update docs/changelog/120494.yaml
idegtiarenko Jan 21, 2025
610b3bc
initial test
idegtiarenko Jan 21, 2025
8675905
add join type
idegtiarenko Jan 21, 2025
2770dbb
upd
idegtiarenko Jan 21, 2025
58b2f27
upd
idegtiarenko Jan 21, 2025
bf9c556
cleanup
idegtiarenko Jan 21, 2025
a7a91b9
fix quotes usage
idegtiarenko Jan 21, 2025
318fd1d
inc lookup version
idegtiarenko Jan 21, 2025
60444a8
add pattern test cases
idegtiarenko Jan 21, 2025
62d53f9
Merge branch 'main' into es-10559
idegtiarenko Jan 21, 2025
a5f713f
Merge branch 'main' into es-10559
idegtiarenko Jan 22, 2025
2b642b9
prohibit * in join patterns
idegtiarenko Jan 22, 2025
52d98e6
Merge branch 'main' into es-10559
idegtiarenko Jan 22, 2025
7b38bec
random identifier generator
idegtiarenko Jan 22, 2025
2cc2643
use random identifier generator
idegtiarenko Jan 23, 2025
9529874
Merge branch 'main' into es-10559
idegtiarenko Jan 23, 2025
e7d794d
fix merge
idegtiarenko Jan 23, 2025
62ba2fe
add date math generation
idegtiarenko Jan 23, 2025
f562bc1
rename
idegtiarenko Jan 23, 2025
9e7f125
upd
idegtiarenko Jan 23, 2025
a9ecc24
Merge branch 'main' into es-10559
idegtiarenko Jan 23, 2025
4574fd1
fix new tests
idegtiarenko Jan 23, 2025
856a6ec
add IT with quotes
idegtiarenko Jan 23, 2025
0a39bb2
Merge branch 'main' into es-10559
idegtiarenko Jan 24, 2025
c1b399d
add missing capability
idegtiarenko Jan 24, 2025
1116eb7
Merge branch 'main' into es-10559
idegtiarenko Jan 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/changelog/120494.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 120494
summary: Update grammar to rely on `indexPattern` instead of identifier in join target
area: ES|QL
type: enhancement
issues: []
18 changes: 9 additions & 9 deletions ...n/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java
View file
Open in desktop
Copy link
Member

@costin costin Jan 22, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add randomization to have the index specified with and without double quotes.

Copy link
Contributor Author

@idegtiarenko idegtiarenko Jan 22, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would like to keep this test simple. All possible index name variations are covered in dedicated parser test.

Original file line number Diff line number Diff line change
Expand Up @@ -548,12 +548,12 @@ record Listen(long timestamp, String songId, double duration) {
public void testLookupJoinIndexAllowed() throws Exception {
assumeTrue(
"Requires LOOKUP JOIN capability",
EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V11.capabilityName()))
EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName()))
);

Response resp = runESQLCommand(
"metadata1_read2",
"ROW x = 40.0 | EVAL value = x | LOOKUP JOIN `lookup-user2` ON value | KEEP x, org"
"ROW x = 40.0 | EVAL value = x | LOOKUP JOIN lookup-user2 ON value | KEEP x, org"
);
assertOK(resp);
Map<String, Object> respMap = entityAsMap(resp);
Expand All @@ -564,7 +564,7 @@ public void testLookupJoinIndexAllowed() throws Exception {
assertThat(respMap.get("values"), equalTo(List.of(List.of(40.0, "sales"))));

// Alias, should find the index and the row
resp = runESQLCommand("alias_user1", "ROW x = 31.0 | EVAL value = x | LOOKUP JOIN `lookup-first-alias` ON value | KEEP x, org");
resp = runESQLCommand("alias_user1", "ROW x = 31.0 | EVAL value = x | LOOKUP JOIN lookup-first-alias ON value | KEEP x, org");
assertOK(resp);
respMap = entityAsMap(resp);
assertThat(
Expand All @@ -574,7 +574,7 @@ public void testLookupJoinIndexAllowed() throws Exception {
assertThat(respMap.get("values"), equalTo(List.of(List.of(31.0, "sales"))));

// Alias, for a row that's filtered out
resp = runESQLCommand("alias_user1", "ROW x = 123.0 | EVAL value = x | LOOKUP JOIN `lookup-first-alias` ON value | KEEP x, org");
resp = runESQLCommand("alias_user1", "ROW x = 123.0 | EVAL value = x | LOOKUP JOIN lookup-first-alias ON value | KEEP x, org");
assertOK(resp);
respMap = entityAsMap(resp);
assertThat(
Expand All @@ -587,12 +587,12 @@ public void testLookupJoinIndexAllowed() throws Exception {
public void testLookupJoinIndexForbidden() throws Exception {
assumeTrue(
"Requires LOOKUP JOIN capability",
EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V11.capabilityName()))
EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName()))
);

var resp = expectThrows(
ResponseException.class,
() -> runESQLCommand("metadata1_read2", "FROM lookup-user2 | EVAL value = 10.0 | LOOKUP JOIN `lookup-user1` ON value | KEEP x")
() -> runESQLCommand("metadata1_read2", "FROM lookup-user2 | EVAL value = 10.0 | LOOKUP JOIN lookup-user1 ON value | KEEP x")
);
assertThat(resp.getMessage(), containsString("Unknown index [lookup-user1]"));
assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));
Expand All @@ -601,22 +601,22 @@ public void testLookupJoinIndexForbidden() throws Exception {
ResponseException.class,
() -> runESQLCommand(
"metadata1_read2",
"FROM lookup-user2 | EVAL value = 10.0 | LOOKUP JOIN `lookup-first-alias` ON value | KEEP x"
"FROM lookup-user2 | EVAL value = 10.0 | LOOKUP JOIN lookup-first-alias ON value | KEEP x"
)
);
assertThat(resp.getMessage(), containsString("Unknown index [lookup-first-alias]"));
assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));

resp = expectThrows(
ResponseException.class,
() -> runESQLCommand("metadata1_read2", "ROW x = 10.0 | EVAL value = x | LOOKUP JOIN `lookup-user1` ON value | KEEP x")
() -> runESQLCommand("metadata1_read2", "ROW x = 10.0 | EVAL value = x | LOOKUP JOIN lookup-user1 ON value | KEEP x")
);
assertThat(resp.getMessage(), containsString("Unknown index [lookup-user1]"));
assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));

resp = expectThrows(
ResponseException.class,
() -> runESQLCommand("alias_user1", "ROW x = 10.0 | EVAL value = x | LOOKUP JOIN `lookup-user1` ON value | KEEP x")
() -> runESQLCommand("alias_user1", "ROW x = 10.0 | EVAL value = x | LOOKUP JOIN lookup-user1 ON value | KEEP x")
);
assertThat(resp.getMessage(), containsString("Unknown index [lookup-user1]"));
assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));
Expand Down
4 changes: 2 additions & 2 deletions ...r/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import java.util.List;

import static org.elasticsearch.xpack.esql.CsvTestUtils.isEnabled;
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V11;
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V12;

public class MixedClusterEsqlSpecIT extends EsqlSpecTestCase {
@ClassRule
Expand Down Expand Up @@ -82,7 +82,7 @@ protected boolean supportsInferenceTestService() {

@Override
protected boolean supportsIndexModeLookup() throws IOException {
return hasCapabilities(List.of(JOIN_LOOKUP_V11.capabilityName()));
return hasCapabilities(List.of(JOIN_LOOKUP_V12.capabilityName()));
}

@Override
Expand Down
4 changes: 2 additions & 2 deletions ...i-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources;
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS;
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V2;
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V11;
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V12;
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_PLANNING_V1;
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METADATA_FIELDS_REMOTE_TEST;
import static org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase.Mode.SYNC;
Expand Down Expand Up @@ -124,7 +124,7 @@ protected void shouldSkipTest(String testName) throws IOException {
assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS.capabilityName()));
assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V2.capabilityName()));
assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_PLANNING_V1.capabilityName()));
assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP_V11.capabilityName()));
assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP_V12.capabilityName()));
}

private TestFeatureService remoteFeaturesService() throws IOException {
Expand Down
2 changes: 1 addition & 1 deletion ...ver/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ public void testIndicesDontExist() throws IOException {
assertThat(e.getMessage(), containsString("index_not_found_exception"));
assertThat(e.getMessage(), anyOf(containsString("no such index [foo]"), containsString("no such index [remote_cluster:foo]")));

if (EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()) {
if (EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()) {
e = expectThrows(
ResponseException.class,
() -> runEsql(timestampFilter("gte", "2020-01-01").query(from("test1") + " | LOOKUP JOIN foo ON id1"))
Expand Down
Loading