ESQL: Enable LOOKUP JOIN in non-snapshot builds (#121193)

This effectively releases LOOKUP JOIN into tech preview. Docs will follow in a separate PR.

* Enable the lexing/grammar for LOOKUP JOIN in non-snapshot builds.
* Remove the grammar for the unsupported | JOIN ... command (without LOOKUP as first keyword). The way the lexer modes work, otherwise we'd also have to enable | JOIN ... syntax on non-snapshot builds and would have to add additional validation to provide appropriate error messages.
* Remove grammar for LOOKUP JOIN index AS ... because qualifiers are not yet supported. Otherwise we'd have to put in additional validation as well to prevent such queries.

---------

Co-authored-by: Bogdan Pintea <[email protected]>
Co-authored-by: elasticsearchmachine <[email protected]>
Co-authored-by: Nik Everett <[email protected]>

Author: 4 people

docs/changelog/121193.yaml

@@ -0,0 +1,18 @@
+pr: 121193
+summary: Enable LOOKUP JOIN in non-snapshot builds
+area: ES|QL
+type: enhancement
+issues:
+ - 121185
+highlight:
+  title: Enable LOOKUP JOIN in non-snapshot builds
+  body: |-
+    This effectively releases LOOKUP JOIN into tech preview. Docs will
+    follow in a separate PR.
+
+    - Enable the lexing/grammar for LOOKUP JOIN in non-snapshot builds.
+    - Remove the grammar for the unsupported `| JOIN ...` command (without `LOOKUP` as first keyword). The way the lexer modes work, otherwise we'd also have to enable `| JOIN ...` syntax on non-snapshot builds and would have to add additional validation to provide appropriate error messages.
+    - Remove grammar for `LOOKUP JOIN index AS ...` because qualifiers are not yet supported. Otherwise we'd have to put in additional validation as well to prevent such queries.
+
+    Also fix https://github.com/elastic/elasticsearch/issues/121185
+  notable: true

muted-tests.yml

@@ -314,8 +314,6 @@ tests:
 - class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
   method: testSetEnabled
   issue: https://github.com/elastic/elasticsearch/issues/121183
-- class: org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizerTests
-  issue: https://github.com/elastic/elasticsearch/issues/121185
 - class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
   method: test {yaml=cat.aliases/10_basic/Simple alias}
   issue: https://github.com/elastic/elasticsearch/issues/121186

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java

@@ -232,7 +232,7 @@ protected boolean supportsIndexModeLookup() throws IOException {
     protected final void doTest() throws Throwable {
         RequestObjectBuilder builder = new RequestObjectBuilder(randomFrom(XContentType.values()));
 
-        if (testCase.query.toUpperCase(Locale.ROOT).contains("LOOKUP")) {
+        if (testCase.query.toUpperCase(Locale.ROOT).contains("LOOKUP_\uD83D\uDC14")) {
             builder.tables(tables());
         }
 

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TelemetryIT.java

@@ -142,11 +142,9 @@ public static Iterable<Object[]> parameters() {
                         | EVAL y = to_str(host)
                         | LOOKUP JOIN lookup_idx ON host
                         """,
-                    Build.current().isSnapshot()
-                        ? Map.ofEntries(Map.entry("FROM", 1), Map.entry("EVAL", 1), Map.entry("LOOKUP JOIN", 1))
-                        : Collections.emptyMap(),
-                    Build.current().isSnapshot() ? Map.ofEntries(Map.entry("TO_STRING", 1)) : Collections.emptyMap(),
-                    Build.current().isSnapshot()
+                    Map.ofEntries(Map.entry("FROM", 1), Map.entry("EVAL", 1), Map.entry("LOOKUP JOIN", 1)),
+                    Map.ofEntries(Map.entry("TO_STRING", 1)),
+                    true
                 ) },
             new Object[] {
                 new Test(

x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4

@@ -73,6 +73,7 @@ SHOW : 'show'                 -> pushMode(SHOW_MODE);
 SORT : 'sort'                 -> pushMode(EXPRESSION_MODE);
 STATS : 'stats'               -> pushMode(EXPRESSION_MODE);
 WHERE : 'where'               -> pushMode(EXPRESSION_MODE);
+JOIN_LOOKUP : 'lookup'        -> pushMode(JOIN_MODE);
 //
 // in development
 //
@@ -88,11 +89,9 @@ DEV_INLINESTATS : {this.isDevVersion()}? 'inlinestats'   -> pushMode(EXPRESSION_
 DEV_LOOKUP :      {this.isDevVersion()}? 'lookup_🐔'      -> pushMode(LOOKUP_MODE);
 DEV_METRICS :     {this.isDevVersion()}? 'metrics'       -> pushMode(METRICS_MODE);
 // list of all JOIN commands
-DEV_JOIN :        {this.isDevVersion()}? 'join'          -> pushMode(JOIN_MODE);
 DEV_JOIN_FULL :   {this.isDevVersion()}? 'full'          -> pushMode(JOIN_MODE);
 DEV_JOIN_LEFT :   {this.isDevVersion()}? 'left'          -> pushMode(JOIN_MODE);
 DEV_JOIN_RIGHT :  {this.isDevVersion()}? 'right'         -> pushMode(JOIN_MODE);
-DEV_JOIN_LOOKUP : {this.isDevVersion()}? 'lookup'        -> pushMode(JOIN_MODE);
 
 
 //
@@ -556,7 +555,7 @@ LOOKUP_FIELD_WS
 //
 mode JOIN_MODE;
 JOIN_PIPE : PIPE -> type(PIPE), popMode;
-JOIN_JOIN : DEV_JOIN -> type(DEV_JOIN);
+JOIN : 'join';
 JOIN_AS : AS -> type(AS);
 JOIN_ON : ON -> type(ON), popMode, pushMode(EXPRESSION_MODE);
 USING : 'USING' -> popMode, pushMode(EXPRESSION_MODE);