Esql/lookup join grammar (elastic#116515) (elastic#117107)

First PR for adding LOOKUP JOIN in ESQL.
Introduces grammar and wires the main building blocks to execute a query
 ; follow-ups are required (see elastic#116208 for more details).

Co-authored-by: Nik Everett <[email protected]>
(cherry picked from commit bc785f5)

Author: costin

docs/changelog/116515.yaml

@@ -0,0 +1,5 @@
+pr: 116515
+summary: Esql/lookup join grammar
+area: ES|QL
+type: feature
+issues: []

docs/reference/esql/esql-commands.asciidoc

@@ -38,12 +38,12 @@ image::images/esql/processing-command.svg[A processing command changing an input
 * <<esql-eval>>
 * <<esql-grok>>
 ifeval::["{release-state}"=="unreleased"]
-* experimental:[] <<esql-inlinestats-by>>
+//* experimental:[] <<esql-inlinestats-by>>
 endif::[]
 * <<esql-keep>>
 * <<esql-limit>>
 ifeval::["{release-state}"=="unreleased"]
-* experimental:[] <<esql-lookup>>
+//* experimental:[] <<esql-lookup>>
 endif::[]
 * experimental:[] <<esql-mv_expand>>
 * <<esql-rename>>
@@ -63,12 +63,12 @@ include::processing-commands/enrich.asciidoc[]
 include::processing-commands/eval.asciidoc[]
 include::processing-commands/grok.asciidoc[]
 ifeval::["{release-state}"=="unreleased"]
-include::processing-commands/inlinestats.asciidoc[]
+//include::processing-commands/inlinestats.asciidoc[]
 endif::[]
 include::processing-commands/keep.asciidoc[]
 include::processing-commands/limit.asciidoc[]
 ifeval::["{release-state}"=="unreleased"]
-include::processing-commands/lookup.asciidoc[]
+//include::processing-commands/lookup.asciidoc[]
 endif::[]
 include::processing-commands/mv_expand.asciidoc[]
 include::processing-commands/rename.asciidoc[]

docs/reference/esql/esql-query-api.asciidoc

@@ -92,8 +92,8 @@ https://en.wikipedia.org/wiki/Query_plan[EXPLAIN PLAN].
 
 
 ifeval::["{release-state}"=="unreleased"]
-`table`::
-(Optional, object) Named "table" parameters that can be referenced by the <<esql-lookup>> command.
+//`table`::
+//(Optional, object) Named "table" parameters that can be referenced by the <<esql-lookup>> command.
 endif::[]
 
 [discrete]

docs/reference/esql/processing-commands/inlinestats.asciidoc renamed to docs/reference/esql/processing-commands/inlinestats.disabled

@@ -47,6 +47,7 @@
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V2;
+import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_PLANNING_V1;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METADATA_FIELDS_REMOTE_TEST;
 import static org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase.Mode.SYNC;
@@ -124,6 +125,7 @@ protected void shouldSkipTest(String testName) throws IOException {
         assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS.capabilityName()));
         assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V2.capabilityName()));
         assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_PLANNING_V1.capabilityName()));
+        assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP.capabilityName()));
     }
 
     private TestFeatureService remoteFeaturesService() throws IOException {

docs/reference/esql/processing-commands/lookup.asciidoc renamed to docs/reference/esql/processing-commands/lookup.disabled

@@ -93,6 +93,8 @@ public class CsvTestsDataLoader {
     private static final TestsDataset BOOKS = new TestsDataset("books");
     private static final TestsDataset SEMANTIC_TEXT = new TestsDataset("semantic_text").withInferenceEndpoint(true);
 
+    private static final String LOOKUP_INDEX_SUFFIX = "_lookup";
+
     public static final Map<String, TestsDataset> CSV_DATASET_MAP = Map.ofEntries(
         Map.entry(EMPLOYEES.indexName, EMPLOYEES),
         Map.entry(HOSTS.indexName, HOSTS),
@@ -128,7 +130,9 @@ public class CsvTestsDataLoader {
         Map.entry(DISTANCES.indexName, DISTANCES),
         Map.entry(ADDRESSES.indexName, ADDRESSES),
         Map.entry(BOOKS.indexName, BOOKS),
-        Map.entry(SEMANTIC_TEXT.indexName, SEMANTIC_TEXT)
+        Map.entry(SEMANTIC_TEXT.indexName, SEMANTIC_TEXT),
+        // JOIN LOOKUP alias
+        Map.entry(LANGUAGES.indexName + LOOKUP_INDEX_SUFFIX, LANGUAGES.withIndex(LANGUAGES.indexName + LOOKUP_INDEX_SUFFIX))
     );
 
     private static final EnrichConfig LANGUAGES_ENRICH = new EnrichConfig("languages_policy", "enrich-policy-languages.json");

x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java

@@ -0,0 +1,48 @@
+//
+// CSV spec for LOOKUP JOIN command
+// Reuses the sample dataset and commands from enrich.csv-spec
+//
+
+basicOnTheDataNode
+required_capability: join_lookup
+
+//TODO: this returns different results in CI then locally
+// sometimes null, sometimes spanish (likely related to the execution order)
+FROM employees
+| EVAL language_code = languages
+| LOOKUP JOIN languages_lookup ON language_code
+| WHERE emp_no < 500
+| KEEP emp_no, language_name
+| SORT emp_no
+| LIMIT 1
+;
+
+emp_no:integer | language_name:keyword
+//10091          | Spanish
+;
+
+basicRow-Ignore
+required_capability: join_lookup
+
+ROW language_code = 1
+| LOOKUP JOIN languages_lookup ON language_code
+;
+
+language_code:keyword  | language_name:keyword
+1                      | English
+;
+
+basicOnTheCoordinator
+required_capability: join_lookup
+
+FROM employees
+| SORT emp_no
+| LIMIT 1
+| EVAL language_code = languages
+| LOOKUP JOIN languages_lookup ON language_code
+| KEEP emp_no, language_name
+;
+
+emp_no:integer | language_name:keyword
+10001          | French
+;