non-correlated subquery in from command

Author: fang-xing-esql

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java

@@ -35,7 +35,9 @@ public abstract class Attribute extends NamedExpression {
     /**
      * Changing this will break bwc with 8.15, see {@link FieldAttribute#fieldName()}.
      */
-    protected static final String SYNTHETIC_ATTRIBUTE_NAME_PREFIX = "$$";
+    public static final String SYNTHETIC_ATTRIBUTE_NAME_PREFIX = "$$";
+
+    public static final String SYNTHETIC_ATTRIBUTE_NAME_SEPARATOR = "$";
 
     private static final TransportVersion ESQL_QUALIFIERS_IN_ATTRIBUTES = TransportVersion.fromName("esql_qualifiers_in_attributes");
 
@@ -77,7 +79,7 @@ public Attribute(
     }
 
     public static String rawTemporaryName(String... parts) {
-        var name = String.join("$", parts);
+        var name = String.join(SYNTHETIC_ATTRIBUTE_NAME_SEPARATOR, parts);
         return name.isEmpty() || name.startsWith(SYNTHETIC_ATTRIBUTE_NAME_PREFIX) ? name : SYNTHETIC_ATTRIBUTE_NAME_PREFIX + name;
     }
 

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java

@@ -55,6 +55,11 @@ protected void shouldSkipTest(String testName) throws IOException {
             testCase.requiredCapabilities.contains(UNMAPPED_FIELDS.capabilityName())
         );
 
+        assumeFalse(
+            "Tests using subqueries are skipped since we don't support nested subqueries",
+            testCase.requiredCapabilities.contains(SUBQUERY_IN_FROM_COMMAND.capabilityName())
+        );
+
         assumeTrue("Cluster needs to support FORK", hasCapabilities(adminClient(), List.of(FORK_V9.capabilityName())));
     }
 }

x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec

@@ -108,8 +108,21 @@ timeSeriesCommand
     : TS indexPatternAndMetadataFields
     ;
 
-indexPatternAndMetadataFields:
-    indexPattern (COMMA indexPattern)* metadata?
+indexPatternAndMetadataFields
+    : indexPatternOrSubquery (COMMA indexPatternOrSubquery)* metadata?
+    ;
+
+indexPatternOrSubquery
+    : indexPattern
+    | {this.isDevVersion()}? subquery
+    ;
+
+subquery
+    : LP fromCommand (PIPE subqueryProcessingCommand)* RP
+    ;
+
+subqueryProcessingCommand
+    : processingCommand
     ;
 
 indexPattern

x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4

@@ -23,7 +23,11 @@ FROM_ASSIGN : ASSIGN -> type(ASSIGN);
 METADATA : 'metadata';
 
 // we need this for EXPLAIN
-FROM_RP : RP -> type(RP), popMode;
+// change to double popMode to accommodate subquerys in FROM, when see ')' pop out of subquery(default) mode and from mode
+FROM_RP : RP -> type(RP), popMode, popMode;
+
+// accommodate subQuery inside FROM
+FROM_LP : LP -> type(LP), pushMode(DEFAULT_MODE);
 
 // in 8.14 ` were not allowed
 // this has been relaxed in 8.15 since " is used for quoting

x-pack/plugin/esql/src/main/antlr/lexer/From.g4

@@ -1120,6 +1120,11 @@ public enum Cap {
          */
         FORK_UNION_TYPES,
 
+        /**
+         * Support non-correlated subqueries in the FROM clause.
+         */
+        SUBQUERY_IN_FROM_COMMAND(Build.current().isSnapshot()),
+
         /**
          * Support for the {@code leading_zeros} named parameter.
          */

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

@@ -151,6 +151,10 @@ public boolean includeCCSMetadata() {
         return includeCCSMetadata;
     }
 
+    public Predicate<String> skipOnFailurePredicate() {
+        return skipOnFailurePredicate;
+    }
+
     /**
      * Call when ES|QL "planning" phase is complete and query execution (in ComputeService) is about to start.
      * Note this is currently only built for a single phase planning/execution model. When INLINE STATS

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java

@@ -20,7 +20,8 @@ public record AnalyzerContext(
     IndexResolution indexResolution,
     Map<String, IndexResolution> lookupResolution,
     EnrichResolution enrichResolution,
-    InferenceResolution inferenceResolution
+    InferenceResolution inferenceResolution,
+    Map<String, IndexResolution> subqueryResolution
 ) {
     // Currently for tests only, since most do not test lookups
     // TODO: make this even simpler, remove the enrichResolution for tests that do not require it (most tests)
@@ -31,6 +32,17 @@ public AnalyzerContext(
         EnrichResolution enrichResolution,
         InferenceResolution inferenceResolution
     ) {
-        this(configuration, functionRegistry, indexResolution, Map.of(), enrichResolution, inferenceResolution);
+        this(configuration, functionRegistry, indexResolution, Map.of(), enrichResolution, inferenceResolution, Map.of());
+    }
+
+    public AnalyzerContext(
+        Configuration configuration,
+        EsqlFunctionRegistry functionRegistry,
+        IndexResolution indexResolution,
+        Map<String, IndexResolution> lookupResolution,
+        EnrichResolution enrichResolution,
+        InferenceResolution inferenceResolution
+    ) {
+        this(configuration, functionRegistry, indexResolution, lookupResolution, enrichResolution, inferenceResolution, Map.of());
     }
 }

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java

@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.esql.analysis;
 
 import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
 import org.elasticsearch.xpack.esql.core.util.Holder;
 import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction;
 import org.elasticsearch.xpack.esql.plan.IndexPattern;
@@ -16,7 +17,9 @@
 import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation;
 
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 /**
  * This class is part of the planner.  Acts somewhat like a linker, to find the indices and enrich policies referenced by the query.
@@ -29,9 +32,10 @@ public record PreAnalysis(
         List<Enrich> enriches,
         List<IndexPattern> lookupIndices,
         boolean supportsAggregateMetricDouble,
-        boolean supportsDenseVector
+        boolean supportsDenseVector,
+        Set<IndexPattern> subqueryIndices
     ) {
-        public static final PreAnalysis EMPTY = new PreAnalysis(null, null, List.of(), List.of(), false, false);
+        public static final PreAnalysis EMPTY = new PreAnalysis(null, null, List.of(), List.of(), false, false, Set.of());
     }
 
     public PreAnalysis preAnalyze(LogicalPlan plan) {
@@ -46,12 +50,18 @@ protected PreAnalysis doPreAnalyze(LogicalPlan plan) {
         Holder<IndexMode> indexMode = new Holder<>();
         Holder<IndexPattern> index = new Holder<>();
         List<IndexPattern> lookupIndices = new ArrayList<>();
+        Set<IndexPattern> subqueryIndices = new HashSet<>();
         plan.forEachUp(UnresolvedRelation.class, p -> {
             if (p.indexMode() == IndexMode.LOOKUP) {
                 lookupIndices.add(p.indexPattern());
             } else if (indexMode.get() == null || indexMode.get() == p.indexMode()) {
                 indexMode.set(p.indexMode());
-                index.set(p.indexPattern());
+                // the index pattern from main query is always the first to be seen
+                index.setIfAbsent(p.indexPattern());
+                // collect subquery index patterns
+                if (EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND.isEnabled()) {
+                    collectSubqueryIndexPattern(p, subqueryIndices, index.get());
+                }
             } else {
                 throw new IllegalStateException("index mode is already set");
             }
@@ -96,7 +106,32 @@ protected PreAnalysis doPreAnalyze(LogicalPlan plan) {
             unresolvedEnriches,
             lookupIndices,
             indexMode.get() == IndexMode.TIME_SERIES || supportsAggregateMetricDouble.get(),
-            supportsDenseVector.get()
+            supportsDenseVector.get(),
+            subqueryIndices
         );
     }
+
+    private void collectSubqueryIndexPattern(
+        UnresolvedRelation relation,
+        Set<IndexPattern> subqueryIndices,
+        IndexPattern mainIndexPattern
+    ) {
+        if (relation.preAnalyzed()) {
+            return;
+        }
+
+        IndexPattern pattern = relation.indexPattern();
+        boolean isLookup = relation.indexMode() == IndexMode.LOOKUP;
+        boolean isMainIndexPattern = pattern == mainIndexPattern;
+        /*if the subquery's index pattern is the same as the main query, it won't be added
+        * to the subquery indices set, if Analyzer doesn't find the subquery' indexResolution,
+        * it falls back to the main query's indexResolution
+         */
+        if (isLookup || isMainIndexPattern) {
+            return;
+        }
+        subqueryIndices.add(pattern);
+        System.out.println("collected subquery index pattern: " + pattern);
+        System.out.println("subquery indices now: " + subqueryIndices);
+    }
 }