Integrate with fangs UnionAll/Fork solution for branched views

Author: craigtaverner

server/src/main/resources/transport/definitions/referable/esql_views.csv

@@ -1 +1 @@
-9177000
+9183000

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java

@@ -35,7 +35,9 @@ public abstract class Attribute extends NamedExpression {
     /**
      * Changing this will break bwc with 8.15, see {@link FieldAttribute#fieldName()}.
      */
-    protected static final String SYNTHETIC_ATTRIBUTE_NAME_PREFIX = "$$";
+    public static final String SYNTHETIC_ATTRIBUTE_NAME_PREFIX = "$$";
+
+    public static final String SYNTHETIC_ATTRIBUTE_NAME_SEPARATOR = "$";
 
     private static final TransportVersion ESQL_QUALIFIERS_IN_ATTRIBUTES = TransportVersion.fromName("esql_qualifiers_in_attributes");
 
@@ -77,7 +79,7 @@ public Attribute(
     }
 
     public static String rawTemporaryName(String... parts) {
-        var name = String.join("$", parts);
+        var name = String.join(SYNTHETIC_ATTRIBUTE_NAME_SEPARATOR, parts);
         return name.isEmpty() || name.startsWith(SYNTHETIC_ATTRIBUTE_NAME_PREFIX) ? name : SYNTHETIC_ATTRIBUTE_NAME_PREFIX + name;
     }
 

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java

@@ -14,6 +14,7 @@
 import java.util.List;
 
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.FORK_V9;
+import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.UNMAPPED_FIELDS;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.VIEWS_V1;
 import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.hasCapabilities;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/views.csv-spec

@@ -8,7 +8,7 @@ ignoreOrder:true
 count:long | country:keyword
 1          | Japan
 1          | Netherlands
-1          | United States of America
+1          | United States
 ;
 
 airports
@@ -63,6 +63,26 @@ count:long | country:keyword
 11         | Nigeria
 ;
 
+airportsCountriesFiltered
+required_capability: views_v1
+
+FROM airports_countries
+| WHERE count > 15
+;
+ignoreOrder:true
+
+count:long | country:keyword
+129        | United States
+50         | India
+45         | Mexico
+41         | China
+37         | Canada
+31         | Brazil
+26         | Russia
+17         | Australia
+17         | United Kingdom
+;
+
 countriesAirportsAddresses
 required_capability: views_v1
 
@@ -87,25 +107,39 @@ count:long | country:keyword
 12         | Indonesia
 ;
 
+countriesAirportsAddressesUS
+required_capability: views_v1
+
+FROM address_countries, airports_countries
+| WHERE country == "United States"
+;
+ignoreOrder:true
+
+count:long | country:keyword
+130        | United States
+1          | United States
+;
+
 languages
 required_capability: views_v1
 
 FROM languages_lookup_non_unique_key
 | MV_EXPAND country
 | WHERE country IS NOT NULL AND country NOT LIKE ("*-Land*")
+| EVAL country = CASE(country == "United States of America", "United States", country)
 | STATS count=COUNT() BY country
 | SORT count DESC, country ASC
 ;
 ignoreOrder:true
 
-count:long | country:text
+count:long | country:keyword
 1          | Atlantis
 1          | Austria
 1          | Canada
 1          | Germany
 1          | Switzerland
 1          | United Kingdom
-1          | United States of America
+1          | United States
 ;
 
 languagesCountries
@@ -115,12 +149,12 @@ FROM languages_countries
 ;
 ignoreOrder:true
 
-count:long | country:text
+count:long | country:keyword
 1          | Atlantis
 1          | Austria
 1          | Canada
 1          | Germany
 1          | Switzerland
 1          | United Kingdom
-1          | United States of America
+1          | United States
 ;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/views/address_countries.esql

@@ -1,3 +1,4 @@
 FROM addresses
 | RENAME city.country.name AS country
+| EVAL country = CASE(country == "United States of America", "United States", country)
 | STATS count=COUNT() BY country

x-pack/plugin/esql/qa/testFixtures/src/main/resources/views/languages_countries.esql

@@ -1,4 +1,5 @@
 FROM languages_lookup_non_unique_key
 | MV_EXPAND country
 | WHERE country IS NOT NULL AND country NOT LIKE ("*-Land*")
+| EVAL country = CASE(country == "United States of America", "United States", country)
 | STATS count=COUNT() BY country

x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4

@@ -108,21 +108,8 @@ timeSeriesCommand
     : TS indexPatternAndMetadataFields
     ;
 
-indexPatternAndMetadataFields
-    : indexPatternOrSubquery (COMMA indexPatternOrSubquery)* metadata?
-    ;
-
-indexPatternOrSubquery
-    : indexPattern
-    | {this.isDevVersion()}? subquery
-    ;
-
-subquery
-    : LP fromCommand (PIPE subqueryProcessingCommand)* RP
-    ;
-
-subqueryProcessingCommand
-    : processingCommand
+indexPatternAndMetadataFields:
+    indexPattern (COMMA indexPattern)* metadata?
     ;
 
 indexPattern

x-pack/plugin/esql/src/main/antlr/lexer/From.g4

@@ -23,11 +23,7 @@ FROM_ASSIGN : ASSIGN -> type(ASSIGN);
 METADATA : 'metadata';
 
 // we need this for EXPLAIN
-// change to double popMode to accommodate subquerys in FROM, when see ')' pop out of subquery(default) mode and from mode
-FROM_RP : RP -> type(RP), popMode, popMode;
-
-// accommodate subQuery inside FROM
-FROM_LP : LP -> type(LP), pushMode(DEFAULT_MODE);
+FROM_RP : RP -> type(RP), popMode;
 
 // in 8.14 ` were not allowed
 // this has been relaxed in 8.15 since " is used for quoting

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java

@@ -250,19 +250,9 @@ private static class ResolveTable extends ParameterizedAnalyzerRule<UnresolvedRe
 
         @Override
         protected LogicalPlan rule(UnresolvedRelation plan, AnalyzerContext context) {
-            String indexPattern = plan.indexPattern().indexPattern();
-            IndexResolution indexResolution;
-            if (plan.indexMode().equals(IndexMode.LOOKUP)) { // index on the RHS of lookup join
-                indexResolution = context.lookupResolution().get(indexPattern);
-            } else { // main index or index in subquery
-                indexResolution = context.indexResolution();
-                if (indexPattern != null
-                    && indexResolution.matches(indexPattern) == false
-                    && context.subqueryResolution().isEmpty() == false) {
-                    // index pattern does not match main index
-                    indexResolution = context.subqueryResolution().getOrDefault(indexPattern, indexResolution);
-                }
-            }
+            IndexResolution indexResolution = plan.indexMode().equals(IndexMode.LOOKUP)
+                ? context.lookupResolution().get(plan.indexPattern().indexPattern())
+                : context.indexResolution().get(plan.indexPattern());
             return resolveIndex(plan, indexResolution);
         }
 
@@ -548,7 +538,7 @@ protected LogicalPlan rule(LogicalPlan plan, AnalyzerContext context) {
             }
 
             if (plan instanceof Insist i) {
-                return resolveInsist(i, childrenOutput, context.indexResolution());
+                return resolveInsist(i, childrenOutput, context.indexResolution().values());
             }
 
             if (plan instanceof Fuse fuse) {
@@ -979,23 +969,29 @@ private List<Attribute> resolveUsingColumns(List<Attribute> cols, List<Attribute
             return resolved;
         }
 
-        private LogicalPlan resolveInsist(Insist insist, List<Attribute> childrenOutput, IndexResolution indexResolution) {
+        private LogicalPlan resolveInsist(Insist insist, List<Attribute> childrenOutput, Collection<IndexResolution> indexResolution) {
             List<Attribute> list = new ArrayList<>();
             for (Attribute a : insist.insistedAttributes()) {
                 list.add(resolveInsistAttribute(a, childrenOutput, indexResolution));
             }
             return insist.withAttributes(list);
         }
 
-        private Attribute resolveInsistAttribute(Attribute attribute, List<Attribute> childrenOutput, IndexResolution indexResolution) {
+        private Attribute resolveInsistAttribute(
+            Attribute attribute,
+            List<Attribute> childrenOutput,
+            Collection<IndexResolution> indexResolution
+        ) {
             Attribute resolvedCol = maybeResolveAttribute((UnresolvedAttribute) attribute, childrenOutput);
             // Field isn't mapped anywhere.
             if (resolvedCol instanceof UnresolvedAttribute) {
                 return insistKeyword(attribute);
             }
 
             // Field is partially unmapped.
-            if (resolvedCol instanceof FieldAttribute fa && indexResolution.get().isPartiallyUnmappedField(fa.name())) {
+            // TODO: Should the check for partially unmapped fields be done specific to each sub-query in a fork?
+            if (resolvedCol instanceof FieldAttribute fa
+                && indexResolution.stream().anyMatch(r -> r.get().isPartiallyUnmappedField(fa.name()))) {
                 return fa.dataType() == KEYWORD ? insistKeyword(fa) : invalidInsistAttribute(fa);
             }
 

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerContext.java

@@ -10,39 +10,28 @@
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
 import org.elasticsearch.xpack.esql.index.IndexResolution;
 import org.elasticsearch.xpack.esql.inference.InferenceResolution;
+import org.elasticsearch.xpack.esql.plan.IndexPattern;
 import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.util.Map;
 
 public record AnalyzerContext(
     Configuration configuration,
     EsqlFunctionRegistry functionRegistry,
-    IndexResolution indexResolution,
+    Map<IndexPattern, IndexResolution> indexResolution,
     Map<String, IndexResolution> lookupResolution,
     EnrichResolution enrichResolution,
-    InferenceResolution inferenceResolution,
-    Map<String, IndexResolution> subqueryResolution
+    InferenceResolution inferenceResolution
 ) {
     // Currently for tests only, since most do not test lookups
     // TODO: make this even simpler, remove the enrichResolution for tests that do not require it (most tests)
     public AnalyzerContext(
         Configuration configuration,
         EsqlFunctionRegistry functionRegistry,
-        IndexResolution indexResolution,
+        Map<IndexPattern, IndexResolution> indexResolution,
         EnrichResolution enrichResolution,
         InferenceResolution inferenceResolution
     ) {
-        this(configuration, functionRegistry, indexResolution, Map.of(), enrichResolution, inferenceResolution, Map.of());
-    }
-
-    public AnalyzerContext(
-        Configuration configuration,
-        EsqlFunctionRegistry functionRegistry,
-        IndexResolution indexResolution,
-        Map<String, IndexResolution> lookupResolution,
-        EnrichResolution enrichResolution,
-        InferenceResolution inferenceResolution
-    ) {
-        this(configuration, functionRegistry, indexResolution, lookupResolution, enrichResolution, inferenceResolution, Map.of());
+        this(configuration, functionRegistry, indexResolution, Map.of(), enrichResolution, inferenceResolution);
     }
 }