ESQL load all dimensions when doing a Time Series query

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java

@@ -161,6 +161,7 @@ static Map<String, IndexFieldCapabilities> retrieveFieldCaps(
         boolean includeEmptyFields
     ) {
         boolean includeParentObjects = checkIncludeParents(filters);
+        boolean includeDimensions = checkIncludeDimensions(filters);
 
         Predicate<MappedFieldType> filter = buildFilter(filters, types, context);
         boolean isTimeSeriesIndex = context.getIndexSettings().getTimestampBounds() != null;
@@ -169,10 +170,10 @@ static Map<String, IndexFieldCapabilities> retrieveFieldCaps(
         Map<String, IndexFieldCapabilities> responseMap = new HashMap<>();
         for (Map.Entry<String, MappedFieldType> entry : context.getAllFields()) {
             final String field = entry.getKey();
-            if (fieldNameFilter.test(field) == false) {
+            MappedFieldType ft = entry.getValue();
+            if (fieldNameFilter.test(field) == false && ((ft.isDimension() && includeDimensions) == false)) {
                 continue;
             }
-            MappedFieldType ft = entry.getValue();
             if ((includeEmptyFields || ft.fieldHasValue(fieldInfos))
                 && (fieldPredicate.test(ft.name()) || context.isMetadataField(ft.name()))
                 && (filter == null || filter.test(ft))) {
@@ -234,6 +235,15 @@ private static boolean checkIncludeParents(String[] filters) {
         return true;
     }
 
+    private static boolean checkIncludeDimensions(String[] filters) {
+        for (String filter : filters) {
+            if ("+dimension".equals(filter)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     private static boolean canMatchShard(
         ShardId shardId,
         QueryBuilder indexFilter,
@@ -267,7 +277,8 @@ private static Predicate<MappedFieldType> buildFilter(String[] filters, String[]
         }
 
         for (String filter : filters) {
-            if ("parent".equals(filter) || "-parent".equals(filter)) {
+            // These "filters" are handled differently, in that they are not ANDed with the field name pattern
+            if ("parent".equals(filter) || "-parent".equals(filter) || "+dimension".equals(filter)) {
                 continue;
             }
             Predicate<MappedFieldType> next = switch (filter) {

server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFilterTests.java

@@ -11,6 +11,7 @@
 
 import org.apache.lucene.index.FieldInfos;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperServiceTestCase;
 import org.elasticsearch.index.query.SearchExecutionContext;
@@ -97,6 +98,54 @@ public void testMetadataFilters() throws IOException {
         }
     }
 
+    public void testDimensionFilters() throws IOException {
+        MapperService mapperService = createMapperService(
+            Settings.builder().put("index.mode", "time_series").put("index.routing_path", "dim.*").build(),
+            """
+                { "_doc" : {
+                  "properties" : {
+                    "metric" : { "type" : "long" },
+                    "dimension_1" : { "type" : "keyword", "time_series_dimension" : "true" },
+                    "dimension_2" : { "type" : "long", "time_series_dimension" : "true" }
+                  }
+                } }
+                """
+        );
+        SearchExecutionContext sec = createSearchExecutionContext(mapperService);
+
+        {
+            // First, test without the filter
+            Map<String, IndexFieldCapabilities> response = FieldCapabilitiesFetcher.retrieveFieldCaps(
+                sec,
+                s -> s.equals("metric"),
+                Strings.EMPTY_ARRAY,
+                Strings.EMPTY_ARRAY,
+                FieldPredicate.ACCEPT_ALL,
+                getMockIndexShard(),
+                true
+            );
+            assertNotNull(response.get("metric"));
+            assertNull(response.get("dimension_1"));
+            assertNull(response.get("dimension_2"));
+        }
+
+        {
+            // then, test with the filter
+            Map<String, IndexFieldCapabilities> response = FieldCapabilitiesFetcher.retrieveFieldCaps(
+                sec,
+                s -> s.equals("metric"),
+                new String[] { "+dimension" },
+                Strings.EMPTY_ARRAY,
+                FieldPredicate.ACCEPT_ALL,
+                getMockIndexShard(),
+                true
+            );
+            assertNotNull(response.get("dimension_1"));
+            assertNotNull(response.get("dimension_2"));
+            assertNotNull(response.get("metric"));
+        }
+    }
+
     public void testExcludeMultifields() throws IOException {
         MapperService mapperService = createMapperService("""
             { "_doc" : {

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java

@@ -449,7 +449,7 @@ public void messageReceived(LookupRequest request, TransportChannel channel, Tas
                             } else {
                                 failures.put(policyName, indexResult.toString());
                             }
-                        }));
+                        }), false);
                     }
                 }
             }

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedRelation.java

@@ -166,4 +166,12 @@ public List<Object> nodeProperties() {
     public String toString() {
         return UNRESOLVED_PREFIX + indexPattern.indexPattern();
     }
+
+    /**
+     * @return true if and only if this relation is being loaded in "time series mode",
+     *         which changes a number of behaviors in the planner.
+     */
+    public boolean isTimeSeriesMode() {
+        return indexMode == IndexMode.TIME_SERIES;
+    }
 }

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java

@@ -423,7 +423,8 @@ private void preAnalyzeLookupIndex(
             patternWithRemotes,
             fieldNames,
             null,
-            listener.map(indexResolution -> receiveLookupIndexResolution(result, localPattern, executionInfo, indexResolution))
+            listener.map(indexResolution -> receiveLookupIndexResolution(result, localPattern, executionInfo, indexResolution)),
+            false
         );
     }
 
@@ -641,7 +642,8 @@ private void preAnalyzeMainIndices(
                     requestFilter,
                     listener.delegateFailure((l, indexResolution) -> {
                         l.onResponse(result.withIndexResolution(indexResolution));
-                    })
+                    }),
+                    false
                 );
             }
         } else {
@@ -782,11 +784,17 @@ public record PreAnalysisResult(
         EnrichResolution enrichResolution,
         Set<String> fieldNames,
         Set<String> wildcardJoinIndices,
-        InferenceResolution inferenceResolution
+        InferenceResolution inferenceResolution,
+        boolean collectAllDimensions
     ) {
 
-        public PreAnalysisResult(EnrichResolution enrichResolution, Set<String> fieldNames, Set<String> wildcardJoinIndices) {
-            this(null, new HashMap<>(), enrichResolution, fieldNames, wildcardJoinIndices, InferenceResolution.EMPTY);
+        public PreAnalysisResult(
+            EnrichResolution enrichResolution,
+            Set<String> fieldNames,
+            Set<String> wildcardJoinIndices,
+            boolean collectAllDimensions
+        ) {
+            this(null, new HashMap<>(), enrichResolution, fieldNames, wildcardJoinIndices, InferenceResolution.EMPTY, collectAllDimensions);
         }
 
         PreAnalysisResult withInferenceResolution(InferenceResolution newInferenceResolution) {
@@ -796,7 +804,8 @@ PreAnalysisResult withInferenceResolution(InferenceResolution newInferenceResolu
                 enrichResolution(),
                 fieldNames(),
                 wildcardJoinIndices(),
-                newInferenceResolution
+                newInferenceResolution,
+                collectAllDimensions()
             );
         }
 
@@ -807,7 +816,8 @@ PreAnalysisResult withIndexResolution(IndexResolution newIndexResolution) {
                 enrichResolution(),
                 fieldNames(),
                 wildcardJoinIndices(),
-                inferenceResolution()
+                inferenceResolution(),
+                collectAllDimensions()
             );
         }
 

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java

@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.esql.session;
 
 import org.elasticsearch.common.regex.Regex;
-import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
@@ -59,22 +58,31 @@ public class FieldNameUtils {
     public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, EnrichResolution enrichResolution) {
 
         // we need the match_fields names from enrich policies and THEN, with an updated list of fields, we call field_caps API
-        var enrichPolicyMatchFields = enrichResolution.resolvedEnrichPolicies()
+        Set<String> enrichPolicyMatchFields = enrichResolution.resolvedEnrichPolicies()
             .stream()
             .map(ResolvedEnrichPolicy::matchField)
             .collect(Collectors.toSet());
 
         // get the field names from the parsed plan combined with the ENRICH match fields from the ENRICH policy
         List<LogicalPlan> inlinestats = parsed.collect(InlineStats.class::isInstance);
         Set<Aggregate> inlinestatsAggs = new HashSet<>();
-        for (var i : inlinestats) {
+        for (LogicalPlan i : inlinestats) {
             inlinestatsAggs.add(((InlineStats) i).aggregate());
         }
 
+        boolean shouldCollectAllDimensions = false;
+        // Detect if we are in TS mode
+        List<LogicalPlan> relations = parsed.collect(UnresolvedRelation.class::isInstance);
+        for (LogicalPlan i : relations) {
+            if (((UnresolvedRelation) i).isTimeSeriesMode()) {
+                shouldCollectAllDimensions = true;
+            }
+        }
+
         if (false == parsed.anyMatch(p -> shouldCollectReferencedFields(p, inlinestatsAggs))) {
             // no explicit columns selection, for example "from employees"
             // also, inlinestats only adds columns to the existent output, its Aggregate shouldn't interfere with potentially using "*"
-            return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of());
+            return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of(), shouldCollectAllDimensions);
         }
 
         Holder<Boolean> projectAll = new Holder<>(false);
@@ -86,7 +94,7 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, EnrichReso
         });
 
         if (projectAll.get()) {
-            return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of());
+            return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of(), shouldCollectAllDimensions);
         }
 
         var referencesBuilder = new Holder<>(AttributeSet.builder());
@@ -162,7 +170,7 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, EnrichReso
                 }
             } else {
                 referencesBuilder.get().addAll(p.references());
-                if (p instanceof UnresolvedRelation ur && ur.indexMode() == IndexMode.TIME_SERIES) {
+                if (p instanceof UnresolvedRelation ur && ur.isTimeSeriesMode()) {
                     // METRICS aggs generally rely on @timestamp without the user having to mention it.
                     referencesBuilder.get().add(new UnresolvedAttribute(ur.source(), MetadataAttribute.TIMESTAMP_FIELD));
                 }
@@ -221,7 +229,7 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, EnrichReso
         parsed.forEachDownMayReturnEarly(forEachDownProcessor.get());
 
         if (projectAll.get()) {
-            return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of());
+            return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of(), shouldCollectAllDimensions);
         }
 
         // Add JOIN ON column references afterward to avoid Alias removal
@@ -235,12 +243,17 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, EnrichReso
 
         if (fieldNames.isEmpty() && enrichPolicyMatchFields.isEmpty()) {
             // there cannot be an empty list of fields, we'll ask the simplest and lightest one instead: _index
-            return new PreAnalysisResult(enrichResolution, IndexResolver.INDEX_METADATA_FIELD, wildcardJoinIndices);
+            return new PreAnalysisResult(
+                enrichResolution,
+                IndexResolver.INDEX_METADATA_FIELD,
+                wildcardJoinIndices,
+                shouldCollectAllDimensions
+            );
         } else {
             fieldNames.addAll(subfields(fieldNames));
             fieldNames.addAll(enrichPolicyMatchFields);
             fieldNames.addAll(subfields(enrichPolicyMatchFields));
-            return new PreAnalysisResult(enrichResolution, fieldNames, wildcardJoinIndices);
+            return new PreAnalysisResult(enrichResolution, fieldNames, wildcardJoinIndices, shouldCollectAllDimensions);
         }
     }
 

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java

@@ -81,11 +81,12 @@ public void resolveAsMergedMapping(
         String indexWildcard,
         Set<String> fieldNames,
         QueryBuilder requestFilter,
-        ActionListener<IndexResolution> listener
+        ActionListener<IndexResolution> listener,
+        boolean includeAllDimensions
     ) {
         client.execute(
             EsqlResolveFieldsAction.TYPE,
-            createFieldCapsRequest(indexWildcard, fieldNames, requestFilter),
+            createFieldCapsRequest(indexWildcard, fieldNames, requestFilter, includeAllDimensions),
             listener.delegateFailureAndWrap((l, response) -> l.onResponse(mergedMappings(indexWildcard, response)))
         );
     }
@@ -273,7 +274,12 @@ private static EsField conflictingMetricTypes(String name, String fullName, Fiel
         return new InvalidMappedField(name, "mapped as different metric types in indices: " + indices);
     }
 
-    private static FieldCapabilitiesRequest createFieldCapsRequest(String index, Set<String> fieldNames, QueryBuilder requestFilter) {
+    private static FieldCapabilitiesRequest createFieldCapsRequest(
+        String index,
+        Set<String> fieldNames,
+        QueryBuilder requestFilter,
+        boolean includeAllDimensions
+    ) {
         FieldCapabilitiesRequest req = new FieldCapabilitiesRequest().indices(Strings.commaDelimitedListToStringArray(index));
         req.fields(fieldNames.toArray(String[]::new));
         req.includeUnmapped(true);
@@ -282,7 +288,11 @@ private static FieldCapabilitiesRequest createFieldCapsRequest(String index, Set
         // also because this way security doesn't throw authorization exceptions but rather honors ignore_unavailable
         req.indicesOptions(FIELD_CAPS_INDICES_OPTIONS);
         // we ignore the nested data type fields starting with https://github.com/elastic/elasticsearch/pull/111495
-        req.filters("-nested");
+        if (includeAllDimensions) {
+            req.filters("-nested", "+dimensions");
+        } else {
+            req.filters("-nested");
+        }
         req.setMergeResults(false);
         return req;
     }

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/FieldNameUtilsTests.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.xpack.esql.parser.ParsingException;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -1612,7 +1613,7 @@ public void testMetrics() {
             assertThat(e.getMessage(), containsString("line 1:1: mismatched input 'TS' expecting {"));
             return;
         }
-        assertFieldNames(
+        assertTsFieldNames(
             query,
             Set.of(
                 "@timestamp",
@@ -2995,9 +2996,27 @@ private void assertFieldNames(String query, Set<String> expected, Set<String> wi
     }
 
     private void assertFieldNames(String query, EnrichResolution enrichResolution, Set<String> expected, Set<String> wildCardIndices) {
-        var preAnalysisResult = FieldNameUtils.resolveFieldNames(parser.createStatement(query, EsqlTestUtils.TEST_CFG), enrichResolution);
+        EsqlSession.PreAnalysisResult preAnalysisResult = FieldNameUtils.resolveFieldNames(
+            parser.createStatement(query, EsqlTestUtils.TEST_CFG),
+            enrichResolution
+        );
         assertThat("Query-wide field names", preAnalysisResult.fieldNames(), equalTo(expected));
         assertThat("Lookup Indices that expect wildcard lookups", preAnalysisResult.wildcardJoinIndices(), equalTo(wildCardIndices));
+        assertThat(preAnalysisResult.collectAllDimensions(), equalTo(false));
+    }
+
+    private void assertTsFieldNames(String query, Set<String> expected) {
+        // Expected may be unmodifiable
+        Set<String> tsExpected = new HashSet<>(expected);
+        tsExpected.add("@timestamp");
+        tsExpected.add("@timestamp.*");
+
+        EsqlSession.PreAnalysisResult preAnalysisResult = FieldNameUtils.resolveFieldNames(
+            parser.createStatement(query, EsqlTestUtils.TEST_CFG),
+            new EnrichResolution()
+        );
+        assertThat("Query-wide field names", preAnalysisResult.fieldNames(), equalTo(tsExpected));
+        assertThat("TS mode query should collect all dimensions", preAnalysisResult.collectAllDimensions(), equalTo(true));
     }
 
     private static EnrichResolution enrichResolutionWith(String enrichPolicyMatchField) {