From e2c2a7ca29a5027843aa9970fd0f4f3676bd417e Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Thu, 31 Jul 2025 14:34:15 -0400 Subject: [PATCH 01/15] add dimension filter to field caps --- .../fieldcaps/FieldCapabilitiesFetcher.java | 17 +++++-- .../FieldCapabilitiesFilterTests.java | 51 +++++++++++++++++++ 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java index dc73be9ed7559..a14788eb6ad00 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java @@ -161,6 +161,7 @@ static Map retrieveFieldCaps( boolean includeEmptyFields ) { boolean includeParentObjects = checkIncludeParents(filters); + boolean includeDimensions = checkIncludeDimensions(filters); Predicate filter = buildFilter(filters, types, context); boolean isTimeSeriesIndex = context.getIndexSettings().getTimestampBounds() != null; @@ -169,10 +170,10 @@ static Map retrieveFieldCaps( Map responseMap = new HashMap<>(); for (Map.Entry entry : context.getAllFields()) { final String field = entry.getKey(); - if (fieldNameFilter.test(field) == false) { + MappedFieldType ft = entry.getValue(); + if (fieldNameFilter.test(field) == false && ((ft.isDimension() && includeDimensions) == false)) { continue; } - MappedFieldType ft = entry.getValue(); if ((includeEmptyFields || ft.fieldHasValue(fieldInfos)) && (fieldPredicate.test(ft.name()) || context.isMetadataField(ft.name())) && (filter == null || filter.test(ft))) { @@ -234,6 +235,15 @@ private static boolean checkIncludeParents(String[] filters) { return true; } + private static boolean checkIncludeDimensions(String[] filters) { + for (String filter : filters) { + if ("+dimension".equals(filter)) { + return true; + } + } + return false; + } + private static boolean canMatchShard( ShardId shardId, QueryBuilder indexFilter, @@ -267,7 +277,8 @@ private static Predicate buildFilter(String[] filters, String[] } for (String filter : filters) { - if ("parent".equals(filter) || "-parent".equals(filter)) { + // These "filters" are handled differently, in that they are not ANDed with the field name pattern + if ("parent".equals(filter) || "-parent".equals(filter) || "+dimension".equals(filter)) { continue; } Predicate next = switch (filter) { diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFilterTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFilterTests.java index abafd93036d86..d986f67f1b7d8 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFilterTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFilterTests.java @@ -11,6 +11,7 @@ import org.apache.lucene.index.FieldInfos; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.index.query.SearchExecutionContext; @@ -97,6 +98,56 @@ public void testMetadataFilters() throws IOException { } } + public void testDimensionFilters() throws IOException { + MapperService mapperService = createMapperService( + Settings.builder().put("index.mode", "time_series").put("index.routing_path", "dim.*").build(), + """ + { "_doc" : { + "properties" : { + "metric" : { "type" : "long" }, + "dimension_1" : { "type" : "keyword", "time_series_dimension" : "true" }, + "dimension_2" : { "type" : "long", "time_series_dimension" : "true" } + } + } } + """ + ); + SearchExecutionContext sec = createSearchExecutionContext(mapperService); + + /* + { + // First, test without the filter + Map response = FieldCapabilitiesFetcher.retrieveFieldCaps( + sec, + s -> s.equals("metric"), + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + FieldPredicate.ACCEPT_ALL, + getMockIndexShard(), + true + ); + assertNotNull(response.get("metric")); + assertNull(response.get("dimension_1")); + assertNull(response.get("dimension_2")); + } + + */ + { + // then, test with the filter + Map response = FieldCapabilitiesFetcher.retrieveFieldCaps( + sec, + s -> s.equals("metric"), + new String[] { "+dimension" }, + Strings.EMPTY_ARRAY, + FieldPredicate.ACCEPT_ALL, + getMockIndexShard(), + true + ); + assertNotNull(response.get("dimension_1")); + assertNotNull(response.get("dimension_2")); + assertNotNull(response.get("metric")); + } + } + public void testExcludeMultifields() throws IOException { MapperService mapperService = createMapperService(""" { "_doc" : { From 33f975cfb4014cb869db34b2299d9cc88c914c55 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 11 Aug 2025 19:46:16 +0000 Subject: [PATCH 02/15] [CI] Auto commit changes from spotless --- .../action/fieldcaps/FieldCapabilitiesFilterTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFilterTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFilterTests.java index d986f67f1b7d8..5c74b324bb442 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFilterTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFilterTests.java @@ -129,7 +129,7 @@ public void testDimensionFilters() throws IOException { assertNull(response.get("dimension_1")); assertNull(response.get("dimension_2")); } - + */ { // then, test with the filter From fb350409832d3e4caf3eae46c46b769427aedeed Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Tue, 12 Aug 2025 12:37:33 -0400 Subject: [PATCH 03/15] add flag for collecting dimensions --- .../xpack/esql/session/EsqlSession.java | 18 +++++++++++++----- .../xpack/esql/session/FieldNameUtils.java | 14 +++++++------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index a98b0f3c52735..a7d5b085e913e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -827,11 +827,17 @@ public record PreAnalysisResult( EnrichResolution enrichResolution, Set fieldNames, Set wildcardJoinIndices, - InferenceResolution inferenceResolution + InferenceResolution inferenceResolution, + boolean collectAllDimensions ) { - public PreAnalysisResult(EnrichResolution enrichResolution, Set fieldNames, Set wildcardJoinIndices) { - this(null, new HashMap<>(), enrichResolution, fieldNames, wildcardJoinIndices, InferenceResolution.EMPTY); + public PreAnalysisResult( + EnrichResolution enrichResolution, + Set fieldNames, + Set wildcardJoinIndices, + boolean collectAllDimensions + ) { + this(null, new HashMap<>(), enrichResolution, fieldNames, wildcardJoinIndices, InferenceResolution.EMPTY, collectAllDimensions); } PreAnalysisResult withInferenceResolution(InferenceResolution newInferenceResolution) { @@ -841,7 +847,8 @@ PreAnalysisResult withInferenceResolution(InferenceResolution newInferenceResolu enrichResolution(), fieldNames(), wildcardJoinIndices(), - newInferenceResolution + newInferenceResolution, + collectAllDimensions() ); } @@ -852,7 +859,8 @@ PreAnalysisResult withIndexResolution(IndexResolution newIndexResolution) { enrichResolution(), fieldNames(), wildcardJoinIndices(), - inferenceResolution() + inferenceResolution(), + collectAllDimensions() ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java index 844f7cb1989df..ece0fa1553330 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java @@ -59,7 +59,7 @@ public class FieldNameUtils { public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, EnrichResolution enrichResolution) { // we need the match_fields names from enrich policies and THEN, with an updated list of fields, we call field_caps API - var enrichPolicyMatchFields = enrichResolution.resolvedEnrichPolicies() + Set enrichPolicyMatchFields = enrichResolution.resolvedEnrichPolicies() .stream() .map(ResolvedEnrichPolicy::matchField) .collect(Collectors.toSet()); @@ -67,14 +67,14 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, EnrichReso // get the field names from the parsed plan combined with the ENRICH match fields from the ENRICH policy List inlinestats = parsed.collect(InlineStats.class::isInstance); Set inlinestatsAggs = new HashSet<>(); - for (var i : inlinestats) { + for (LogicalPlan i : inlinestats) { inlinestatsAggs.add(((InlineStats) i).aggregate()); } if (false == parsed.anyMatch(p -> shouldCollectReferencedFields(p, inlinestatsAggs))) { // no explicit columns selection, for example "from employees" // also, inlinestats only adds columns to the existent output, its Aggregate shouldn't interfere with potentially using "*" - return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of()); + return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of(), false); } Holder projectAll = new Holder<>(false); @@ -86,7 +86,7 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, EnrichReso }); if (projectAll.get()) { - return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of()); + return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of(), false); } var referencesBuilder = new Holder<>(AttributeSet.builder()); @@ -221,7 +221,7 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, EnrichReso parsed.forEachDownMayReturnEarly(forEachDownProcessor.get()); if (projectAll.get()) { - return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of()); + return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of(), false); } // Add JOIN ON column references afterward to avoid Alias removal @@ -235,12 +235,12 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, EnrichReso if (fieldNames.isEmpty() && enrichPolicyMatchFields.isEmpty()) { // there cannot be an empty list of fields, we'll ask the simplest and lightest one instead: _index - return new PreAnalysisResult(enrichResolution, IndexResolver.INDEX_METADATA_FIELD, wildcardJoinIndices); + return new PreAnalysisResult(enrichResolution, IndexResolver.INDEX_METADATA_FIELD, wildcardJoinIndices, false); } else { fieldNames.addAll(subfields(fieldNames)); fieldNames.addAll(enrichPolicyMatchFields); fieldNames.addAll(subfields(enrichPolicyMatchFields)); - return new PreAnalysisResult(enrichResolution, fieldNames, wildcardJoinIndices); + return new PreAnalysisResult(enrichResolution, fieldNames, wildcardJoinIndices, false); } } From 980a99ab93bd225c44bd294937bf41a779a67620 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Tue, 12 Aug 2025 12:52:20 -0400 Subject: [PATCH 04/15] plumb through the flag to fieldcaps --- .../esql/enrich/EnrichPolicyResolver.java | 2 +- .../xpack/esql/session/EsqlSession.java | 6 ++++-- .../xpack/esql/session/IndexResolver.java | 18 ++++++++++++++---- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java index ee68de83ad67e..23d45f1a69909 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java @@ -448,7 +448,7 @@ public void messageReceived(LookupRequest request, TransportChannel channel, Tas } else { failures.put(policyName, indexResult.toString()); } - })); + }), false); } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index a7d5b085e913e..fbb4f9b37f6bd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -468,7 +468,8 @@ private void preAnalyzeLookupIndex( patternWithRemotes, fieldNames, null, - listener.map(indexResolution -> receiveLookupIndexResolution(result, localPattern, executionInfo, indexResolution)) + listener.map(indexResolution -> receiveLookupIndexResolution(result, localPattern, executionInfo, indexResolution)), + false ); } @@ -686,7 +687,8 @@ private void preAnalyzeMainIndices( requestFilter, listener.delegateFailure((l, indexResolution) -> { l.onResponse(result.withIndexResolution(indexResolution)); - }) + }), + false ); } } else { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index d72f3ef0529ad..05dabbb4de6aa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -81,11 +81,12 @@ public void resolveAsMergedMapping( String indexWildcard, Set fieldNames, QueryBuilder requestFilter, - ActionListener listener + ActionListener listener, + boolean includeAllDimensions ) { client.execute( EsqlResolveFieldsAction.TYPE, - createFieldCapsRequest(indexWildcard, fieldNames, requestFilter), + createFieldCapsRequest(indexWildcard, fieldNames, requestFilter, includeAllDimensions), listener.delegateFailureAndWrap((l, response) -> l.onResponse(mergedMappings(indexWildcard, response))) ); } @@ -265,7 +266,12 @@ private static EsField conflictingMetricTypes(String name, String fullName, Fiel return new InvalidMappedField(name, "mapped as different metric types in indices: " + indices); } - private static FieldCapabilitiesRequest createFieldCapsRequest(String index, Set fieldNames, QueryBuilder requestFilter) { + private static FieldCapabilitiesRequest createFieldCapsRequest( + String index, + Set fieldNames, + QueryBuilder requestFilter, + boolean includeAllDimensions + ) { FieldCapabilitiesRequest req = new FieldCapabilitiesRequest().indices(Strings.commaDelimitedListToStringArray(index)); req.fields(fieldNames.toArray(String[]::new)); req.includeUnmapped(true); @@ -274,7 +280,11 @@ private static FieldCapabilitiesRequest createFieldCapsRequest(String index, Set // also because this way security doesn't throw authorization exceptions but rather honors ignore_unavailable req.indicesOptions(FIELD_CAPS_INDICES_OPTIONS); // we ignore the nested data type fields starting with https://github.com/elastic/elasticsearch/pull/111495 - req.filters("-nested"); + if (includeAllDimensions) { + req.filters("-nested", "+dimensions"); + } else { + req.filters("-nested"); + } req.setMergeResults(false); return req; } From 0c11a2ab63cc6e358969952561e14cdf4cef6f60 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Tue, 12 Aug 2025 16:39:28 -0400 Subject: [PATCH 05/15] detect time series mode when resolving field names --- .../esql/plan/logical/UnresolvedRelation.java | 8 +++++++ .../xpack/esql/session/FieldNameUtils.java | 23 +++++++++++++++---- .../esql/session/FieldNameUtilsTests.java | 17 +++++++++++++- 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedRelation.java index 8aaf592e93873..4e0e8ba3adfb7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedRelation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedRelation.java @@ -166,4 +166,12 @@ public List nodeProperties() { public String toString() { return UNRESOLVED_PREFIX + indexPattern.indexPattern(); } + + /** + * @return true if and only if this relation is being loaded in "time series mode", + * which changes a number of behaviors in the planner. + */ + public boolean isTimeSeriesMode() { + return commandName.equalsIgnoreCase("TS"); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java index ece0fa1553330..1ba7604487ecf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java @@ -26,6 +26,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Drop; import org.elasticsearch.xpack.esql.plan.logical.Enrich; +import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Filter; import org.elasticsearch.xpack.esql.plan.logical.Fork; @@ -71,10 +72,19 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, EnrichReso inlinestatsAggs.add(((InlineStats) i).aggregate()); } + boolean shouldCollectAllDimensions = false; + // Detect if we are in TS mode + List relations = parsed.collect(UnresolvedRelation.class::isInstance); + for (LogicalPlan i : relations) { + if (((UnresolvedRelation) i).isTimeSeriesMode()) { + shouldCollectAllDimensions = true; + } + } + if (false == parsed.anyMatch(p -> shouldCollectReferencedFields(p, inlinestatsAggs))) { // no explicit columns selection, for example "from employees" // also, inlinestats only adds columns to the existent output, its Aggregate shouldn't interfere with potentially using "*" - return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of(), false); + return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of(), shouldCollectAllDimensions); } Holder projectAll = new Holder<>(false); @@ -86,7 +96,7 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, EnrichReso }); if (projectAll.get()) { - return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of(), false); + return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of(), shouldCollectAllDimensions); } var referencesBuilder = new Holder<>(AttributeSet.builder()); @@ -221,7 +231,7 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, EnrichReso parsed.forEachDownMayReturnEarly(forEachDownProcessor.get()); if (projectAll.get()) { - return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of(), false); + return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of(), shouldCollectAllDimensions); } // Add JOIN ON column references afterward to avoid Alias removal @@ -235,7 +245,12 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, EnrichReso if (fieldNames.isEmpty() && enrichPolicyMatchFields.isEmpty()) { // there cannot be an empty list of fields, we'll ask the simplest and lightest one instead: _index - return new PreAnalysisResult(enrichResolution, IndexResolver.INDEX_METADATA_FIELD, wildcardJoinIndices, false); + return new PreAnalysisResult( + enrichResolution, + IndexResolver.INDEX_METADATA_FIELD, + wildcardJoinIndices, + shouldCollectAllDimensions + ); } else { fieldNames.addAll(subfields(fieldNames)); fieldNames.addAll(enrichPolicyMatchFields); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/FieldNameUtilsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/FieldNameUtilsTests.java index b6f90beb81a21..02469e60f16c7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/FieldNameUtilsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/FieldNameUtilsTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Enrich; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; @@ -2969,9 +2970,23 @@ private void assertFieldNames(String query, Set expected, Set wi } private void assertFieldNames(String query, EnrichResolution enrichResolution, Set expected, Set wildCardIndices) { - var preAnalysisResult = FieldNameUtils.resolveFieldNames(parser.createStatement(query, EsqlTestUtils.TEST_CFG), enrichResolution); + EsqlSession.PreAnalysisResult preAnalysisResult = FieldNameUtils.resolveFieldNames( + parser.createStatement(query, EsqlTestUtils.TEST_CFG), + enrichResolution + ); + assertThat("Query-wide field names", preAnalysisResult.fieldNames(), equalTo(expected)); + assertThat("Lookup Indices that expect wildcard lookups", preAnalysisResult.wildcardJoinIndices(), equalTo(wildCardIndices)); + assertThat(preAnalysisResult.collectAllDimensions(), equalTo(false)); + + // try again with TS command + String tsQuery = query.toLowerCase(Locale.ROOT).replaceFirst("^from", "ts"); + if (tsQuery.equals(query)) { + return; + } + preAnalysisResult = FieldNameUtils.resolveFieldNames(parser.createStatement(tsQuery, EsqlTestUtils.TEST_CFG), enrichResolution); assertThat("Query-wide field names", preAnalysisResult.fieldNames(), equalTo(expected)); assertThat("Lookup Indices that expect wildcard lookups", preAnalysisResult.wildcardJoinIndices(), equalTo(wildCardIndices)); + assertThat(preAnalysisResult.collectAllDimensions(), equalTo(true)); } private static EnrichResolution enrichResolutionWith(String enrichPolicyMatchField) { From d65202f9a2a93d789d2db50349522ee3389d307a Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 12 Aug 2025 20:45:58 +0000 Subject: [PATCH 06/15] [CI] Auto commit changes from spotless --- .../org/elasticsearch/xpack/esql/session/FieldNameUtils.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java index 1ba7604487ecf..b9695a69535b8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java @@ -26,7 +26,6 @@ import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Drop; import org.elasticsearch.xpack.esql.plan.logical.Enrich; -import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Filter; import org.elasticsearch.xpack.esql.plan.logical.Fork; From 74fa6a84d420d9f98d647c3b0a77de32bc4801a1 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Wed, 13 Aug 2025 15:37:28 -0400 Subject: [PATCH 07/15] fix tests --- .../esql/plan/logical/UnresolvedRelation.java | 2 +- .../xpack/esql/session/FieldNameUtils.java | 2 +- .../esql/session/FieldNameUtilsTests.java | 24 +++++++++++-------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedRelation.java index 4e0e8ba3adfb7..49269ea57f5ac 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedRelation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedRelation.java @@ -172,6 +172,6 @@ public String toString() { * which changes a number of behaviors in the planner. */ public boolean isTimeSeriesMode() { - return commandName.equalsIgnoreCase("TS"); + return indexMode == IndexMode.TIME_SERIES; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java index 1ba7604487ecf..e2df27acce806 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java @@ -172,7 +172,7 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, EnrichReso } } else { referencesBuilder.get().addAll(p.references()); - if (p instanceof UnresolvedRelation ur && ur.indexMode() == IndexMode.TIME_SERIES) { + if (p instanceof UnresolvedRelation ur && ur.isTimeSeriesMode()) { // METRICS aggs generally rely on @timestamp without the user having to mention it. referencesBuilder.get().add(new UnresolvedAttribute(ur.source(), MetadataAttribute.TIMESTAMP_FIELD)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/FieldNameUtilsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/FieldNameUtilsTests.java index 02469e60f16c7..cd2130e18d7df 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/FieldNameUtilsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/FieldNameUtilsTests.java @@ -17,8 +17,8 @@ import org.elasticsearch.xpack.esql.parser.ParsingException; import org.elasticsearch.xpack.esql.plan.logical.Enrich; +import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; @@ -1613,7 +1613,7 @@ public void testMetrics() { assertThat(e.getMessage(), containsString("line 1:1: mismatched input 'TS' expecting {")); return; } - assertFieldNames( + assertTsFieldNames( query, Set.of( "@timestamp", @@ -2977,15 +2977,19 @@ private void assertFieldNames(String query, EnrichResolution enrichResolution, S assertThat("Query-wide field names", preAnalysisResult.fieldNames(), equalTo(expected)); assertThat("Lookup Indices that expect wildcard lookups", preAnalysisResult.wildcardJoinIndices(), equalTo(wildCardIndices)); assertThat(preAnalysisResult.collectAllDimensions(), equalTo(false)); + } - // try again with TS command - String tsQuery = query.toLowerCase(Locale.ROOT).replaceFirst("^from", "ts"); - if (tsQuery.equals(query)) { - return; - } - preAnalysisResult = FieldNameUtils.resolveFieldNames(parser.createStatement(tsQuery, EsqlTestUtils.TEST_CFG), enrichResolution); - assertThat("Query-wide field names", preAnalysisResult.fieldNames(), equalTo(expected)); - assertThat("Lookup Indices that expect wildcard lookups", preAnalysisResult.wildcardJoinIndices(), equalTo(wildCardIndices)); + private void assertTsFieldNames(String query, Set expected) { + // Expected may be unmodifiable + Set tsExpected = new HashSet<>(expected); + tsExpected.add("@timestamp"); + tsExpected.add("@timestamp.*"); + + EsqlSession.PreAnalysisResult preAnalysisResult = FieldNameUtils.resolveFieldNames( + parser.createStatement(query, EsqlTestUtils.TEST_CFG), + new EnrichResolution() + ); + assertThat("Query-wide field names", preAnalysisResult.fieldNames(), equalTo(tsExpected)); assertThat(preAnalysisResult.collectAllDimensions(), equalTo(true)); } From d0dcf7c4d465076d202db716d44f38ad358f3487 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 13 Aug 2025 19:53:12 +0000 Subject: [PATCH 08/15] [CI] Auto commit changes from spotless --- .../org/elasticsearch/xpack/esql/session/FieldNameUtils.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java index f4186ce013327..59b44542395ac 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.esql.session; import org.elasticsearch.common.regex.Regex; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.xpack.esql.analysis.EnrichResolution; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; From 5292071f9861a1b258185958874c3915d1914c9e Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Thu, 14 Aug 2025 09:34:28 -0400 Subject: [PATCH 09/15] missed a spot --- .../org/elasticsearch/xpack/esql/session/FieldNameUtils.java | 2 +- .../elasticsearch/xpack/esql/session/FieldNameUtilsTests.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java index 59b44542395ac..cb9aaee05b9ab 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java @@ -253,7 +253,7 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, EnrichReso fieldNames.addAll(subfields(fieldNames)); fieldNames.addAll(enrichPolicyMatchFields); fieldNames.addAll(subfields(enrichPolicyMatchFields)); - return new PreAnalysisResult(enrichResolution, fieldNames, wildcardJoinIndices, false); + return new PreAnalysisResult(enrichResolution, fieldNames, wildcardJoinIndices, shouldCollectAllDimensions); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/FieldNameUtilsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/FieldNameUtilsTests.java index 41715a4164e48..a8aa5382afa41 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/FieldNameUtilsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/FieldNameUtilsTests.java @@ -3016,7 +3016,7 @@ private void assertTsFieldNames(String query, Set expected) { new EnrichResolution() ); assertThat("Query-wide field names", preAnalysisResult.fieldNames(), equalTo(tsExpected)); - assertThat(preAnalysisResult.collectAllDimensions(), equalTo(true)); + assertThat("TS mode query should collect all dimensions", preAnalysisResult.collectAllDimensions(), equalTo(true)); } private static EnrichResolution enrichResolutionWith(String enrichPolicyMatchField) { From 0c9014e66de2c3bb3c3b8171491b628296f11747 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Thu, 14 Aug 2025 13:11:41 -0400 Subject: [PATCH 10/15] this test shouldn't be commented out --- .../action/fieldcaps/FieldCapabilitiesFilterTests.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFilterTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFilterTests.java index 5c74b324bb442..ced723db6ded1 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFilterTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFilterTests.java @@ -113,7 +113,6 @@ public void testDimensionFilters() throws IOException { ); SearchExecutionContext sec = createSearchExecutionContext(mapperService); - /* { // First, test without the filter Map response = FieldCapabilitiesFetcher.retrieveFieldCaps( @@ -130,7 +129,6 @@ public void testDimensionFilters() throws IOException { assertNull(response.get("dimension_2")); } - */ { // then, test with the filter Map response = FieldCapabilitiesFetcher.retrieveFieldCaps( From dc6c6182c9fa0831548e3ac2f2feabff4d22f23c Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Thu, 21 Aug 2025 10:02:13 -0400 Subject: [PATCH 11/15] response to PR feedback --- .../action/fieldcaps/FieldCapabilitiesFetcher.java | 2 +- .../org/elasticsearch/xpack/esql/session/EsqlSession.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java index a14788eb6ad00..aee7858d84b13 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java @@ -171,7 +171,7 @@ static Map retrieveFieldCaps( for (Map.Entry entry : context.getAllFields()) { final String field = entry.getKey(); MappedFieldType ft = entry.getValue(); - if (fieldNameFilter.test(field) == false && ((ft.isDimension() && includeDimensions) == false)) { + if (fieldNameFilter.test(field) == false && ((includeDimensions && ft.isDimension()) == false)) { continue; } if ((includeEmptyFields || ft.fieldHasValue(fieldInfos)) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index f3ab816144e40..2d065e328d6f7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -626,8 +626,10 @@ private void preAnalyzeMainIndices( result.withIndexResolution(IndexResolution.valid(new EsIndex(table.indexPattern(), Map.of(), Map.of()))) ); } else { + boolean includeAllDimensions = false; // call the EsqlResolveFieldsAction (field-caps) to resolve indices and get field types if (preAnalysis.indexMode == IndexMode.TIME_SERIES) { + includeAllDimensions = true; // TODO: Maybe if no indices are returned, retry without index mode and provide a clearer error message. var indexModeFilter = new TermQueryBuilder(IndexModeFieldMapper.NAME, IndexMode.TIME_SERIES.getName()); if (requestFilter != null) { @@ -643,7 +645,7 @@ private void preAnalyzeMainIndices( listener.delegateFailure((l, indexResolution) -> { l.onResponse(result.withIndexResolution(indexResolution)); }), - false + includeAllDimensions ); } } else { From ada3b48c1960e84abf9f74f957a7414876bda616 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Thu, 21 Aug 2025 10:54:06 -0400 Subject: [PATCH 12/15] remove unnecessary changes to FieldNameUtils --- .../xpack/esql/session/EsqlSession.java | 18 ++++---------- .../xpack/esql/session/FieldNameUtils.java | 24 ++++--------------- .../esql/session/FieldNameUtilsTests.java | 23 ++---------------- 3 files changed, 12 insertions(+), 53 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 2d065e328d6f7..e67b2f8be6aa8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -786,17 +786,11 @@ public record PreAnalysisResult( EnrichResolution enrichResolution, Set fieldNames, Set wildcardJoinIndices, - InferenceResolution inferenceResolution, - boolean collectAllDimensions + InferenceResolution inferenceResolution ) { - public PreAnalysisResult( - EnrichResolution enrichResolution, - Set fieldNames, - Set wildcardJoinIndices, - boolean collectAllDimensions - ) { - this(null, new HashMap<>(), enrichResolution, fieldNames, wildcardJoinIndices, InferenceResolution.EMPTY, collectAllDimensions); + public PreAnalysisResult(EnrichResolution enrichResolution, Set fieldNames, Set wildcardJoinIndices) { + this(null, new HashMap<>(), enrichResolution, fieldNames, wildcardJoinIndices, InferenceResolution.EMPTY); } PreAnalysisResult withInferenceResolution(InferenceResolution newInferenceResolution) { @@ -806,8 +800,7 @@ PreAnalysisResult withInferenceResolution(InferenceResolution newInferenceResolu enrichResolution(), fieldNames(), wildcardJoinIndices(), - newInferenceResolution, - collectAllDimensions() + newInferenceResolution ); } @@ -818,8 +811,7 @@ PreAnalysisResult withIndexResolution(IndexResolution newIndexResolution) { enrichResolution(), fieldNames(), wildcardJoinIndices(), - inferenceResolution(), - collectAllDimensions() + inferenceResolution() ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java index cb9aaee05b9ab..5911e4e64ecc4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java @@ -70,19 +70,10 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, EnrichReso inlinestatsAggs.add(((InlineStats) i).aggregate()); } - boolean shouldCollectAllDimensions = false; - // Detect if we are in TS mode - List relations = parsed.collect(UnresolvedRelation.class::isInstance); - for (LogicalPlan i : relations) { - if (((UnresolvedRelation) i).isTimeSeriesMode()) { - shouldCollectAllDimensions = true; - } - } - if (false == parsed.anyMatch(p -> shouldCollectReferencedFields(p, inlinestatsAggs))) { // no explicit columns selection, for example "from employees" // also, inlinestats only adds columns to the existent output, its Aggregate shouldn't interfere with potentially using "*" - return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of(), shouldCollectAllDimensions); + return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of()); } Holder projectAll = new Holder<>(false); @@ -94,7 +85,7 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, EnrichReso }); if (projectAll.get()) { - return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of(), shouldCollectAllDimensions); + return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of()); } var referencesBuilder = new Holder<>(AttributeSet.builder()); @@ -229,7 +220,7 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, EnrichReso parsed.forEachDownMayReturnEarly(forEachDownProcessor.get()); if (projectAll.get()) { - return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of(), shouldCollectAllDimensions); + return new PreAnalysisResult(enrichResolution, IndexResolver.ALL_FIELDS, Set.of()); } // Add JOIN ON column references afterward to avoid Alias removal @@ -243,17 +234,12 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, EnrichReso if (fieldNames.isEmpty() && enrichPolicyMatchFields.isEmpty()) { // there cannot be an empty list of fields, we'll ask the simplest and lightest one instead: _index - return new PreAnalysisResult( - enrichResolution, - IndexResolver.INDEX_METADATA_FIELD, - wildcardJoinIndices, - shouldCollectAllDimensions - ); + return new PreAnalysisResult(enrichResolution, IndexResolver.INDEX_METADATA_FIELD, wildcardJoinIndices); } else { fieldNames.addAll(subfields(fieldNames)); fieldNames.addAll(enrichPolicyMatchFields); fieldNames.addAll(subfields(enrichPolicyMatchFields)); - return new PreAnalysisResult(enrichResolution, fieldNames, wildcardJoinIndices, shouldCollectAllDimensions); + return new PreAnalysisResult(enrichResolution, fieldNames, wildcardJoinIndices); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/FieldNameUtilsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/FieldNameUtilsTests.java index a8aa5382afa41..58934fe68f8cb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/FieldNameUtilsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/FieldNameUtilsTests.java @@ -17,7 +17,6 @@ import org.elasticsearch.xpack.esql.parser.ParsingException; import org.elasticsearch.xpack.esql.plan.logical.Enrich; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -1613,7 +1612,7 @@ public void testMetrics() { assertThat(e.getMessage(), containsString("line 1:1: mismatched input 'TS' expecting {")); return; } - assertTsFieldNames( + assertFieldNames( query, Set.of( "@timestamp", @@ -2996,27 +2995,9 @@ private void assertFieldNames(String query, Set expected, Set wi } private void assertFieldNames(String query, EnrichResolution enrichResolution, Set expected, Set wildCardIndices) { - EsqlSession.PreAnalysisResult preAnalysisResult = FieldNameUtils.resolveFieldNames( - parser.createStatement(query, EsqlTestUtils.TEST_CFG), - enrichResolution - ); + var preAnalysisResult = FieldNameUtils.resolveFieldNames(parser.createStatement(query, EsqlTestUtils.TEST_CFG), enrichResolution); assertThat("Query-wide field names", preAnalysisResult.fieldNames(), equalTo(expected)); assertThat("Lookup Indices that expect wildcard lookups", preAnalysisResult.wildcardJoinIndices(), equalTo(wildCardIndices)); - assertThat(preAnalysisResult.collectAllDimensions(), equalTo(false)); - } - - private void assertTsFieldNames(String query, Set expected) { - // Expected may be unmodifiable - Set tsExpected = new HashSet<>(expected); - tsExpected.add("@timestamp"); - tsExpected.add("@timestamp.*"); - - EsqlSession.PreAnalysisResult preAnalysisResult = FieldNameUtils.resolveFieldNames( - parser.createStatement(query, EsqlTestUtils.TEST_CFG), - new EnrichResolution() - ); - assertThat("Query-wide field names", preAnalysisResult.fieldNames(), equalTo(tsExpected)); - assertThat("TS mode query should collect all dimensions", preAnalysisResult.collectAllDimensions(), equalTo(true)); } private static EnrichResolution enrichResolutionWith(String enrichPolicyMatchField) { From 9e2946d3481ef3f5e8868d5e8f1bc849da9b503d Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Thu, 21 Aug 2025 11:15:58 -0400 Subject: [PATCH 13/15] spelled the magic word wrong --- .../org/elasticsearch/xpack/esql/session/IndexResolver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index ba9a1fd35009f..dfc0e2489bd3e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -289,7 +289,7 @@ private static FieldCapabilitiesRequest createFieldCapsRequest( req.indicesOptions(FIELD_CAPS_INDICES_OPTIONS); // we ignore the nested data type fields starting with https://github.com/elastic/elasticsearch/pull/111495 if (includeAllDimensions) { - req.filters("-nested", "+dimensions"); + req.filters("-nested", "+dimension"); } else { req.filters("-nested"); } From da094a4d65332976fee7af638168d404bfe4d18c Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Fri, 22 Aug 2025 14:10:39 -0400 Subject: [PATCH 14/15] add appropriate capability check? --- .../rest-api-spec/test/esql/46_downsample.yml | 368 +++++++++--------- 1 file changed, 184 insertions(+), 184 deletions(-) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml index ff01c4a53cc7b..9f256564650a1 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml @@ -78,12 +78,12 @@ setup: --- "Query stats on downsampled index": - requires: - test_runner_features: [capabilities] + test_runner_features: [ capabilities ] capabilities: - method: POST path: /_query - parameters: [] - capabilities: [aggregate_metric_double] + parameters: [ ] + capabilities: [ aggregate_metric_double ] reason: "Support for aggregate_metric_double" - do: indices.downsample: @@ -102,30 +102,30 @@ setup: STATS max(k8s.pod.network.rx), min(k8s.pod.network.rx), sum(k8s.pod.network.rx), count(k8s.pod.network.rx) | LIMIT 100" - - length: {values: 1} - - length: {values.0: 4} - - match: {columns.0.name: "max(k8s.pod.network.rx)"} - - match: {columns.0.type: "double"} - - match: {columns.1.name: "min(k8s.pod.network.rx)"} - - match: {columns.1.type: "double"} - - match: {columns.2.name: "sum(k8s.pod.network.rx)"} - - match: {columns.2.type: "double"} - - match: {columns.3.name: "count(k8s.pod.network.rx)"} - - match: {columns.3.type: "long"} - - match: {values.0.0: 803685.0} - - match: {values.0.1: 530575.0} - - match: {values.0.2: 5332018.0} - - match: {values.0.3: 8} + - length: { values: 1 } + - length: { values.0: 4 } + - match: { columns.0.name: "max(k8s.pod.network.rx)" } + - match: { columns.0.type: "double" } + - match: { columns.1.name: "min(k8s.pod.network.rx)" } + - match: { columns.1.type: "double" } + - match: { columns.2.name: "sum(k8s.pod.network.rx)" } + - match: { columns.2.type: "double" } + - match: { columns.3.name: "count(k8s.pod.network.rx)" } + - match: { columns.3.type: "long" } + - match: { values.0.0: 803685.0 } + - match: { values.0.1: 530575.0 } + - match: { values.0.2: 5332018.0 } + - match: { values.0.3: 8 } --- "Render stats from downsampled index": - requires: - test_runner_features: [capabilities] + test_runner_features: [ capabilities ] capabilities: - method: POST path: /_query - parameters: [] - capabilities: [aggregate_metric_double_rendering] + parameters: [ ] + capabilities: [ aggregate_metric_double_rendering ] reason: "Support for rendering aggregate_metric_doubles" - do: indices.downsample: @@ -141,21 +141,21 @@ setup: esql.query: body: query: "FROM test-downsample | WHERE @timestamp == \"2021-04-28T19:00:00.000Z\" | KEEP k8s.pod.network.rx | LIMIT 100" - - length: {values: 1} - - length: {values.0: 1} - - match: {columns.0.name: "k8s.pod.network.rx"} - - match: {columns.0.type: "aggregate_metric_double"} - - match: {values.0.0: '{"min":530604.0,"max":530605.0,"sum":1061209.0,"value_count":2}'} + - length: { values: 1 } + - length: { values.0: 1 } + - match: { columns.0.name: "k8s.pod.network.rx" } + - match: { columns.0.type: "aggregate_metric_double" } + - match: { values.0.0: '{"min":530604.0,"max":530605.0,"sum":1061209.0,"value_count":2}' } --- "Stats from downsampled and non-downsampled index simultaneously": - requires: - test_runner_features: [capabilities] + test_runner_features: [ capabilities ] capabilities: - method: POST path: /_query - parameters: [] - capabilities: [aggregate_metric_double_convert_to] + parameters: [ ] + capabilities: [ aggregate_metric_double_convert_to ] reason: "Support for to_aggregate_metric_double function" - do: @@ -237,30 +237,30 @@ setup: STATS max(rx), min(rx), sum(rx), count(rx) | LIMIT 100" - - length: {values: 1} - - length: {values.0: 4} - - match: {columns.0.name: "max(rx)"} - - match: {columns.0.type: "double"} - - match: {columns.1.name: "min(rx)"} - - match: {columns.1.type: "double"} - - match: {columns.2.name: "sum(rx)"} - - match: {columns.2.type: "double"} - - match: {columns.3.name: "count(rx)"} - - match: {columns.3.type: "long"} - - match: {values.0.0: 803685.0} - - match: {values.0.1: 800479.0} - - match: {values.0.2: 4812452.0} - - match: {values.0.3: 6} + - length: { values: 1 } + - length: { values.0: 4 } + - match: { columns.0.name: "max(rx)" } + - match: { columns.0.type: "double" } + - match: { columns.1.name: "min(rx)" } + - match: { columns.1.type: "double" } + - match: { columns.2.name: "sum(rx)" } + - match: { columns.2.type: "double" } + - match: { columns.3.name: "count(rx)" } + - match: { columns.3.type: "long" } + - match: { values.0.0: 803685.0 } + - match: { values.0.1: 800479.0 } + - match: { values.0.2: 4812452.0 } + - match: { values.0.3: 6 } --- "Stats from downsampled and non-downsampled index simultaneously with implicit casting": - requires: - test_runner_features: [capabilities] + test_runner_features: [ capabilities ] capabilities: - method: POST path: /_query - parameters: [] - capabilities: [aggregate_metric_double_implicit_casting_in_aggs] + parameters: [ ] + capabilities: [ aggregate_metric_double_implicit_casting_in_aggs ] reason: "Support for casting aggregate metric double implicitly when present in aggregations" - do: @@ -341,43 +341,43 @@ setup: STATS max(k8s.pod.network.rx), min(k8s.pod.network.rx), sum(k8s.pod.network.rx), count(k8s.pod.network.rx), avg(k8s.pod.network.rx) | LIMIT 100" - - length: {values: 1} - - length: {values.0: 5} - - match: {columns.0.name: "max(k8s.pod.network.rx)"} - - match: {columns.0.type: "double"} - - match: {columns.1.name: "min(k8s.pod.network.rx)"} - - match: {columns.1.type: "double"} - - match: {columns.2.name: "sum(k8s.pod.network.rx)"} - - match: {columns.2.type: "double"} - - match: {columns.3.name: "count(k8s.pod.network.rx)"} - - match: {columns.3.type: "long"} - - match: {columns.4.name: "avg(k8s.pod.network.rx)"} - - match: {columns.4.type: "double"} - - match: {values.0.0: 803685.0} - - match: {values.0.1: 800479.0} - - match: {values.0.2: 4812452.0} - - match: {values.0.3: 6} - - match: {values.0.4: 802075.3333333334} + - length: { values: 1 } + - length: { values.0: 5 } + - match: { columns.0.name: "max(k8s.pod.network.rx)" } + - match: { columns.0.type: "double" } + - match: { columns.1.name: "min(k8s.pod.network.rx)" } + - match: { columns.1.type: "double" } + - match: { columns.2.name: "sum(k8s.pod.network.rx)" } + - match: { columns.2.type: "double" } + - match: { columns.3.name: "count(k8s.pod.network.rx)" } + - match: { columns.3.type: "long" } + - match: { columns.4.name: "avg(k8s.pod.network.rx)" } + - match: { columns.4.type: "double" } + - match: { values.0.0: 803685.0 } + - match: { values.0.1: 800479.0 } + - match: { values.0.2: 4812452.0 } + - match: { values.0.3: 6 } + - match: { values.0.4: 802075.3333333334 } - do: esql.query: body: query: "TS test-* | STATS max = max(k8s.pod.network.rx) | LIMIT 100" - - length: {values: 1} - - length: {values.0: 1} - - match: {columns.0.name: "max"} - - match: {columns.0.type: "double"} - - match: {values.0.0: 803685.0} + - length: { values: 1 } + - length: { values.0: 1 } + - match: { columns.0.name: "max" } + - match: { columns.0.type: "double" } + - match: { values.0.0: 803685.0 } --- "Over time functions from downsampled and non-downsampled indices simultaneously, no grouping": - requires: - test_runner_features: [capabilities] + test_runner_features: [ capabilities ] capabilities: - method: POST path: /_query - parameters: [] - capabilities: [aggregate_metric_double_implicit_casting_in_aggs] + parameters: [ ] + capabilities: [ metrics_command, aggregate_metric_double_implicit_casting_in_aggs ] reason: "Support for casting aggregate metric double implicitly when present in aggregations" - do: @@ -464,42 +464,42 @@ setup: BY time_bucket = bucket(@timestamp, 1 hour) | SORT time_bucket | LIMIT 10" - - length: {values: 4} - - length: {values.0: 4} - - match: {columns.0.name: "avg"} - - match: {columns.0.type: "double"} - - match: {columns.1.name: "count"} - - match: {columns.1.type: "long"} - - match: {columns.2.name: "sum"} - - match: {columns.2.type: "double"} - - match: {columns.3.name: "time_bucket"} - - match: {columns.3.type: "date"} - - match: {values.0.0: 1332393.5} - - match: {values.0.1: 4} - - match: {values.0.2: 2664787.0} - - match: {values.0.3: "2021-04-28T18:00:00.000Z"} - - match: {values.1.0: 530604.5} - - match: {values.1.1: 2} - - match: {values.1.2: 1061209.0} - - match: {values.1.3: "2021-04-28T19:00:00.000Z"} - - match: {values.2.0: 803011.0} - - match: {values.2.1: 2} - - match: {values.2.2: 1606022.0} - - match: {values.2.3: "2021-04-28T20:00:00.000Z"} - - match: {values.3.0: 1334011.0} - - match: {values.3.1: 4} - - match: {values.3.2: 2668022.0} - - match: {values.3.3: "2021-04-29T21:00:00.000Z"} + - length: { values: 4 } + - length: { values.0: 4 } + - match: { columns.0.name: "avg" } + - match: { columns.0.type: "double" } + - match: { columns.1.name: "count" } + - match: { columns.1.type: "long" } + - match: { columns.2.name: "sum" } + - match: { columns.2.type: "double" } + - match: { columns.3.name: "time_bucket" } + - match: { columns.3.type: "date" } + - match: { values.0.0: 1332393.5 } + - match: { values.0.1: 4 } + - match: { values.0.2: 2664787.0 } + - match: { values.0.3: "2021-04-28T18:00:00.000Z" } + - match: { values.1.0: 530604.5 } + - match: { values.1.1: 2 } + - match: { values.1.2: 1061209.0 } + - match: { values.1.3: "2021-04-28T19:00:00.000Z" } + - match: { values.2.0: 803011.0 } + - match: { values.2.1: 2 } + - match: { values.2.2: 1606022.0 } + - match: { values.2.3: "2021-04-28T20:00:00.000Z" } + - match: { values.3.0: 1334011.0 } + - match: { values.3.1: 4 } + - match: { values.3.2: 2668022.0 } + - match: { values.3.3: "2021-04-29T21:00:00.000Z" } --- "Over time functions from downsampled and non-downsampled indices simultaneously, with grouping": - requires: - test_runner_features: [capabilities] + test_runner_features: [ capabilities ] capabilities: - method: POST path: /_query - parameters: [] - capabilities: [aggregate_metric_double_implicit_casting_in_aggs] + parameters: [ ] + capabilities: [ metrics_command, aggregate_metric_double_implicit_casting_in_aggs ] reason: "Support for casting aggregate metric double implicitly when present in aggregations" - do: @@ -587,58 +587,58 @@ setup: SORT time_bucket, k8s.pod.name | LIMIT 10" - - length: {values: 6} - - length: {values.0: 5} - - match: {columns.0.name: "avg"} - - match: {columns.0.type: "double"} - - match: {columns.1.name: "count"} - - match: {columns.1.type: "long"} - - match: {columns.2.name: "sum"} - - match: {columns.2.type: "double"} - - match: {columns.3.name: "k8s.pod.name"} - - match: {columns.3.type: "keyword"} - - match: {columns.4.name: "time_bucket"} - - match: {columns.4.type: "date"} - - match: {values.0.0: 801806.0} - - match: {values.0.1: 2} - - match: {values.0.2: 1603612.0} - - match: {values.0.3: "cat"} - - match: {values.0.4: "2021-04-28T18:00:00.000Z"} - - match: {values.1.0: 530587.5} - - match: {values.1.1: 2} - - match: {values.1.2: 1061175.0} - - match: {values.1.3: "dog"} - - match: {values.1.4: "2021-04-28T18:00:00.000Z"} - - match: {values.2.0: 530604.5} - - match: {values.2.1: 2} - - match: {values.2.2: 1061209.0} - - match: {values.2.3: "dog"} - - match: {values.2.4: "2021-04-28T19:00:00.000Z"} - - match: {values.3.0: 803011.0} - - match: {values.3.1: 2} - - match: {values.3.2: 1606022.0} - - match: {values.3.3: "cat"} - - match: {values.3.4: "2021-04-28T20:00:00.000Z"} - - match: {values.4.0: 801409.0} - - match: {values.4.1: 2} - - match: {values.4.2: 1602818.0} - - match: {values.4.3: "cat"} - - match: {values.4.4: "2021-04-29T21:00:00.000Z"} - - match: {values.5.0: 532602.0} - - match: {values.5.1: 2} - - match: {values.5.2: 1065204.0} - - match: {values.5.3: "dog"} - - match: {values.5.4: "2021-04-29T21:00:00.000Z"} + - length: { values: 6 } + - length: { values.0: 5 } + - match: { columns.0.name: "avg" } + - match: { columns.0.type: "double" } + - match: { columns.1.name: "count" } + - match: { columns.1.type: "long" } + - match: { columns.2.name: "sum" } + - match: { columns.2.type: "double" } + - match: { columns.3.name: "k8s.pod.name" } + - match: { columns.3.type: "keyword" } + - match: { columns.4.name: "time_bucket" } + - match: { columns.4.type: "date" } + - match: { values.0.0: 801806.0 } + - match: { values.0.1: 2 } + - match: { values.0.2: 1603612.0 } + - match: { values.0.3: "cat" } + - match: { values.0.4: "2021-04-28T18:00:00.000Z" } + - match: { values.1.0: 530587.5 } + - match: { values.1.1: 2 } + - match: { values.1.2: 1061175.0 } + - match: { values.1.3: "dog" } + - match: { values.1.4: "2021-04-28T18:00:00.000Z" } + - match: { values.2.0: 530604.5 } + - match: { values.2.1: 2 } + - match: { values.2.2: 1061209.0 } + - match: { values.2.3: "dog" } + - match: { values.2.4: "2021-04-28T19:00:00.000Z" } + - match: { values.3.0: 803011.0 } + - match: { values.3.1: 2 } + - match: { values.3.2: 1606022.0 } + - match: { values.3.3: "cat" } + - match: { values.3.4: "2021-04-28T20:00:00.000Z" } + - match: { values.4.0: 801409.0 } + - match: { values.4.1: 2 } + - match: { values.4.2: 1602818.0 } + - match: { values.4.3: "cat" } + - match: { values.4.4: "2021-04-29T21:00:00.000Z" } + - match: { values.5.0: 532602.0 } + - match: { values.5.1: 2 } + - match: { values.5.2: 1065204.0 } + - match: { values.5.3: "dog" } + - match: { values.5.4: "2021-04-29T21:00:00.000Z" } --- "Sort from multiple indices one with aggregate metric double": - requires: - test_runner_features: [capabilities] + test_runner_features: [ capabilities ] capabilities: - method: POST path: /_query - parameters: [] - capabilities: [aggregate_metric_double_sorting_fixed] + parameters: [ ] + capabilities: [ aggregate_metric_double_sorting_fixed ] reason: "Fix sorting for rows comprised of docs from multiple indices where agg metric is missing from some" - do: @@ -673,39 +673,39 @@ setup: body: query: "FROM test-* | SORT some_field, @timestamp, k8s.pod.uid | KEEP k8s.pod.network.rx, some_field, @timestamp | LIMIT 10" - - length: {values: 5} - - length: {values.0: 3} - - match: {columns.0.name: "k8s.pod.network.rx"} - - match: {columns.0.type: "aggregate_metric_double"} - - match: {columns.1.name: "some_field"} - - match: {columns.1.type: "keyword"} - - match: {columns.2.name: "@timestamp"} - - match: {columns.2.type: "date"} - - match: {values.0.0: null} - - match: {values.0.1: "im a keyword!!!!!"} - - match: {values.0.2: null} - - match: {values.1.0: '{"min":801479.0,"max":802133.0,"sum":1603612.0,"value_count":2}'} - - match: {values.1.1: null} - - match: {values.1.2: "2021-04-28T18:00:00.000Z"} - - match: {values.2.0: '{"min":530575.0,"max":530600.0,"sum":1061175.0,"value_count":2}'} - - match: {values.2.1: null} - - match: {values.2.2: "2021-04-28T18:00:00.000Z"} - - match: {values.3.0: '{"min":530604.0,"max":530605.0,"sum":1061209.0,"value_count":2}'} - - match: {values.3.1: null} - - match: {values.3.2: "2021-04-28T19:00:00.000Z"} - - match: {values.4.0: '{"min":802337.0,"max":803685.0,"sum":1606022.0,"value_count":2}'} - - match: {values.4.1: null} - - match: {values.4.2: "2021-04-28T20:00:00.000Z"} + - length: { values: 5 } + - length: { values.0: 3 } + - match: { columns.0.name: "k8s.pod.network.rx" } + - match: { columns.0.type: "aggregate_metric_double" } + - match: { columns.1.name: "some_field" } + - match: { columns.1.type: "keyword" } + - match: { columns.2.name: "@timestamp" } + - match: { columns.2.type: "date" } + - match: { values.0.0: null } + - match: { values.0.1: "im a keyword!!!!!" } + - match: { values.0.2: null } + - match: { values.1.0: '{"min":801479.0,"max":802133.0,"sum":1603612.0,"value_count":2}' } + - match: { values.1.1: null } + - match: { values.1.2: "2021-04-28T18:00:00.000Z" } + - match: { values.2.0: '{"min":530575.0,"max":530600.0,"sum":1061175.0,"value_count":2}' } + - match: { values.2.1: null } + - match: { values.2.2: "2021-04-28T18:00:00.000Z" } + - match: { values.3.0: '{"min":530604.0,"max":530605.0,"sum":1061209.0,"value_count":2}' } + - match: { values.3.1: null } + - match: { values.3.2: "2021-04-28T19:00:00.000Z" } + - match: { values.4.0: '{"min":802337.0,"max":803685.0,"sum":1606022.0,"value_count":2}' } + - match: { values.4.1: null } + - match: { values.4.2: "2021-04-28T20:00:00.000Z" } --- "MV_EXPAND on non-MV aggregate metric double": - requires: - test_runner_features: [capabilities] + test_runner_features: [ capabilities ] capabilities: - method: POST path: /_query - parameters: [] - capabilities: [aggregate_metric_double_mv_expand] + parameters: [ ] + capabilities: [ aggregate_metric_double_mv_expand ] reason: "Have MV_EXPAND not error out when applied to aggregate_metric_doubles (is a no-op)" - do: @@ -723,17 +723,17 @@ setup: body: query: "FROM test-downsample | MV_EXPAND k8s.pod.network.rx | SORT @timestamp, k8s.pod.uid | KEEP k8s.pod.network.rx, @timestamp | LIMIT 10" - - length: {values: 4} - - length: {values.0: 2} - - match: {columns.0.name: "k8s.pod.network.rx"} - - match: {columns.0.type: "aggregate_metric_double"} - - match: {columns.1.name: "@timestamp"} - - match: {columns.1.type: "date"} - - match: {values.0.0: '{"min":801479.0,"max":802133.0,"sum":1603612.0,"value_count":2}'} - - match: {values.0.1: "2021-04-28T18:00:00.000Z"} - - match: {values.1.0: '{"min":530575.0,"max":530600.0,"sum":1061175.0,"value_count":2}'} - - match: {values.1.1: "2021-04-28T18:00:00.000Z"} - - match: {values.2.0: '{"min":530604.0,"max":530605.0,"sum":1061209.0,"value_count":2}'} - - match: {values.2.1: "2021-04-28T19:00:00.000Z"} - - match: {values.3.0: '{"min":802337.0,"max":803685.0,"sum":1606022.0,"value_count":2}'} - - match: {values.3.1: "2021-04-28T20:00:00.000Z"} + - length: { values: 4 } + - length: { values.0: 2 } + - match: { columns.0.name: "k8s.pod.network.rx" } + - match: { columns.0.type: "aggregate_metric_double" } + - match: { columns.1.name: "@timestamp" } + - match: { columns.1.type: "date" } + - match: { values.0.0: '{"min":801479.0,"max":802133.0,"sum":1603612.0,"value_count":2}' } + - match: { values.0.1: "2021-04-28T18:00:00.000Z" } + - match: { values.1.0: '{"min":530575.0,"max":530600.0,"sum":1061175.0,"value_count":2}' } + - match: { values.1.1: "2021-04-28T18:00:00.000Z" } + - match: { values.2.0: '{"min":530604.0,"max":530605.0,"sum":1061209.0,"value_count":2}' } + - match: { values.2.1: "2021-04-28T19:00:00.000Z" } + - match: { values.3.0: '{"min":802337.0,"max":803685.0,"sum":1606022.0,"value_count":2}' } + - match: { values.3.1: "2021-04-28T20:00:00.000Z" } From 3b5037fa96d0adab5689a4e9651f788c4656a659 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Wed, 27 Aug 2025 10:52:31 -0400 Subject: [PATCH 15/15] revert changes to downsample.yml; Those were applied in a different PR --- .../rest-api-spec/test/esql/46_downsample.yml | 368 +++++++++--------- 1 file changed, 184 insertions(+), 184 deletions(-) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml index 9f256564650a1..be81d25d5bac0 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml @@ -78,12 +78,12 @@ setup: --- "Query stats on downsampled index": - requires: - test_runner_features: [ capabilities ] + test_runner_features: [capabilities] capabilities: - method: POST path: /_query - parameters: [ ] - capabilities: [ aggregate_metric_double ] + parameters: [] + capabilities: [aggregate_metric_double] reason: "Support for aggregate_metric_double" - do: indices.downsample: @@ -102,30 +102,30 @@ setup: STATS max(k8s.pod.network.rx), min(k8s.pod.network.rx), sum(k8s.pod.network.rx), count(k8s.pod.network.rx) | LIMIT 100" - - length: { values: 1 } - - length: { values.0: 4 } - - match: { columns.0.name: "max(k8s.pod.network.rx)" } - - match: { columns.0.type: "double" } - - match: { columns.1.name: "min(k8s.pod.network.rx)" } - - match: { columns.1.type: "double" } - - match: { columns.2.name: "sum(k8s.pod.network.rx)" } - - match: { columns.2.type: "double" } - - match: { columns.3.name: "count(k8s.pod.network.rx)" } - - match: { columns.3.type: "long" } - - match: { values.0.0: 803685.0 } - - match: { values.0.1: 530575.0 } - - match: { values.0.2: 5332018.0 } - - match: { values.0.3: 8 } + - length: {values: 1} + - length: {values.0: 4} + - match: {columns.0.name: "max(k8s.pod.network.rx)"} + - match: {columns.0.type: "double"} + - match: {columns.1.name: "min(k8s.pod.network.rx)"} + - match: {columns.1.type: "double"} + - match: {columns.2.name: "sum(k8s.pod.network.rx)"} + - match: {columns.2.type: "double"} + - match: {columns.3.name: "count(k8s.pod.network.rx)"} + - match: {columns.3.type: "long"} + - match: {values.0.0: 803685.0} + - match: {values.0.1: 530575.0} + - match: {values.0.2: 5332018.0} + - match: {values.0.3: 8} --- "Render stats from downsampled index": - requires: - test_runner_features: [ capabilities ] + test_runner_features: [capabilities] capabilities: - method: POST path: /_query - parameters: [ ] - capabilities: [ aggregate_metric_double_rendering ] + parameters: [] + capabilities: [aggregate_metric_double_rendering] reason: "Support for rendering aggregate_metric_doubles" - do: indices.downsample: @@ -141,21 +141,21 @@ setup: esql.query: body: query: "FROM test-downsample | WHERE @timestamp == \"2021-04-28T19:00:00.000Z\" | KEEP k8s.pod.network.rx | LIMIT 100" - - length: { values: 1 } - - length: { values.0: 1 } - - match: { columns.0.name: "k8s.pod.network.rx" } - - match: { columns.0.type: "aggregate_metric_double" } - - match: { values.0.0: '{"min":530604.0,"max":530605.0,"sum":1061209.0,"value_count":2}' } + - length: {values: 1} + - length: {values.0: 1} + - match: {columns.0.name: "k8s.pod.network.rx"} + - match: {columns.0.type: "aggregate_metric_double"} + - match: {values.0.0: '{"min":530604.0,"max":530605.0,"sum":1061209.0,"value_count":2}'} --- "Stats from downsampled and non-downsampled index simultaneously": - requires: - test_runner_features: [ capabilities ] + test_runner_features: [capabilities] capabilities: - method: POST path: /_query - parameters: [ ] - capabilities: [ aggregate_metric_double_convert_to ] + parameters: [] + capabilities: [aggregate_metric_double_convert_to] reason: "Support for to_aggregate_metric_double function" - do: @@ -237,30 +237,30 @@ setup: STATS max(rx), min(rx), sum(rx), count(rx) | LIMIT 100" - - length: { values: 1 } - - length: { values.0: 4 } - - match: { columns.0.name: "max(rx)" } - - match: { columns.0.type: "double" } - - match: { columns.1.name: "min(rx)" } - - match: { columns.1.type: "double" } - - match: { columns.2.name: "sum(rx)" } - - match: { columns.2.type: "double" } - - match: { columns.3.name: "count(rx)" } - - match: { columns.3.type: "long" } - - match: { values.0.0: 803685.0 } - - match: { values.0.1: 800479.0 } - - match: { values.0.2: 4812452.0 } - - match: { values.0.3: 6 } + - length: {values: 1} + - length: {values.0: 4} + - match: {columns.0.name: "max(rx)"} + - match: {columns.0.type: "double"} + - match: {columns.1.name: "min(rx)"} + - match: {columns.1.type: "double"} + - match: {columns.2.name: "sum(rx)"} + - match: {columns.2.type: "double"} + - match: {columns.3.name: "count(rx)"} + - match: {columns.3.type: "long"} + - match: {values.0.0: 803685.0} + - match: {values.0.1: 800479.0} + - match: {values.0.2: 4812452.0} + - match: {values.0.3: 6} --- "Stats from downsampled and non-downsampled index simultaneously with implicit casting": - requires: - test_runner_features: [ capabilities ] + test_runner_features: [capabilities] capabilities: - method: POST path: /_query - parameters: [ ] - capabilities: [ aggregate_metric_double_implicit_casting_in_aggs ] + parameters: [] + capabilities: [aggregate_metric_double_implicit_casting_in_aggs] reason: "Support for casting aggregate metric double implicitly when present in aggregations" - do: @@ -341,43 +341,43 @@ setup: STATS max(k8s.pod.network.rx), min(k8s.pod.network.rx), sum(k8s.pod.network.rx), count(k8s.pod.network.rx), avg(k8s.pod.network.rx) | LIMIT 100" - - length: { values: 1 } - - length: { values.0: 5 } - - match: { columns.0.name: "max(k8s.pod.network.rx)" } - - match: { columns.0.type: "double" } - - match: { columns.1.name: "min(k8s.pod.network.rx)" } - - match: { columns.1.type: "double" } - - match: { columns.2.name: "sum(k8s.pod.network.rx)" } - - match: { columns.2.type: "double" } - - match: { columns.3.name: "count(k8s.pod.network.rx)" } - - match: { columns.3.type: "long" } - - match: { columns.4.name: "avg(k8s.pod.network.rx)" } - - match: { columns.4.type: "double" } - - match: { values.0.0: 803685.0 } - - match: { values.0.1: 800479.0 } - - match: { values.0.2: 4812452.0 } - - match: { values.0.3: 6 } - - match: { values.0.4: 802075.3333333334 } + - length: {values: 1} + - length: {values.0: 5} + - match: {columns.0.name: "max(k8s.pod.network.rx)"} + - match: {columns.0.type: "double"} + - match: {columns.1.name: "min(k8s.pod.network.rx)"} + - match: {columns.1.type: "double"} + - match: {columns.2.name: "sum(k8s.pod.network.rx)"} + - match: {columns.2.type: "double"} + - match: {columns.3.name: "count(k8s.pod.network.rx)"} + - match: {columns.3.type: "long"} + - match: {columns.4.name: "avg(k8s.pod.network.rx)"} + - match: {columns.4.type: "double"} + - match: {values.0.0: 803685.0} + - match: {values.0.1: 800479.0} + - match: {values.0.2: 4812452.0} + - match: {values.0.3: 6} + - match: {values.0.4: 802075.3333333334} - do: esql.query: body: query: "TS test-* | STATS max = max(k8s.pod.network.rx) | LIMIT 100" - - length: { values: 1 } - - length: { values.0: 1 } - - match: { columns.0.name: "max" } - - match: { columns.0.type: "double" } - - match: { values.0.0: 803685.0 } + - length: {values: 1} + - length: {values.0: 1} + - match: {columns.0.name: "max"} + - match: {columns.0.type: "double"} + - match: {values.0.0: 803685.0} --- "Over time functions from downsampled and non-downsampled indices simultaneously, no grouping": - requires: - test_runner_features: [ capabilities ] + test_runner_features: [capabilities] capabilities: - method: POST path: /_query - parameters: [ ] - capabilities: [ metrics_command, aggregate_metric_double_implicit_casting_in_aggs ] + parameters: [] + capabilities: [metrics_command, aggregate_metric_double_implicit_casting_in_aggs] reason: "Support for casting aggregate metric double implicitly when present in aggregations" - do: @@ -464,42 +464,42 @@ setup: BY time_bucket = bucket(@timestamp, 1 hour) | SORT time_bucket | LIMIT 10" - - length: { values: 4 } - - length: { values.0: 4 } - - match: { columns.0.name: "avg" } - - match: { columns.0.type: "double" } - - match: { columns.1.name: "count" } - - match: { columns.1.type: "long" } - - match: { columns.2.name: "sum" } - - match: { columns.2.type: "double" } - - match: { columns.3.name: "time_bucket" } - - match: { columns.3.type: "date" } - - match: { values.0.0: 1332393.5 } - - match: { values.0.1: 4 } - - match: { values.0.2: 2664787.0 } - - match: { values.0.3: "2021-04-28T18:00:00.000Z" } - - match: { values.1.0: 530604.5 } - - match: { values.1.1: 2 } - - match: { values.1.2: 1061209.0 } - - match: { values.1.3: "2021-04-28T19:00:00.000Z" } - - match: { values.2.0: 803011.0 } - - match: { values.2.1: 2 } - - match: { values.2.2: 1606022.0 } - - match: { values.2.3: "2021-04-28T20:00:00.000Z" } - - match: { values.3.0: 1334011.0 } - - match: { values.3.1: 4 } - - match: { values.3.2: 2668022.0 } - - match: { values.3.3: "2021-04-29T21:00:00.000Z" } + - length: {values: 4} + - length: {values.0: 4} + - match: {columns.0.name: "avg"} + - match: {columns.0.type: "double"} + - match: {columns.1.name: "count"} + - match: {columns.1.type: "long"} + - match: {columns.2.name: "sum"} + - match: {columns.2.type: "double"} + - match: {columns.3.name: "time_bucket"} + - match: {columns.3.type: "date"} + - match: {values.0.0: 1332393.5} + - match: {values.0.1: 4} + - match: {values.0.2: 2664787.0} + - match: {values.0.3: "2021-04-28T18:00:00.000Z"} + - match: {values.1.0: 530604.5} + - match: {values.1.1: 2} + - match: {values.1.2: 1061209.0} + - match: {values.1.3: "2021-04-28T19:00:00.000Z"} + - match: {values.2.0: 803011.0} + - match: {values.2.1: 2} + - match: {values.2.2: 1606022.0} + - match: {values.2.3: "2021-04-28T20:00:00.000Z"} + - match: {values.3.0: 1334011.0} + - match: {values.3.1: 4} + - match: {values.3.2: 2668022.0} + - match: {values.3.3: "2021-04-29T21:00:00.000Z"} --- "Over time functions from downsampled and non-downsampled indices simultaneously, with grouping": - requires: - test_runner_features: [ capabilities ] + test_runner_features: [capabilities] capabilities: - method: POST path: /_query - parameters: [ ] - capabilities: [ metrics_command, aggregate_metric_double_implicit_casting_in_aggs ] + parameters: [] + capabilities: [metrics_command, aggregate_metric_double_implicit_casting_in_aggs] reason: "Support for casting aggregate metric double implicitly when present in aggregations" - do: @@ -587,58 +587,58 @@ setup: SORT time_bucket, k8s.pod.name | LIMIT 10" - - length: { values: 6 } - - length: { values.0: 5 } - - match: { columns.0.name: "avg" } - - match: { columns.0.type: "double" } - - match: { columns.1.name: "count" } - - match: { columns.1.type: "long" } - - match: { columns.2.name: "sum" } - - match: { columns.2.type: "double" } - - match: { columns.3.name: "k8s.pod.name" } - - match: { columns.3.type: "keyword" } - - match: { columns.4.name: "time_bucket" } - - match: { columns.4.type: "date" } - - match: { values.0.0: 801806.0 } - - match: { values.0.1: 2 } - - match: { values.0.2: 1603612.0 } - - match: { values.0.3: "cat" } - - match: { values.0.4: "2021-04-28T18:00:00.000Z" } - - match: { values.1.0: 530587.5 } - - match: { values.1.1: 2 } - - match: { values.1.2: 1061175.0 } - - match: { values.1.3: "dog" } - - match: { values.1.4: "2021-04-28T18:00:00.000Z" } - - match: { values.2.0: 530604.5 } - - match: { values.2.1: 2 } - - match: { values.2.2: 1061209.0 } - - match: { values.2.3: "dog" } - - match: { values.2.4: "2021-04-28T19:00:00.000Z" } - - match: { values.3.0: 803011.0 } - - match: { values.3.1: 2 } - - match: { values.3.2: 1606022.0 } - - match: { values.3.3: "cat" } - - match: { values.3.4: "2021-04-28T20:00:00.000Z" } - - match: { values.4.0: 801409.0 } - - match: { values.4.1: 2 } - - match: { values.4.2: 1602818.0 } - - match: { values.4.3: "cat" } - - match: { values.4.4: "2021-04-29T21:00:00.000Z" } - - match: { values.5.0: 532602.0 } - - match: { values.5.1: 2 } - - match: { values.5.2: 1065204.0 } - - match: { values.5.3: "dog" } - - match: { values.5.4: "2021-04-29T21:00:00.000Z" } + - length: {values: 6} + - length: {values.0: 5} + - match: {columns.0.name: "avg"} + - match: {columns.0.type: "double"} + - match: {columns.1.name: "count"} + - match: {columns.1.type: "long"} + - match: {columns.2.name: "sum"} + - match: {columns.2.type: "double"} + - match: {columns.3.name: "k8s.pod.name"} + - match: {columns.3.type: "keyword"} + - match: {columns.4.name: "time_bucket"} + - match: {columns.4.type: "date"} + - match: {values.0.0: 801806.0} + - match: {values.0.1: 2} + - match: {values.0.2: 1603612.0} + - match: {values.0.3: "cat"} + - match: {values.0.4: "2021-04-28T18:00:00.000Z"} + - match: {values.1.0: 530587.5} + - match: {values.1.1: 2} + - match: {values.1.2: 1061175.0} + - match: {values.1.3: "dog"} + - match: {values.1.4: "2021-04-28T18:00:00.000Z"} + - match: {values.2.0: 530604.5} + - match: {values.2.1: 2} + - match: {values.2.2: 1061209.0} + - match: {values.2.3: "dog"} + - match: {values.2.4: "2021-04-28T19:00:00.000Z"} + - match: {values.3.0: 803011.0} + - match: {values.3.1: 2} + - match: {values.3.2: 1606022.0} + - match: {values.3.3: "cat"} + - match: {values.3.4: "2021-04-28T20:00:00.000Z"} + - match: {values.4.0: 801409.0} + - match: {values.4.1: 2} + - match: {values.4.2: 1602818.0} + - match: {values.4.3: "cat"} + - match: {values.4.4: "2021-04-29T21:00:00.000Z"} + - match: {values.5.0: 532602.0} + - match: {values.5.1: 2} + - match: {values.5.2: 1065204.0} + - match: {values.5.3: "dog"} + - match: {values.5.4: "2021-04-29T21:00:00.000Z"} --- "Sort from multiple indices one with aggregate metric double": - requires: - test_runner_features: [ capabilities ] + test_runner_features: [capabilities] capabilities: - method: POST path: /_query - parameters: [ ] - capabilities: [ aggregate_metric_double_sorting_fixed ] + parameters: [] + capabilities: [aggregate_metric_double_sorting_fixed] reason: "Fix sorting for rows comprised of docs from multiple indices where agg metric is missing from some" - do: @@ -673,39 +673,39 @@ setup: body: query: "FROM test-* | SORT some_field, @timestamp, k8s.pod.uid | KEEP k8s.pod.network.rx, some_field, @timestamp | LIMIT 10" - - length: { values: 5 } - - length: { values.0: 3 } - - match: { columns.0.name: "k8s.pod.network.rx" } - - match: { columns.0.type: "aggregate_metric_double" } - - match: { columns.1.name: "some_field" } - - match: { columns.1.type: "keyword" } - - match: { columns.2.name: "@timestamp" } - - match: { columns.2.type: "date" } - - match: { values.0.0: null } - - match: { values.0.1: "im a keyword!!!!!" } - - match: { values.0.2: null } - - match: { values.1.0: '{"min":801479.0,"max":802133.0,"sum":1603612.0,"value_count":2}' } - - match: { values.1.1: null } - - match: { values.1.2: "2021-04-28T18:00:00.000Z" } - - match: { values.2.0: '{"min":530575.0,"max":530600.0,"sum":1061175.0,"value_count":2}' } - - match: { values.2.1: null } - - match: { values.2.2: "2021-04-28T18:00:00.000Z" } - - match: { values.3.0: '{"min":530604.0,"max":530605.0,"sum":1061209.0,"value_count":2}' } - - match: { values.3.1: null } - - match: { values.3.2: "2021-04-28T19:00:00.000Z" } - - match: { values.4.0: '{"min":802337.0,"max":803685.0,"sum":1606022.0,"value_count":2}' } - - match: { values.4.1: null } - - match: { values.4.2: "2021-04-28T20:00:00.000Z" } + - length: {values: 5} + - length: {values.0: 3} + - match: {columns.0.name: "k8s.pod.network.rx"} + - match: {columns.0.type: "aggregate_metric_double"} + - match: {columns.1.name: "some_field"} + - match: {columns.1.type: "keyword"} + - match: {columns.2.name: "@timestamp"} + - match: {columns.2.type: "date"} + - match: {values.0.0: null} + - match: {values.0.1: "im a keyword!!!!!"} + - match: {values.0.2: null} + - match: {values.1.0: '{"min":801479.0,"max":802133.0,"sum":1603612.0,"value_count":2}'} + - match: {values.1.1: null} + - match: {values.1.2: "2021-04-28T18:00:00.000Z"} + - match: {values.2.0: '{"min":530575.0,"max":530600.0,"sum":1061175.0,"value_count":2}'} + - match: {values.2.1: null} + - match: {values.2.2: "2021-04-28T18:00:00.000Z"} + - match: {values.3.0: '{"min":530604.0,"max":530605.0,"sum":1061209.0,"value_count":2}'} + - match: {values.3.1: null} + - match: {values.3.2: "2021-04-28T19:00:00.000Z"} + - match: {values.4.0: '{"min":802337.0,"max":803685.0,"sum":1606022.0,"value_count":2}'} + - match: {values.4.1: null} + - match: {values.4.2: "2021-04-28T20:00:00.000Z"} --- "MV_EXPAND on non-MV aggregate metric double": - requires: - test_runner_features: [ capabilities ] + test_runner_features: [capabilities] capabilities: - method: POST path: /_query - parameters: [ ] - capabilities: [ aggregate_metric_double_mv_expand ] + parameters: [] + capabilities: [aggregate_metric_double_mv_expand] reason: "Have MV_EXPAND not error out when applied to aggregate_metric_doubles (is a no-op)" - do: @@ -723,17 +723,17 @@ setup: body: query: "FROM test-downsample | MV_EXPAND k8s.pod.network.rx | SORT @timestamp, k8s.pod.uid | KEEP k8s.pod.network.rx, @timestamp | LIMIT 10" - - length: { values: 4 } - - length: { values.0: 2 } - - match: { columns.0.name: "k8s.pod.network.rx" } - - match: { columns.0.type: "aggregate_metric_double" } - - match: { columns.1.name: "@timestamp" } - - match: { columns.1.type: "date" } - - match: { values.0.0: '{"min":801479.0,"max":802133.0,"sum":1603612.0,"value_count":2}' } - - match: { values.0.1: "2021-04-28T18:00:00.000Z" } - - match: { values.1.0: '{"min":530575.0,"max":530600.0,"sum":1061175.0,"value_count":2}' } - - match: { values.1.1: "2021-04-28T18:00:00.000Z" } - - match: { values.2.0: '{"min":530604.0,"max":530605.0,"sum":1061209.0,"value_count":2}' } - - match: { values.2.1: "2021-04-28T19:00:00.000Z" } - - match: { values.3.0: '{"min":802337.0,"max":803685.0,"sum":1606022.0,"value_count":2}' } - - match: { values.3.1: "2021-04-28T20:00:00.000Z" } + - length: {values: 4} + - length: {values.0: 2} + - match: {columns.0.name: "k8s.pod.network.rx"} + - match: {columns.0.type: "aggregate_metric_double"} + - match: {columns.1.name: "@timestamp"} + - match: {columns.1.type: "date"} + - match: {values.0.0: '{"min":801479.0,"max":802133.0,"sum":1603612.0,"value_count":2}'} + - match: {values.0.1: "2021-04-28T18:00:00.000Z"} + - match: {values.1.0: '{"min":530575.0,"max":530600.0,"sum":1061175.0,"value_count":2}'} + - match: {values.1.1: "2021-04-28T18:00:00.000Z"} + - match: {values.2.0: '{"min":530604.0,"max":530605.0,"sum":1061209.0,"value_count":2}'} + - match: {values.2.1: "2021-04-28T19:00:00.000Z"} + - match: {values.3.0: '{"min":802337.0,"max":803685.0,"sum":1606022.0,"value_count":2}'} + - match: {values.3.1: "2021-04-28T20:00:00.000Z"}