Merge branch 'main' into pkar/resolve-index-force-reconn

Author: pawankartik-elastic

server/src/main/resources/transport/definitions/referable/esql_timestamps_info.csv

@@ -0,0 +1 @@
+9228000

server/src/main/resources/transport/upper_bounds/9.3.csv

@@ -1 +1 @@
-aggregate_metric_double_typed_block,9227000
+esql_timestamps_info,9228000

test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java

@@ -169,6 +169,9 @@ public void testSortByManyLongsGiantTopN() throws IOException {
                 .entry("values", List.of(List.of(9)))
                 .entry("documents_found", greaterThan(0))
                 .entry("values_loaded", greaterThan(0))
+                .entry("completion_time_in_millis", greaterThan(0L))
+                .entry("expiration_time_in_millis", greaterThan(0L))
+                .entry("start_time_in_millis", greaterThan(0L))
         );
     }
 

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -2767,13 +2767,20 @@ protected static MapMatcher getProfileMatcher() {
             .entry("plans", instanceOf(List.class));
     }
 
-    protected static MapMatcher getResultMatcher(boolean includePartial, boolean includeDocumentsFound) {
+    protected static MapMatcher getResultMatcher(boolean includePartial, boolean includeDocumentsFound, boolean includeTimestamps) {
         MapMatcher mapMatcher = matchesMap();
         if (includeDocumentsFound) {
             // Older versions may not return documents_found and values_loaded.
             mapMatcher = mapMatcher.entry("documents_found", greaterThanOrEqualTo(0));
             mapMatcher = mapMatcher.entry("values_loaded", greaterThanOrEqualTo(0));
         }
+        if (includeTimestamps) {
+            // Older versions may not return start_time_in_millis, completion_time_in_millis and expiration_time_in_millis
+            mapMatcher = mapMatcher.entry("start_time_in_millis", greaterThanOrEqualTo(0L));
+            mapMatcher = mapMatcher.entry("completion_time_in_millis", greaterThanOrEqualTo(0L));
+            mapMatcher = mapMatcher.entry("expiration_time_in_millis", greaterThanOrEqualTo(0L));
+        }
+
         mapMatcher = mapMatcher.entry("took", greaterThanOrEqualTo(0));
         // Older version may not have is_partial
         if (includePartial) {
@@ -2786,7 +2793,11 @@ protected static MapMatcher getResultMatcher(boolean includePartial, boolean inc
      * Create empty result matcher from result, taking into account all metadata items.
      */
     protected static MapMatcher getResultMatcher(Map<String, Object> result) {
-        return getResultMatcher(result.containsKey("is_partial"), result.containsKey("documents_found"));
+        return getResultMatcher(
+            result.containsKey("is_partial"),
+            result.containsKey("documents_found"),
+            result.containsKey("start_time_in_millis")
+        );
     }
 
     /**

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Node.java

@@ -176,6 +176,23 @@ public List<T> collect(Predicate<? super T> predicate) {
         return l.isEmpty() ? emptyList() : l;
     }
 
+    public <E extends T> List<E> collect(Class<E> typeToken) {
+        return collect(typeToken, n -> true);
+    }
+
+    public <E extends T> List<E> collect(Class<E> typeToken, Predicate<? super E> predicate) {
+        List<E> l = new ArrayList<>();
+        forEachDown(n -> {
+            if (typeToken.isInstance(n)) {
+                E e = typeToken.cast(n);
+                if (predicate.test(e)) {
+                    l.add(e);
+                }
+            }
+        });
+        return l.isEmpty() ? emptyList() : l;
+    }
+
     public List<T> collectLeaves() {
         return collect(n -> n.children().isEmpty());
     }

x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterTimeSeriesIT.java

@@ -311,7 +311,11 @@ private boolean capabilitiesSupportedNewAndOld(List<String> requiredCapabilities
     }
 
     private void assertResultMap(boolean includeCCSMetadata, Map<String, Object> result, Map<String, Object> expectedResult) {
-        MapMatcher mapMatcher = getResultMatcher(result.containsKey("is_partial"), result.containsKey("documents_found")).extraOk();
+        MapMatcher mapMatcher = getResultMatcher(
+            result.containsKey("is_partial"),
+            result.containsKey("documents_found"),
+            result.containsKey("start_time_in_millis")
+        ).extraOk();
         if (includeCCSMetadata) {
             mapMatcher = mapMatcher.entry("_clusters", any(Map.class));
         }

x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java

@@ -262,7 +262,11 @@ private boolean capabilitiesSupportedNewAndOld(List<String> requiredCapabilities
     }
 
     private <C, V> void assertResultMap(boolean includeCCSMetadata, Map<String, Object> result, C columns, V values, boolean remoteOnly) {
-        MapMatcher mapMatcher = getResultMatcher(result.containsKey("is_partial"), result.containsKey("documents_found")).extraOk();
+        MapMatcher mapMatcher = getResultMatcher(
+            result.containsKey("is_partial"),
+            result.containsKey("documents_found"),
+            result.containsKey("start_time_in_millis")
+        ).extraOk();
         if (includeCCSMetadata) {
             mapMatcher = mapMatcher.entry("_clusters", any(Map.class));
         }
@@ -523,7 +527,11 @@ public void testLookupJoinAliasesSkipOld() throws IOException {
             var columns = List.of(Map.of("name", "c", "type", "long"));
             var values = List.of(List.of(localDocs.size()));
 
-            MapMatcher mapMatcher = getResultMatcher(false, result.containsKey("documents_found")).extraOk();
+            MapMatcher mapMatcher = getResultMatcher(
+                false,
+                result.containsKey("documents_found"),
+                result.containsKey("start_time_in_millis")
+            ).extraOk();
             mapMatcher = mapMatcher.entry("_clusters", any(Map.class));
             mapMatcher = mapMatcher.entry("is_partial", true);
             assertMap(result, mapMatcher.entry("columns", columns).entry("values", values));

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java

@@ -74,6 +74,7 @@
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.emptyOrNullString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
@@ -288,7 +289,7 @@ public static RequestObjectBuilder jsonBuilder() throws IOException {
 
     public void testGetAnswer() throws IOException {
         Map<String, Object> answer = runEsql(requestObjectBuilder().query("row a = 1, b = 2"));
-        assertEquals(6, answer.size());
+        assertEquals(9, answer.size());
         assertThat(((Integer) answer.get("took")).intValue(), greaterThanOrEqualTo(0));
         Map<String, String> colA = Map.of("name", "a", "type", "integer");
         Map<String, String> colB = Map.of("name", "b", "type", "integer");
@@ -300,6 +301,9 @@ public void testGetAnswer() throws IOException {
                 .entry("values_loaded", 0)
                 .entry("columns", List.of(colA, colB))
                 .entry("values", List.of(List.of(1, 2)))
+                .entry("completion_time_in_millis", greaterThan(0L))
+                .entry("expiration_time_in_millis", greaterThan(0L))
+                .entry("start_time_in_millis", greaterThan(0L))
         );
     }
 

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java

@@ -60,7 +60,10 @@ protected void shouldSkipTest(String testName) throws IOException {
             testCase.requiredCapabilities.contains(SUBQUERY_IN_FROM_COMMAND.capabilityName())
         );
 
-        assumeFalse("Tests using PROMQL are not supported for now", testCase.requiredCapabilities.contains(PROMQL_V0.capabilityName()));
+        assumeFalse(
+            "Tests using PROMQL are not supported for now",
+            testCase.requiredCapabilities.contains(PROMQL_PRE_TECH_PREVIEW_V1.capabilityName())
+        );
 
         assumeFalse(
             "Tests using GROUP_BY_ALL are skipped since we add a new _timeseries field",

x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries-avg-over-time.csv-spec

@@ -17,6 +17,25 @@ cost:double       | time_bucket:datetime
 36.375            | 2024-05-10T00:11:00.000Z
 ;
 
+avg_over_time_of_double_no_grouping_promql
+required_capability: promql_pre_tech_preview_v1
+TS k8s
+| PROMQL step 1m (sum(avg_over_time(network.cost[1m])))
+| SORT `sum(avg_over_time(network.cost[1m]))` DESC, step DESC | LIMIT 10;
+
+sum(avg_over_time(network.cost[1m])):double | step:date
+69.6875                                     | 2024-05-10T00:09:00.000Z
+56.5625                                     | 2024-05-10T00:08:00.000Z
+49.5                                        | 2024-05-10T00:17:00.000Z
+48.3125                                     | 2024-05-10T00:22:00.000Z
+45.8125                                     | 2024-05-10T00:15:00.000Z
+40.4375                                     | 2024-05-10T00:06:00.000Z
+39.54166666666667                           | 2024-05-10T00:13:00.000Z
+37.9375                                     | 2024-05-10T00:12:00.000Z
+36.6875                                     | 2024-05-10T00:19:00.000Z
+36.375                                      | 2024-05-10T00:11:00.000Z
+;
+
 avg_over_time_of_double_no_grouping_single_bucket
 required_capability: ts_command_v0
 required_capability: tbucket
@@ -28,11 +47,11 @@ cost:double       | time_bucket:datetime
 ;
 
 avg_over_time_of_double_no_grouping_single_bucket_promql
-required_capability: promql_v0
+required_capability: promql_pre_tech_preview_v1
 TS k8s
 | PROMQL step 1h (sum(avg_over_time(network.cost[1h])));
 
-sum(avg_over_time(network.cost[1h])):double | TBUCKET:date
+sum(avg_over_time(network.cost[1h])):double | step:date
 56.32254035241995                           | 2024-05-10T00:00:00.000Z
 ;