Merge branch 'main' into fix-missing-index-handling

Author: smalyshev

docs/changelog/126550.yaml

@@ -0,0 +1,6 @@
+pr: 126550
+summary: Add leniency to missing array values in mustache
+area: Infra/Scripting
+type: bug
+issues:
+ - 55200

modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java

@@ -111,11 +111,11 @@ public Object get(Object key) {
             if ("size".equals(key)) {
                 return size();
             } else if (key instanceof Number number) {
-                return Array.get(array, number.intValue());
+                return number.intValue() >= 0 && number.intValue() < length ? Array.get(array, number.intValue()) : null;
             }
             try {
                 int index = Integer.parseInt(key.toString());
-                return Array.get(array, index);
+                return index >= 0 && index < length ? Array.get(array, index) : null;
             } catch (NumberFormatException nfe) {
                 // if it's not a number it is as if the key doesn't exist
                 return null;
@@ -169,11 +169,11 @@ public Object get(Object key) {
             if ("size".equals(key)) {
                 return col.size();
             } else if (key instanceof Number number) {
-                return Iterables.get(col, number.intValue());
+                return number.intValue() >= 0 && number.intValue() < col.size() ? Iterables.get(col, number.intValue()) : null;
             }
             try {
                 int index = Integer.parseInt(key.toString());
-                return Iterables.get(col, index);
+                return index >= 0 && index < col.size() ? Iterables.get(col, index) : null;
             } catch (NumberFormatException nfe) {
                 // if it's not a number it is as if the key doesn't exist
                 return null;

modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java

@@ -141,6 +141,26 @@ public void testMapInArrayAccess() throws Exception {
         assertThat(output, both(containsString("foo")).and(containsString("bar")));
     }
 
+    public void testMapInArrayBadAccess() throws Exception {
+        String template = "{{data.0.key}} {{data.2.key}}";
+        TemplateScript.Factory factory = engine.compile(null, template, TemplateScript.CONTEXT, Collections.emptyMap());
+        Map<String, Object> vars = new HashMap<>();
+        vars.put("data", new Object[] { singletonMap("key", "foo"), singletonMap("key", "bar") });
+        // assertThat(factory.newInstance(vars).execute(), equalTo("foo "));
+
+        vars.put("data", Arrays.asList(singletonMap("key", "foo"), singletonMap("key", "bar")));
+        factory.newInstance(vars);
+        assertThat(factory.newInstance(vars).execute(), equalTo("foo "));
+
+        // HashSet iteration order isn't fixed
+        Set<Object> setData = new HashSet<>();
+        setData.add(singletonMap("key", "foo"));
+        setData.add(singletonMap("key", "bar"));
+        vars.put("data", setData);
+        String output = factory.newInstance(vars).execute();
+        assertThat(output, both(containsString("foo")).and(not(containsString("bar"))));
+    }
+
     public void testSizeAccessForCollectionsAndArrays() throws Exception {
         String[] randomArrayValues = generateRandomStringArray(10, 20, false);
         List<String> randomList = Arrays.asList(generateRandomStringArray(10, 20, false));

muted-tests.yml

@@ -423,6 +423,21 @@ tests:
 - class: org.elasticsearch.smoketest.MlWithSecurityIT
   method: test {yaml=ml/get_trained_model_stats/Test get stats given trained models}
   issue: https://github.com/elastic/elasticsearch/issues/126510
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=transform/transforms_stats/Test get multiple transform stats}
+  issue: https://github.com/elastic/elasticsearch/issues/126567
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=transform/transforms_stats/Test get single transform stats when it does not have a task}
+  issue: https://github.com/elastic/elasticsearch/issues/126568
+- class: org.elasticsearch.repositories.blobstore.testkit.rest.SnapshotRepoTestKitClientYamlTestSuiteIT
+  method: test {p0=/10_analyze/Analysis without details}
+  issue: https://github.com/elastic/elasticsearch/issues/126569
+- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeIT
+  method: test
+  issue: https://github.com/elastic/elasticsearch/issues/126573
+- class: org.elasticsearch.repositories.blobstore.testkit.analyze.S3RepositoryAnalysisRestIT
+  method: testRepositoryAnalysis
+  issue: https://github.com/elastic/elasticsearch/issues/126576
 
 # Examples:
 #

x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec

@@ -164,3 +164,22 @@ null                                     | three       | 2024-05-10T00:05:00.000
 null                                     | three       | 2024-05-10T00:02:00.000Z
 null                                     | three       | 2024-05-10T00:01:00.000Z
 ;
+
+
+max_over_time
+required_capability: metrics_command
+required_capability: max_over_time
+TS k8s | STATS cost=sum(max_over_time(network.cost)) BY cluster, time_bucket = bucket(@timestamp,1minute) | SORT cost DESC, time_bucket DESC, cluster | LIMIT 10;
+
+cost:double | cluster:keyword | time_bucket:datetime    
+32.75       | qa              | 2024-05-10T00:17:00.000Z
+32.25       | staging         | 2024-05-10T00:09:00.000Z
+31.75       | qa              | 2024-05-10T00:06:00.000Z
+29.0        | prod            | 2024-05-10T00:19:00.000Z
+28.625      | qa              | 2024-05-10T00:09:00.000Z
+24.625      | qa              | 2024-05-10T00:18:00.000Z
+23.25       | qa              | 2024-05-10T00:11:00.000Z
+23.125      | staging         | 2024-05-10T00:08:00.000Z
+22.75       | prod            | 2024-05-10T00:13:00.000Z
+22.75       | qa              | 2024-05-10T00:08:00.000Z
+;

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

@@ -959,7 +959,12 @@ public enum Cap {
         /**
          * Listing queries and getting information on a specific query.
          */
-        QUERY_MONITORING;
+        QUERY_MONITORING,
+
+        /**
+         * Support max_over_time aggregation
+         */
+        MAX_OVER_TIME(Build.current().isSnapshot());
 
         private final boolean enabled;
 

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Count;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.CountDistinct;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Max;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.MaxOverTime;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Median;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.MedianAbsoluteDeviation;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Min;
@@ -432,6 +433,7 @@ private static FunctionDefinition[][] snapshotFunctions() {
                 // This is an experimental function and can be removed without notice.
                 def(Delay.class, Delay::new, "delay"),
                 def(Rate.class, Rate::withUnresolvedTimestamp, "rate"),
+                def(MaxOverTime.class, uni(MaxOverTime::new), "max_over_time"),
                 def(Term.class, bi(Term::new), "term") } };
     }
 

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateWritables.java

@@ -30,6 +30,7 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
             Sum.ENTRY,
             Top.ENTRY,
             Values.ENTRY,
+            MaxOverTime.ENTRY,
             // internal functions
             ToPartial.ENTRY,
             FromPartial.ENTRY,

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxOverTime.java

@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.aggregate;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.FunctionType;
+import org.elasticsearch.xpack.esql.expression.function.Param;
+
+import java.io.IOException;
+import java.util.List;
+
+import static java.util.Collections.emptyList;
+
+/**
+ * Similar to {@link Max}, but it is used to calculate the maximum value over a time series of values from the given field.
+ */
+public class MaxOverTime extends TimeSeriesAggregateFunction {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        Expression.class,
+        "MaxOverTime",
+        MaxOverTime::new
+    );
+
+    @FunctionInfo(
+        returnType = { "boolean", "double", "integer", "long", "date", "date_nanos", "ip", "keyword", "long", "version" },
+        description = "The maximum over time value of a field.",
+        type = FunctionType.AGGREGATE
+    )
+    public MaxOverTime(
+        Source source,
+        @Param(
+            name = "field",
+            type = {
+                "aggregate_metric_double",
+                "boolean",
+                "double",
+                "integer",
+                "long",
+                "date",
+                "date_nanos",
+                "ip",
+                "keyword",
+                "text",
+                "long",
+                "version" }
+        ) Expression field
+    ) {
+        this(source, field, Literal.TRUE);
+    }
+
+    public MaxOverTime(Source source, Expression field, Expression filter) {
+        super(source, field, filter, emptyList());
+    }
+
+    private MaxOverTime(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
+    @Override
+    public MaxOverTime withFilter(Expression filter) {
+        return new MaxOverTime(source(), field(), filter);
+    }
+
+    @Override
+    protected NodeInfo<MaxOverTime> info() {
+        return NodeInfo.create(this, MaxOverTime::new, field(), filter());
+    }
+
+    @Override
+    public MaxOverTime replaceChildren(List<Expression> newChildren) {
+        return new MaxOverTime(source(), newChildren.get(0), newChildren.get(1));
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        return perTimeSeriesAggregation().resolveType();
+    }
+
+    @Override
+    public DataType dataType() {
+        return perTimeSeriesAggregation().dataType();
+    }
+
+    @Override
+    public Max perTimeSeriesAggregation() {
+        return new Max(source(), field(), filter());
+    }
+}

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java

@@ -25,15 +25,15 @@
 import org.elasticsearch.xpack.esql.expression.function.OptionalArgument;
 import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
-import org.elasticsearch.xpack.esql.planner.ToTimeSeriesAggregator;
+import org.elasticsearch.xpack.esql.planner.ToAggregator;
 
 import java.io.IOException;
 import java.util.List;
 
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
 
-public class Rate extends AggregateFunction implements OptionalArgument, ToTimeSeriesAggregator {
+public class Rate extends TimeSeriesAggregateFunction implements OptionalArgument, ToAggregator {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Rate", Rate::new);
 
     private final Expression timestamp;
@@ -119,6 +119,11 @@ public AggregatorFunctionSupplier supplier() {
         };
     }
 
+    @Override
+    public Rate perTimeSeriesAggregation() {
+        return this;
+    }
+
     @Override
     public String toString() {
         return "rate(" + field() + ")";