Merge branch 'main' into jdk/api-extractor-fix-inherited-access

Author: mosche

muted-tests.yml

@@ -606,6 +606,9 @@ tests:
 - class: org.elasticsearch.xpack.esql.expression.function.aggregate.IncreaseTests
   method: testGroupingAggregate {TestCase=<big positive doubles>}
   issue: https://github.com/elastic/elasticsearch/issues/135755
+- class: org.elasticsearch.xpack.esql.ccq.AllSupportedFieldsIT
+  method: testFetchDenseVector {pref=null mode=time_series}
+  issue: https://github.com/elastic/elasticsearch/issues/135762
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java

@@ -2099,7 +2099,7 @@ public void onResponse(SearchResponse searchResponse) {
                         }
                     }
                 }
-                searchResponseMetrics.incrementResponseCount(responseCountTotalStatus);
+                searchResponseMetrics.incrementResponseCount(responseCountTotalStatus, searchRequestAttributes);
 
                 if (collectCCSTelemetry) {
                     extractCCSTelemetry(searchResponse);
@@ -2115,7 +2115,7 @@ public void onResponse(SearchResponse searchResponse) {
 
         @Override
         public void onFailure(Exception e) {
-            searchResponseMetrics.incrementResponseCount(SearchResponseMetrics.ResponseCountTotalStatus.FAILURE);
+            searchResponseMetrics.incrementResponseCount(SearchResponseMetrics.ResponseCountTotalStatus.FAILURE, searchRequestAttributes);
             if (collectCCSTelemetry) {
                 usageBuilder.setFailure(e);
                 recordTelemetry();

server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.telemetry.metric.LongHistogram;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 
+import java.util.HashMap;
 import java.util.Map;
 
 /**
@@ -83,4 +84,10 @@ public void incrementResponseCount(ResponseCountTotalStatus responseCountTotalSt
             Map.of(RESPONSE_COUNT_TOTAL_STATUS_ATTRIBUTE_NAME, responseCountTotalStatus.getDisplayName())
         );
     }
+
+    public void incrementResponseCount(ResponseCountTotalStatus responseCountTotalStatus, Map<String, Object> attributes) {
+        Map<String, Object> attributesWithStatus = new HashMap<>(attributes);
+        attributesWithStatus.put(RESPONSE_COUNT_TOTAL_STATUS_ATTRIBUTE_NAME, responseCountTotalStatus.getDisplayName());
+        responseCountTotalCounter.incrementBy(1L, attributesWithStatus);
+    }
 }

server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchResponseCountTelemetryTests.java

@@ -31,12 +31,15 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery;
 import static org.elasticsearch.rest.action.search.SearchResponseMetrics.RESPONSE_COUNT_TOTAL_COUNTER_NAME;
 import static org.elasticsearch.rest.action.search.SearchResponseMetrics.RESPONSE_COUNT_TOTAL_STATUS_ATTRIBUTE_NAME;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertScrollResponsesAndHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHitsWithoutFailures;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -239,4 +242,48 @@ public void testScrollWithAllShardsFail() throws Exception {
             );
         });
     }
+
+    public void testAdditionalSearchAttributes() {
+        // target index attribute contains the system index name
+        createIndex(".kibana");
+        {
+            SearchResponse searchResponse = client().prepareSearch(".kibana").setQuery(simpleQueryStringQuery("foo")).get();
+            try {
+                assertNoFailures(searchResponse);
+                assertSearchHits(searchResponse);
+            } finally {
+                searchResponse.decRef();
+            }
+            List<Measurement> measurements = getTestTelemetryPlugin().getLongCounterMeasurement(RESPONSE_COUNT_TOTAL_COUNTER_NAME);
+            assertEquals(1, measurements.size());
+            Measurement measurement = measurements.getFirst();
+            assertEquals(1, measurement.getLong());
+            Map<String, Object> attributes = measurement.attributes();
+            assertEquals(4, attributes.size());
+            assertEquals(".kibana", attributes.get("target"));
+            assertEquals("hits_only", attributes.get("query_type"));
+            assertEquals("_score", attributes.get("sort"));
+            assertEquals("success", attributes.get(RESPONSE_COUNT_TOTAL_STATUS_ATTRIBUTE_NAME));
+        }
+        // target index attribute should be "user"
+        createIndex("custom-index");
+        {
+            SearchResponse searchResponse = client().prepareSearch("custom*").setQuery(simpleQueryStringQuery("foo")).get();
+            try {
+                assertNoFailures(searchResponse);
+                assertSearchHits(searchResponse);
+            } finally {
+                searchResponse.decRef();
+            }
+            List<Measurement> measurements = getTestTelemetryPlugin().getLongCounterMeasurement(RESPONSE_COUNT_TOTAL_COUNTER_NAME);
+            assertEquals(2, measurements.size());
+            Measurement measurement = measurements.getLast();
+            Map<String, Object> attributes = measurement.attributes();
+            assertEquals(4, attributes.size());
+            assertEquals("user", attributes.get("target"));
+            assertEquals("hits_only", attributes.get("query_type"));
+            assertEquals("_score", attributes.get("sort"));
+            assertEquals("success", attributes.get(RESPONSE_COUNT_TOTAL_STATUS_ATTRIBUTE_NAME));
+        }
+    }
 }

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java

@@ -66,6 +66,8 @@ public abstract class GenerativeRestTest extends ESRestTestCase implements Query
         "The incoming YAML document exceeds the limit:", // still to investigate, but it seems to be specific to the test framework
         "Data too large", // Circuit breaker exceptions eg. https://github.com/elastic/elasticsearch/issues/130072
         "optimized incorrectly due to missing references", // https://github.com/elastic/elasticsearch/issues/131509
+        "long overflow", // https://github.com/elastic/elasticsearch/issues/135759
+        "can't build page out of released blocks", // https://github.com/elastic/elasticsearch/issues/135679
 
         // Awaiting fixes for correctness
         "Expecting at most \\[.*\\] columns, got \\[.*\\]", // https://github.com/elastic/elasticsearch/issues/129561

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/EsqlQueryGenerator.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.xpack.esql.generator.command.pipe.EvalGenerator;
 import org.elasticsearch.xpack.esql.generator.command.pipe.ForkGenerator;
 import org.elasticsearch.xpack.esql.generator.command.pipe.GrokGenerator;
+import org.elasticsearch.xpack.esql.generator.command.pipe.InlineStatsGenerator;
 import org.elasticsearch.xpack.esql.generator.command.pipe.KeepGenerator;
 import org.elasticsearch.xpack.esql.generator.command.pipe.LimitGenerator;
 import org.elasticsearch.xpack.esql.generator.command.pipe.LookupJoinGenerator;
@@ -67,6 +68,7 @@ public class EsqlQueryGenerator {
         ForkGenerator.INSTANCE,
         GrokGenerator.INSTANCE,
         KeepGenerator.INSTANCE,
+        InlineStatsGenerator.INSTANCE,
         LimitGenerator.INSTANCE,
         LookupJoinGenerator.INSTANCE,
         MvExpandGenerator.INSTANCE,
@@ -341,23 +343,43 @@ yield switch ((randomIntBetween(0, 2))) {
     }
 
     public static String agg(List<Column> previousOutput) {
-        String name = randomNumericOrDateField(previousOutput);
+        String name = randomNumericField(previousOutput);
+        // complex with numerics
         if (name != null && randomBoolean()) {
-            // numerics only
-            return switch (randomIntBetween(0, 1)) {
-                case 0 -> "max(" + name + ")";
-                default -> "min(" + name + ")";
-                // TODO more numerics
-            };
+            int ops = randomIntBetween(1, 3);
+            StringBuilder result = new StringBuilder();
+            for (int i = 0; i < ops; i++) {
+                if (i > 0) {
+                    result.append(" + ");
+                }
+                String agg = switch (randomIntBetween(0, 5)) {
+                    case 0 -> "max(" + name + ")";
+                    case 1 -> "min(" + name + ")";
+                    case 2 -> "avg(" + name + ")";
+                    case 3 -> "median(" + name + ")";
+                    case 4 -> "sum(" + name + ")";
+                    default -> "count(" + name + ")";
+                    // TODO more numerics
+                };
+                result.append(agg);
+            }
+            return result.toString();
         }
         // all types
         name = randomBoolean() ? randomStringField(previousOutput) : randomNumericOrDateField(previousOutput);
         if (name == null) {
             return "count(*)";
         }
-        return switch (randomIntBetween(0, 2)) {
+        if (randomBoolean()) {
+            String exp = expression(previousOutput, false);
+            name = exp == null ? name : exp;
+        }
+        return switch (randomIntBetween(0, 5)) {
             case 0 -> "count(*)";
             case 1 -> "count(" + name + ")";
+            case 2 -> "absent(" + name + ")";
+            case 3 -> "present(" + name + ")";
+            case 4 -> "values(" + name + ")";
             default -> "count_distinct(" + name + ")";
         };
     }
@@ -414,9 +436,50 @@ public static String randomName(List<Column> cols, Set<String> allowedTypes) {
         return items.get(randomIntBetween(0, items.size() - 1));
     }
 
-    public static String expression(List<Column> previousOutput) {
-        // TODO improve!!!
-        return constantExpression();
+    /**
+     * @param previousOutput columns that can be used in the expression
+     * @param allowConstants if set to true, this will never return a constant expression.
+     *                       If no expression can be generated, it will return null
+     * @return an expression or null
+     */
+    public static String expression(List<Column> previousOutput, boolean allowConstants) {
+        if (randomBoolean() && allowConstants) {
+            return constantExpression();
+        }
+        if (randomBoolean()) {
+            StringBuilder result = new StringBuilder();
+            for (int i = 0; i < randomIntBetween(1, 3); i++) {
+                String field = randomNumericField(previousOutput);
+                if (field == null) {
+                    return allowConstants ? constantExpression() : null;
+                }
+                if (i > 0) {
+                    result.append(" + ");
+                }
+                result.append(field);
+            }
+            return result.toString();
+        }
+        if (randomBoolean()) {
+            String field = randomKeywordField(previousOutput);
+            if (field == null) {
+                return allowConstants ? constantExpression() : null;
+            }
+            return switch (randomIntBetween(0, 3)) {
+                case 0 -> "substring(" + field + ", 1, 3)";
+                case 1 -> "to_lower(" + field + ")";
+                case 2 -> "to_upper(" + field + ")";
+                default -> "length(" + field + ")";
+            };
+        }
+        if (randomBoolean() || allowConstants == false) {
+            String field = randomStringField(previousOutput);
+            if (field == null || randomBoolean()) {
+                field = randomNumericOrDateField(previousOutput);
+            }
+            return field;
+        }
+        return allowConstants ? constantExpression() : null;
     }
 
     public static String indexPattern(String indexName) {

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EvalGenerator.java

@@ -15,6 +15,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.test.ESTestCase.randomBoolean;
 import static org.elasticsearch.test.ESTestCase.randomIntBetween;
@@ -34,6 +35,7 @@ public CommandDescription generate(
     ) {
         StringBuilder cmd = new StringBuilder(" | eval ");
         int nFields = randomIntBetween(1, 10);
+        Map<String, Column> usablePrevious = previousOutput.stream().collect(Collectors.toMap(Column::name, c -> c));
         // TODO pass newly created fields to next expressions
         var newColumns = new ArrayList<>();
         for (int i = 0; i < nFields; i++) {
@@ -46,7 +48,7 @@ public CommandDescription generate(
                     name = EsqlQueryGenerator.randomIdentifier();
                 }
             }
-            String expression = EsqlQueryGenerator.expression(previousOutput);
+            String expression = EsqlQueryGenerator.expression(usablePrevious.values().stream().toList(), true);
             if (i > 0) {
                 cmd.append(",");
             }
@@ -56,6 +58,11 @@ public CommandDescription generate(
             newColumns.add(unquote(name));
             cmd.append(" = ");
             cmd.append(expression);
+
+            // there could be collisions in many ways, remove all of them
+            usablePrevious.remove(name);
+            usablePrevious.remove("`" + name + "`");
+            usablePrevious.remove(unquote(name));
         }
         String cmdString = cmd.toString();
         return new CommandDescription(EVAL, this, cmdString, Map.ofEntries(Map.entry(NEW_COLUMNS, newColumns)));

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/InlineStatsGenerator.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.generator.command.pipe;
+
+import org.elasticsearch.xpack.esql.generator.Column;
+import org.elasticsearch.xpack.esql.generator.command.CommandGenerator;
+
+import java.util.List;
+
+public class InlineStatsGenerator extends StatsGenerator {
+    public static final String INLINE_STATS = "inline stats";
+    public static final CommandGenerator INSTANCE = new InlineStatsGenerator();
+
+    @Override
+    public String commandName() {
+        return INLINE_STATS;
+    }
+
+    @Override
+    public ValidationResult validateOutput(
+        List<CommandDescription> previousCommands,
+        CommandDescription commandDescription,
+        List<Column> previousColumns,
+        List<List<Object>> previousOutput,
+        List<Column> columns,
+        List<List<Object>> output
+    ) {
+        if (commandDescription == EMPTY_DESCRIPTION) {
+            return VALIDATION_OK;
+        }
+
+        int prevCols = previousColumns.size();
+
+        if (previousColumns.stream().anyMatch(x -> x.name().equals("<all-fields-projected>"))) {
+            // known bug https://github.com/elastic/elasticsearch/issues/121741
+            prevCols--;
+        }
+
+        if (prevCols > columns.size()) {
+            return new ValidationResult(false, "Expecting at least [" + prevCols + "] columns, got [" + columns.size() + "]");
+        }
+        return VALIDATION_OK;
+    }
+}

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/StatsGenerator.java

@@ -24,6 +24,10 @@ public class StatsGenerator implements CommandGenerator {
     public static final String STATS = "stats";
     public static final CommandGenerator INSTANCE = new StatsGenerator();
 
+    public String commandName() {
+        return STATS;
+    }
+
     @Override
     public CommandDescription generate(
         List<CommandDescription> previousCommands,
@@ -38,7 +42,9 @@ public CommandDescription generate(
         if (nonNull.isEmpty()) {
             return EMPTY_DESCRIPTION;
         }
-        StringBuilder cmd = new StringBuilder(" | stats ");
+        StringBuilder cmd = new StringBuilder(" | ");
+        cmd.append(commandName());
+        cmd.append(" ");
         int nStats = randomIntBetween(1, 5);
         for (int i = 0; i < nStats; i++) {
             String name;
@@ -65,7 +71,7 @@ public CommandDescription generate(
                 cmd.append(" by " + col);
             }
         }
-        return new CommandDescription(STATS, this, cmd.toString(), Map.of());
+        return new CommandDescription(commandName(), this, cmd.toString(), Map.of());
     }
 
     @Override

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/ForkIT.java

@@ -32,7 +32,6 @@ public class ForkIT extends AbstractEsqlIntegTestCase {
 
     @Before
     public void setupIndex() {
-        assumeTrue("requires FORK capability", EsqlCapabilities.Cap.FORK_V9.isEnabled());
         createAndPopulateIndices();
     }