Merge branch 'main' into fix_136365_prune_columns_when_fork

Author: pmpailis

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java

@@ -92,7 +92,8 @@ public void setup() {
             System.nanoTime(),
             false,
             AnalyzerSettings.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
-            AnalyzerSettings.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.get(Settings.EMPTY)
+            AnalyzerSettings.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.get(Settings.EMPTY),
+            null
         );
 
         var fields = 10_000;

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -373,7 +373,8 @@ private static Configuration configuration() {
             0,
             false,
             AnalyzerSettings.QUERY_TIMESERIES_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
-            AnalyzerSettings.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY)
+            AnalyzerSettings.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
+            null
         );
     }
 

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/MockApmServer.java

@@ -114,8 +114,26 @@ public void stop() {
     }
 
     class RootHandler implements HttpHandler {
+        // checked by APM agent to identify the APM server version to adjust its behavior accordingly
+        private static final String FAKE_VERSION = """
+            {
+              "build_date": "2021-12-18T19:59:06Z",
+              "build_sha": "24fe620eeff5a19e2133c940c7e5ce1ceddb1445",
+              "publish_ready": true,
+              "version": "9.0.0"
+            }
+            """;
+
         public void handle(HttpExchange t) {
             try {
+                if ("GET".equals(t.getRequestMethod()) && "/".equals(t.getRequestURI().getPath())) {
+                    t.sendResponseHeaders(200, FAKE_VERSION.length());
+                    try (OutputStream os = t.getResponseBody()) {
+                        os.write(FAKE_VERSION.getBytes());
+                    }
+                    return;
+                }
+
                 InputStream body = t.getRequestBody();
                 if (metricFilter == null && transactionFilter == null) {
                     logRequestBody(body);

docs/changelog/137920.yaml

@@ -0,0 +1,6 @@
+pr: 137920
+summary: Avoid retrieving unnecessary fields on node-reduce phase
+area: "ES|QL"
+type: enhancement
+issues:
+ - 134363

docs/changelog/138457.yaml

@@ -0,0 +1,6 @@
+pr: 138457
+summary: Intercept filters to knn queries
+area: Vector Search
+type: bug
+issues:
+ - 138410

docs/changelog/138631.yaml

@@ -0,0 +1,5 @@
+pr: 138631
+summary: Improved bulk loading for binary doc values
+area: Codec
+type: enhancement
+issues: []

modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java

@@ -192,10 +192,14 @@ public void startTrace(TraceContext traceContext, Traceable traceable, String sp
             }
 
             final Span span = spanBuilder.startSpan();
-            // If not a root span (meaning a local parent exists) and the agent decided not to record the span, discard it immediately.
-            // Root spans (transactions), however, have to be kept to correctly report their duration.
-            if (localParentContext != null && span.isRecording() == false) {
-                logger.trace("Span [{}] [{}] will not be recorded due to transaction_max_spans reached", spanId, spanName);
+            if (span.isRecording() == false) {
+                if (localParentContext == null) {
+                    // this root span (transactions) is dropped due to sampling; the agent might report these when connected to
+                    // very old versions of apm server, however (with an incorrect duration)
+                    logger.trace("Root span [{}] [{}] will not be recorded due to sampling", spanId, spanName);
+                } else {
+                    logger.trace("Span [{}] [{}] will not be recorded due to transaction_max_spans reached", spanId, spanName);
+                }
                 span.end(); // end span immediately to release any resources.
                 return null; // return null to discard and not record in map of spans
             }

modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracerTests.java

@@ -109,9 +109,7 @@ public void test_onTraceStarted_ifNotRecorded_doesNotStartTracing() {
         apmTracer.startTrace(traceContext, TRACEABLE1, "name1_discard", null);
 
         assertThat(traceContext.getTransient(Task.APM_TRACE_CONTEXT), nullValue());
-        // the root span (transaction) is tracked
-        assertThat(apmTracer.getSpans(), aMapWithSize(1));
-        assertThat(apmTracer.getSpans(), hasKey(TRACEABLE1.getSpanId()));
+        assertThat(apmTracer.getSpans(), anEmptyMap());
     }
 
     /**

modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java

@@ -119,16 +119,6 @@ public void testValidateInvalidFieldType() {
         );
     }
 
-    public void testValidateNotIndexed() {
-        Exception e = expectThrows(IllegalArgumentException.class, () -> createMapperService(timestampMapping(true, b -> {
-            b.startObject("@timestamp");
-            b.field("type", "date");
-            b.field("index", false);
-            b.endObject();
-        })));
-        assertThat(e.getMessage(), equalTo("data stream timestamp field [@timestamp] is not indexed"));
-    }
-
     public void testValidateNotDocValues() {
         Exception e = expectThrows(IllegalArgumentException.class, () -> createMapperService(timestampMapping(true, b -> {
             b.startObject("@timestamp");

modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java

@@ -671,4 +671,9 @@ protected IngestScriptSupport ingestScriptSupport() {
     protected List<SortShortcutSupport> getSortShortcutSupport() {
         return List.of();
     }
+
+    @Override
+    protected boolean supportsDocValuesSkippers() {
+        return false;
+    }
 }