Merge branch 'main' into enable-generic-vector-formats

Author: thecoop

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/MockApmServer.java

@@ -114,8 +114,26 @@ public void stop() {
     }
 
     class RootHandler implements HttpHandler {
+        // checked by APM agent to identify the APM server version to adjust its behavior accordingly
+        private static final String FAKE_VERSION = """
+            {
+              "build_date": "2021-12-18T19:59:06Z",
+              "build_sha": "24fe620eeff5a19e2133c940c7e5ce1ceddb1445",
+              "publish_ready": true,
+              "version": "9.0.0"
+            }
+            """;
+
         public void handle(HttpExchange t) {
             try {
+                if ("GET".equals(t.getRequestMethod()) && "/".equals(t.getRequestURI().getPath())) {
+                    t.sendResponseHeaders(200, FAKE_VERSION.length());
+                    try (OutputStream os = t.getResponseBody()) {
+                        os.write(FAKE_VERSION.getBytes());
+                    }
+                    return;
+                }
+
                 InputStream body = t.getRequestBody();
                 if (metricFilter == null && transactionFilter == null) {
                     logRequestBody(body);

docs/changelog/137966.yaml

@@ -0,0 +1,5 @@
+pr: 137966
+summary: Allows PIT to be cross project
+area: Search
+type: enhancement
+issues: []

docs/changelog/138631.yaml

@@ -0,0 +1,5 @@
+pr: 138631
+summary: Improved bulk loading for binary doc values
+area: Codec
+type: enhancement
+issues: []

docs/changelog/138681.yaml

@@ -0,0 +1,5 @@
+pr: 138681
+summary: Field caps to support `project_routing` also in the body of the request
+area: Search
+type: enhancement
+issues: []

modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java

@@ -192,10 +192,14 @@ public void startTrace(TraceContext traceContext, Traceable traceable, String sp
             }
 
             final Span span = spanBuilder.startSpan();
-            // If not a root span (meaning a local parent exists) and the agent decided not to record the span, discard it immediately.
-            // Root spans (transactions), however, have to be kept to correctly report their duration.
-            if (localParentContext != null && span.isRecording() == false) {
-                logger.trace("Span [{}] [{}] will not be recorded due to transaction_max_spans reached", spanId, spanName);
+            if (span.isRecording() == false) {
+                if (localParentContext == null) {
+                    // this root span (transactions) is dropped due to sampling; the agent might report these when connected to
+                    // very old versions of apm server, however (with an incorrect duration)
+                    logger.trace("Root span [{}] [{}] will not be recorded due to sampling", spanId, spanName);
+                } else {
+                    logger.trace("Span [{}] [{}] will not be recorded due to transaction_max_spans reached", spanId, spanName);
+                }
                 span.end(); // end span immediately to release any resources.
                 return null; // return null to discard and not record in map of spans
             }

modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracerTests.java

@@ -109,9 +109,7 @@ public void test_onTraceStarted_ifNotRecorded_doesNotStartTracing() {
         apmTracer.startTrace(traceContext, TRACEABLE1, "name1_discard", null);
 
         assertThat(traceContext.getTransient(Task.APM_TRACE_CONTEXT), nullValue());
-        // the root span (transaction) is tracked
-        assertThat(apmTracer.getSpans(), aMapWithSize(1));
-        assertThat(apmTracer.getSpans(), hasKey(TRACEABLE1.getSpanId()));
+        assertThat(apmTracer.getSpans(), anEmptyMap());
     }
 
     /**

modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java

@@ -119,16 +119,6 @@ public void testValidateInvalidFieldType() {
         );
     }
 
-    public void testValidateNotIndexed() {
-        Exception e = expectThrows(IllegalArgumentException.class, () -> createMapperService(timestampMapping(true, b -> {
-            b.startObject("@timestamp");
-            b.field("type", "date");
-            b.field("index", false);
-            b.endObject();
-        })));
-        assertThat(e.getMessage(), equalTo("data stream timestamp field [@timestamp] is not indexed"));
-    }
-
     public void testValidateNotDocValues() {
         Exception e = expectThrows(IllegalArgumentException.class, () -> createMapperService(timestampMapping(true, b -> {
             b.startObject("@timestamp");

modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java

@@ -671,4 +671,9 @@ protected IngestScriptSupport ingestScriptSupport() {
     protected List<SortShortcutSupport> getSortShortcutSupport() {
         return List.of();
     }
+
+    @Override
+    protected boolean supportsDocValuesSkippers() {
+        return false;
+    }
 }

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java

@@ -193,12 +193,23 @@ protected Parameter<?>[] getParameters() {
             return new Parameter<?>[] { indexed, hasDocValues, stored, ignoreMalformed, meta, scalingFactor, coerce, nullValue, metric };
         }
 
+        private IndexType indexType() {
+            if (indexed.getValue()) {
+                return IndexType.points(true, hasDocValues.getValue());
+            }
+            if (hasDocValues.getValue()
+                && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.STANDARD_INDEXES_USE_SKIPPERS)
+                && indexSettings.useDocValuesSkipper()) {
+                return IndexType.skippers();
+            }
+            return IndexType.points(false, hasDocValues.getValue());
+        }
+
         @Override
         public ScaledFloatFieldMapper build(MapperBuilderContext context) {
-            IndexType indexType = IndexType.points(indexed.get(), hasDocValues.get());
             ScaledFloatFieldType type = new ScaledFloatFieldType(
                 context.buildFullName(leafName()),
-                indexType,
+                indexType(),
                 stored.getValue(),
                 meta.getValue(),
                 scalingFactor.getValue(),

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java

@@ -390,4 +390,9 @@ public void testLoadSyntheticSourceFromStringOrBytesRef() throws IOException {
     protected List<SortShortcutSupport> getSortShortcutSupport() {
         return List.of();
     }
+
+    @Override
+    protected boolean supportsDocValuesSkippers() {
+        return false;
+    }
 }