Merge remote-tracking branch 'es/main' into logsdb_rolling_upgrade_tests_in_serverless

martijnvg · martijnvg · commit d77796ff9748 · 2025-11-28T19:16:16.000+01:00
diff --git a/build-conventions/settings.gradle b/build-conventions/settings.gradle
@@ -8,7 +8,7 @@
  */
 
 plugins {
-  id "com.gradle.develocity" version "4.1.1"
+  id "com.gradle.develocity" version "4.2.2"
 }
 
 rootProject.name = 'build-conventions'
diff --git a/build-tools-internal/settings.gradle b/build-tools-internal/settings.gradle
@@ -9,7 +9,7 @@ pluginManagement {
 }
 
 plugins {
-  id "com.gradle.develocity" version "4.1.1"
+  id "com.gradle.develocity" version "4.2.2"
 }
 
 dependencyResolutionManagement {
diff --git a/build-tools/settings.gradle b/build-tools/settings.gradle
@@ -10,7 +10,7 @@ pluginManagement {
     includeBuild "../build-conventions"
 }
 plugins {
-  id "com.gradle.develocity" version "4.1.1"
+  id "com.gradle.develocity" version "4.2.2"
 }
 include 'reaper'
 
diff --git a/docs/reference/elasticsearch/mapping-reference/mapping-source-field.md b/docs/reference/elasticsearch/mapping-reference/mapping-source-field.md
@@ -356,6 +356,9 @@ An expert-only feature is the ability to prune the contents of the `_source` fie
 Removing fields from the `_source` has similar downsides to disabling `_source`, especially the fact that you cannot reindex documents from one Elasticsearch index to another. Consider using [source filtering](/reference/elasticsearch/rest-apis/retrieve-selected-fields.md#source-filtering) instead.
 ::::
 
+::::{note}
+Source pruning is not available in {{serverless-short}}
+::::
 
 The `includes`/`excludes` parameters (which also accept wildcards) can be used as follows:
 
diff --git a/gradle/build.versions.toml b/gradle/build.versions.toml
@@ -17,7 +17,7 @@ checkstyle = "com.puppycrawl.tools:checkstyle:11.0.1"
 commmons-io = "commons-io:commons-io:2.2"
 docker-compose = "com.avast.gradle:gradle-docker-compose-plugin:0.17.5"
 forbiddenApis = "de.thetaphi:forbiddenapis:3.8"
-gradle-enterprise = "com.gradle:develocity-gradle-plugin:4.1.1"
+gradle-enterprise = "com.gradle:develocity-gradle-plugin:4.2.2"
 hamcrest = "org.hamcrest:hamcrest:3.0"
 httpcore5 = "org.apache.httpcomponents.core5:httpcore5:5.3.3"
 httpclient5 = "org.apache.httpcomponents.client5:httpclient5:5.4.2"
diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
@@ -964,9 +964,9 @@
             <sha256 value="48234cd74e35d91a31a683820a35b5b6d11b55527f32a5b162c6757408b95d7a" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.gradle" name="develocity-gradle-plugin" version="4.1.1">
-         <artifact name="develocity-gradle-plugin-4.1.1.jar">
-            <sha256 value="21a509cfff19aad0eb23dcc2a1979345708ffa61d6f4488f8956caae51cb55ff" origin="Generated by Gradle"/>
+      <component group="com.gradle" name="develocity-gradle-plugin" version="4.2.2">
+         <artifact name="develocity-gradle-plugin-4.2.2.jar">
+            <sha256 value="3e45a8c18b34a8dddeb185c3e7030728634aaf31b9762052e5dae87f3d597fba" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.gradleup.gratatouille" name="gratatouille-runtime" version="0.0.8">
diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParser.java
@@ -225,4 +225,12 @@ <T> Map<String, T> map(Supplier<Map<String, T>> mapFactory, CheckedFunction<XCon
      * The callback to notify when parsing encounters a deprecated field.
      */
     DeprecationHandler getDeprecationHandler();
+
+    /**
+     * Switch to a different underlying parser.
+     * Typically, that's a noop but some filter parsers might want to wrap the underlying parser again.
+     */
+    default XContentParser switchParser(XContentParser parser) throws IOException {
+        return parser;
+    }
 }
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DisabledSecurityDataStreamTestCase.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DisabledSecurityDataStreamTestCase.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.FeatureFlag;
+import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.junit.ClassRule;
@@ -24,12 +25,15 @@
  */
 public abstract class DisabledSecurityDataStreamTestCase extends ESRestTestCase {
 
+    protected static LocalClusterConfigProvider clusterConfig = c -> {};
+
     @ClassRule
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .distribution(DistributionType.DEFAULT)
         .setting("xpack.security.enabled", "false")
         .setting("xpack.watcher.enabled", "false")
         .feature(FeatureFlag.INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG)
+        .apply(() -> clusterConfig) // allows sub-classes to apply a custom cluster configuration
         .build();
 
     @Override
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java
@@ -8,12 +8,15 @@
  */
 package org.elasticsearch.datastreams;
 
+import org.elasticsearch.Build;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.DateFormatters;
 import org.elasticsearch.common.time.FormatNames;
 import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.rest.ObjectPath;
 import org.junit.Before;
 
@@ -25,6 +28,7 @@
 import java.util.Set;
 
 import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.backingIndexEqualTo;
+import static org.elasticsearch.index.IndexSettings.USE_SYNTHETIC_ID;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
@@ -36,6 +40,10 @@
 
 public class TsdbDataStreamRestIT extends DisabledSecurityDataStreamTestCase {
 
+    static {
+        clusterConfig = config -> config.feature(FeatureFlag.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
+    }
+
     private static final String COMPONENT_TEMPLATE = """
         {
             "template": {
@@ -53,7 +61,7 @@ public class TsdbDataStreamRestIT extends DisabledSecurityDataStreamTestCase {
                         "number_of_replicas": 1,
                         "number_of_shards": 2,
                         "mode": "time_series"
-                        SOURCEMODE
+                        RANDOM_INDEX_SETTINGS
                     }
                 },
                 "mappings":{
@@ -202,9 +210,51 @@ public class TsdbDataStreamRestIT extends DisabledSecurityDataStreamTestCase {
             """;
 
     private static String getTemplate() {
-        return TEMPLATE.replace("SOURCEMODE", randomFrom("", """
-            , "mapping": { "source": { "mode": "stored" } }""", """
-            , "mapping": { "source": { "mode": "synthetic" } }"""));
+        String sourceMode = switch (randomInt(2)) {
+            case 0 -> null;
+            case 1 -> """
+                "source": { "mode": "stored" }
+                """;
+            case 2 -> """
+                "source": { "mode": "synthetic" }
+                """;
+            default -> throw new AssertionError("Unknown mode");
+        };
+        String idMode = null;
+        if (Build.current().isSnapshot()) {
+            idMode = switch (randomInt(2)) {
+                case 0 -> null;
+                case 1 -> """
+                    "use_synthetic_id": "false"
+                    """;
+                case 2 -> """
+                    "use_synthetic_id": "true"
+                    """;
+                default -> throw new AssertionError("Unknown mode");
+            };
+        } else {
+            assertFalse(
+                "Setting is enabled by default and must now be tested on non-snapshot build too ",
+                USE_SYNTHETIC_ID.getDefault(Settings.EMPTY)
+            );
+        }
+        if (sourceMode == null && idMode == null) {
+            return TEMPLATE.replace("RANDOM_INDEX_SETTINGS", "");
+        }
+        var mapping = new StringBuilder("""
+            , "mapping": {
+            """);
+        if (sourceMode != null) {
+            mapping.append(sourceMode);
+        }
+        if (idMode != null) {
+            if (sourceMode != null) {
+                mapping.append(',');
+            }
+            mapping.append(idMode);
+        }
+        mapping.append('}');
+        return TEMPLATE.replace("RANDOM_INDEX_SETTINGS", mapping.toString());
     }
 
     private static boolean trialStarted = false;
diff --git a/muted-tests.yml b/muted-tests.yml
@@ -445,6 +445,15 @@ tests:
 - class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
   method: test {p0=search/160_exists_query/Test exists query on object field in empty index}
   issue: https://github.com/elastic/elasticsearch/issues/138733
+- class: org.elasticsearch.repositories.blobstore.testkit.analyze.S3RepositoryAnalysisStrongMpusRestIT
+  method: testRepositoryAnalysis
+  issue: https://github.com/elastic/elasticsearch/issues/138762
+- class: org.elasticsearch.xpack.inference.integration.AuthorizationTaskExecutorIT
+  method: testCreatesEisChatCompletionEndpoint
+  issue: https://github.com/elastic/elasticsearch/issues/138764
+- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT
+  method: test {yaml=index/100_field_name_length_limit/Test field name length limit}
+  issue: https://github.com/elastic/elasticsearch/issues/138768
 
 # Examples:
 #
diff --git a/plugins/examples/settings.gradle b/plugins/examples/settings.gradle
@@ -8,7 +8,7 @@
  */
 
 plugins {
-  id "com.gradle.develocity" version "4.1.1"
+  id "com.gradle.develocity" version "4.2.2"
 }
 
 // Include all subdirectories as example projects
diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java
@@ -445,6 +445,7 @@
 
     provides org.apache.lucene.codecs.PostingsFormat
         with
+            org.elasticsearch.index.codec.tsdb.TSDBSyntheticIdPostingsFormat,
             org.elasticsearch.index.codec.bloomfilter.ES85BloomFilterPostingsFormat,
             org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat,
             org.elasticsearch.index.codec.postings.ES812PostingsFormat;
diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/diskbbq/next/ESNextDiskBBQVectorsWriter.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/diskbbq/next/ESNextDiskBBQVectorsWriter.java
@@ -37,7 +37,6 @@
 import org.elasticsearch.index.codec.vectors.diskbbq.QuantizedVectorValues;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
-import org.elasticsearch.simdvec.ES91OSQVectorsScorer;
 import org.elasticsearch.simdvec.ES92Int7VectorsScorer;
 import org.elasticsearch.simdvec.ESNextOSQVectorsScorer;
 
@@ -120,7 +119,11 @@ public CentroidOffsetAndLength buildAndWritePostingsLists(
         // write the posting lists
         final PackedLongValues.Builder offsets = PackedLongValues.monotonicBuilder(PackedInts.COMPACT);
         final PackedLongValues.Builder lengths = PackedLongValues.monotonicBuilder(PackedInts.COMPACT);
-        DiskBBQBulkWriter bulkWriter = DiskBBQBulkWriter.fromBitSize(quantEncoding.bits(), ES91OSQVectorsScorer.BULK_SIZE, postingsOutput);
+        DiskBBQBulkWriter bulkWriter = DiskBBQBulkWriter.fromBitSize(
+            quantEncoding.bits(),
+            ESNextOSQVectorsScorer.BULK_SIZE,
+            postingsOutput
+        );
         OnHeapQuantizedVectors onHeapQuantizedVectors = new OnHeapQuantizedVectors(
             floatVectorValues,
             quantEncoding,
@@ -156,11 +159,16 @@ public CentroidOffsetAndLength buildAndWritePostingsLists(
                 docDeltas[j] = j == 0 ? docIds[clusterOrds[j]] : docIds[clusterOrds[j]] - docIds[clusterOrds[j - 1]];
             }
             onHeapQuantizedVectors.reset(centroid, size, ord -> cluster[clusterOrds[ord]]);
-            byte encoding = idsWriter.calculateBlockEncoding(i -> docDeltas[i], size, ES91OSQVectorsScorer.BULK_SIZE);
+            byte encoding = idsWriter.calculateBlockEncoding(i -> docDeltas[i], size, ESNextOSQVectorsScorer.BULK_SIZE);
             postingsOutput.writeByte(encoding);
             bulkWriter.writeVectors(onHeapQuantizedVectors, i -> {
                 // for vector i we write `bulk` size docs or the remaining docs
-                idsWriter.writeDocIds(d -> docDeltas[i + d], Math.min(ES91OSQVectorsScorer.BULK_SIZE, size - i), encoding, postingsOutput);
+                idsWriter.writeDocIds(
+                    d -> docDeltas[i + d],
+                    Math.min(ESNextOSQVectorsScorer.BULK_SIZE, size - i),
+                    encoding,
+                    postingsOutput
+                );
             });
             lengths.add(postingsOutput.getFilePointer() - fileOffset - offset);
         }
@@ -275,7 +283,7 @@ public CentroidOffsetAndLength buildAndWritePostingsLists(
             );
             DiskBBQBulkWriter bulkWriter = DiskBBQBulkWriter.fromBitSize(
                 quantEncoding.bits(),
-                ES91OSQVectorsScorer.BULK_SIZE,
+                ESNextOSQVectorsScorer.BULK_SIZE,
                 postingsOutput
             );
             final ByteBuffer buffer = ByteBuffer.allocate(fieldInfo.getVectorDimension() * Float.BYTES).order(ByteOrder.LITTLE_ENDIAN);
@@ -308,7 +316,7 @@ public CentroidOffsetAndLength buildAndWritePostingsLists(
                 for (int j = 0; j < size; j++) {
                     docDeltas[j] = j == 0 ? docIds[clusterOrds[j]] : docIds[clusterOrds[j]] - docIds[clusterOrds[j - 1]];
                 }
-                byte encoding = idsWriter.calculateBlockEncoding(i -> docDeltas[i], size, ES91OSQVectorsScorer.BULK_SIZE);
+                byte encoding = idsWriter.calculateBlockEncoding(i -> docDeltas[i], size, ESNextOSQVectorsScorer.BULK_SIZE);
                 postingsOutput.writeByte(encoding);
                 offHeapQuantizedVectors.reset(size, ord -> isOverspill[clusterOrds[ord]], ord -> cluster[clusterOrds[ord]]);
                 // write vectors
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DotExpandingXContentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DotExpandingXContentParser.java
@@ -35,10 +35,6 @@
  */
 class DotExpandingXContentParser extends FilterXContentParserWrapper {
 
-    static boolean isInstance(XContentParser parser) {
-        return parser instanceof WrappingParser;
-    }
-
     private static final class WrappingParser extends FilterXContentParser {
 
         private final ContentPath contentPath;
@@ -52,6 +48,11 @@ private static final class WrappingParser extends FilterXContentParser {
             }
         }
 
+        @Override
+        public XContentParser switchParser(XContentParser parser) throws IOException {
+            return new WrappingParser(parser, contentPath);
+        }
+
         @Override
         public Token nextToken() throws IOException {
             Token token;
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java
@@ -297,10 +297,9 @@ private static DocumentParserContext cloneDocumentParserContext(
             BytesReference.bytes(builder),
             context.parser().contentType()
         );
-        if (DotExpandingXContentParser.isInstance(context.parser())) {
-            // If we performed dot expanding originally we need to continue to do so when we replace the parser.
-            newParser = DotExpandingXContentParser.expandDots(newParser, context.path());
-        }
+        // If we performed dot expanding originally we need to continue to do so when we replace the parser.
+        // To reliably do that, we need to delegate switching to the parser itself as it might be decorated / wrapped.
+        newParser = context.parser().switchParser(newParser);
 
         DocumentParserContext subcontext = context.switchParser(newParser);
         subcontext.setRecordedSource();  // Avoids double-storing parts of the source for the same parser subtree.
diff --git a/settings.gradle b/settings.gradle
@@ -14,7 +14,7 @@ pluginManagement {
 }
 
 plugins {
-  id "com.gradle.develocity" version "4.1.1"
+  id "com.gradle.develocity" version "4.2.2"
   id 'elasticsearch.java-toolchain'
   id 'elasticsearch.component-metadata-rules'
 }
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java
@@ -27,7 +27,8 @@ public enum FeatureFlag {
     ),
     RANDOM_SAMPLING("es.random_sampling_feature_flag_enabled=true", Version.fromString("9.2.0"), null),
     INFERENCE_API_CCM("es.inference_api_ccm_feature_flag_enabled=true", Version.fromString("9.3.0"), null),
-    GENERIC_VECTOR_FORMAT("es.generic_vector_format_feature_flag_enabled=true", Version.fromString("9.3.0"), null);
+    GENERIC_VECTOR_FORMAT("es.generic_vector_format_feature_flag_enabled=true", Version.fromString("9.3.0"), null),
+    TSDB_SYNTHETIC_ID_FEATURE_FLAG("es.tsdb_synthetic_id_feature_flag_enabled=true", Version.fromString("9.3.0"), null);
 
     public final String systemProperty;
     public final Version from;

Original file line number	Diff line number	Diff line change
`@@ -8,7 +8,7 @@`
`8`	`8`	`*/`
`9`	`9`
`10`	`10`	`plugins {`
`11`		`- id "com.gradle.develocity" version "4.1.1"`
	`11`	`+ id "com.gradle.develocity" version "4.2.2"`
`12`	`12`	`}`
`13`	`13`
`14`	`14`	`rootProject.name = 'build-conventions'`
Original file line number	Diff line number	Diff line change
`@@ -9,7 +9,7 @@ pluginManagement {`
`9`	`9`	`}`
`10`	`10`
`11`	`11`	`plugins {`
`12`		`- id "com.gradle.develocity" version "4.1.1"`
	`12`	`+ id "com.gradle.develocity" version "4.2.2"`
`13`	`13`	`}`
`14`	`14`
`15`	`15`	`dependencyResolutionManagement {`
Original file line number	Diff line number	Diff line change
`@@ -10,7 +10,7 @@ pluginManagement {`
`10`	`10`	`includeBuild "../build-conventions"`
`11`	`11`	`}`
`12`	`12`	`plugins {`
`13`		`- id "com.gradle.develocity" version "4.1.1"`
	`13`	`+ id "com.gradle.develocity" version "4.2.2"`
`14`	`14`	`}`
`15`	`15`	`include 'reaper'`
`16`	`16`