Merge branch '9.0' into fork/mark-vieira/backport/9.0/pr-125545

mark-vieira · mark-vieira · commit aea29afea2ca · 2025-03-27T09:17:03.000-07:00
# Conflicts:
#	build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java
@@ -60,8 +60,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:smoke-test-plugins-ssl");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:smoke-test-security-with-mustache");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:xpack-prefix-rest-compat");
-        map.put(LegacyRestTestBasePlugin.class, ":modules:ingest-geoip:qa:file-based-update");
-        map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-gce:qa:gce");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:multi-cluster-search-security:legacy-with-basic-license");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:multi-cluster-search-security:legacy-with-full-license");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:multi-cluster-search-security:legacy-with-restricted-trust");
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestPlugin.java
@@ -16,6 +16,7 @@
 import org.gradle.api.provider.Provider;
 import org.gradle.api.tasks.testing.Test;
 
+import java.io.File;
 import java.util.Arrays;
 import java.util.List;
 
@@ -26,21 +27,22 @@ public class MutedTestPlugin implements Plugin<Project> {
 
     @Override
     public void apply(Project project) {
+        project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class);
+        var buildParams = loadBuildParams(project).get();
+
+        File settingsRoot = project.getLayout().getSettingsDirectory().getAsFile();
         String additionalFilePaths = project.hasProperty(ADDITIONAL_FILES_PROPERTY)
             ? project.property(ADDITIONAL_FILES_PROPERTY).toString()
             : "";
         List<RegularFile> additionalFiles = Arrays.stream(additionalFilePaths.split(","))
             .filter(p -> p.isEmpty() == false)
-            .map(p -> project.getRootProject().getLayout().getProjectDirectory().file(p))
+            .map(p -> project.getLayout().getSettingsDirectory().file(p))
             .toList();
 
-        project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class);
-        var buildParams = loadBuildParams(project).get();
-
         Provider<MutedTestsBuildService> mutedTestsProvider = project.getGradle()
             .getSharedServices()
             .registerIfAbsent("mutedTests", MutedTestsBuildService.class, spec -> {
-                spec.getParameters().getInfoPath().set(project.getRootProject().getProjectDir());
+                spec.getParameters().getInfoPath().set(settingsRoot);
                 spec.getParameters().getAdditionalFiles().set(additionalFiles);
             });
 
diff --git a/docs/changelog/125666.yaml b/docs/changelog/125666.yaml
@@ -0,0 +1,5 @@
+pr: 125666
+summary: Minor-Fixes Support 7x segments as archive in 8x / 9x
+area: Search
+type: bug
+issues: []
diff --git a/docs/changelog/125716.yaml b/docs/changelog/125716.yaml
@@ -0,0 +1,5 @@
+pr: 125716
+summary: Return appropriate error on null dims update instead of npe
+area: Vector Search
+type: bug
+issues: []
diff --git a/docs/reference/release-notes/9.0.0.asciidoc b/docs/reference/release-notes/9.0.0.asciidoc
@@ -12,6 +12,9 @@ Also see <<breaking-changes-9.0,Breaking changes in 9.0>>.
 TLS::
 * Drop `TLS_RSA` cipher support for JDK 24 {es-pull}123600[#123600]
 
+Infra/Core::
+* Update the response format when detailed error information is disabled {es-pull}90529[#90529]
+
 [[bug-9.0.0]]
 [float]
 === Bug fixes
diff --git a/muted-tests.yml b/muted-tests.yml
@@ -261,6 +261,12 @@ tests:
 - class: org.elasticsearch.index.shard.StoreRecoveryTests
   method: testAddIndices
   issue: https://github.com/elastic/elasticsearch/issues/124104
+- class: org.elasticsearch.xpack.esql.spatial.SpatialExtentAggregationNoLicenseIT
+  method: testStExtentAggregationWithShapes
+  issue: https://github.com/elastic/elasticsearch/issues/125734
+- class: org.elasticsearch.smoketest.MlWithSecurityIT
+  method: test {yaml=ml/trained_model_cat_apis/Test cat trained models}
+  issue: https://github.com/elastic/elasticsearch/issues/125750
 
 # Examples:
 #
diff --git a/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java b/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java
@@ -17,8 +17,6 @@
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.Subject;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
-import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.PrivilegesToCheck;
-import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.PrivilegesCheckResult;
 import org.elasticsearch.xpack.core.security.authz.ResolvedIndices;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges;
@@ -34,12 +32,12 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.function.Supplier;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 /**
diff --git a/plugins/examples/security-authorization-engine/src/test/java/org/elasticsearch/example/CustomAuthorizationEngineTests.java b/plugins/examples/security-authorization-engine/src/test/java/org/elasticsearch/example/CustomAuthorizationEngineTests.java
@@ -11,8 +11,6 @@
 
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.cluster.metadata.IndexAbstraction;
-import org.elasticsearch.cluster.metadata.IndexAbstraction.ConcreteIndex;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.settings.Settings;
@@ -30,9 +28,6 @@
 import org.elasticsearch.xpack.core.security.user.User;
 
 import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.stream.Stream;
 
 import static org.hamcrest.Matchers.is;
 
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml
@@ -605,3 +605,28 @@ setup:
   - match: { hits.hits.0._score: $knn_score0 }
   - match: { hits.hits.1._score: $knn_score1 }
   - match: { hits.hits.2._score: $knn_score2 }
+---
+"Updating dim to null is not allowed":
+  - requires:
+      cluster_features: "mapper.npe_on_dims_update_fix"
+      reason: "dims update fix"
+  - do:
+      indices.create:
+        index: test_index
+
+  - do:
+      indices.put_mapping:
+        index: test_index
+        body:
+          properties:
+            embedding:
+              type: dense_vector
+              dims: 4
+  - do:
+      catch: bad_request
+      indices.put_mapping:
+        index: test_index
+        body:
+          properties:
+            embedding:
+              type: dense_vector
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
@@ -31,6 +31,7 @@ public class MapperFeatures implements FeatureSpecification {
     public static final NodeFeature SPARSE_VECTOR_STORE_SUPPORT = new NodeFeature("mapper.sparse_vector.store_support");
     public static final NodeFeature SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX = new NodeFeature("mapper.nested.sorting_fields_check_fix");
     public static final NodeFeature DYNAMIC_HANDLING_IN_COPY_TO = new NodeFeature("mapper.copy_to.dynamic_handling");
+    static final NodeFeature NPE_ON_DIMS_UPDATE_FIX = new NodeFeature("mapper.npe_on_dims_update_fix");
 
     @Override
     public Set<NodeFeature> getTestFeatures() {
@@ -51,7 +52,8 @@ public Set<NodeFeature> getTestFeatures() {
             DYNAMIC_HANDLING_IN_COPY_TO,
             SourceFieldMapper.SYNTHETIC_RECOVERY_SOURCE,
             ObjectMapper.SUBOBJECTS_FALSE_MAPPING_UPDATE_FIX,
-            DateFieldMapper.INVALID_DATE_FIX
+            DateFieldMapper.INVALID_DATE_FIX,
+            NPE_ON_DIMS_UPDATE_FIX
         );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java
@@ -144,7 +144,7 @@ public static class Builder extends FieldMapper.Builder {
             }
 
             return XContentMapValues.nodeIntegerValue(o);
-        }, m -> toType(m).fieldType().dims, XContentBuilder::field, Object::toString).setSerializerCheck((id, ic, v) -> v != null)
+        }, m -> toType(m).fieldType().dims, XContentBuilder::field, Objects::toString).setSerializerCheck((id, ic, v) -> v != null)
             .setMergeValidator((previous, current, c) -> previous == null || Objects.equals(previous, current))
             .addValidator(dims -> {
                 if (dims == null) {
diff --git a/test/external-modules/delayed-aggs/build.gradle b/test/external-modules/delayed-aggs/build.gradle
@@ -7,11 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-apply plugin: 'elasticsearch.legacy-yaml-rest-test'
-
-tasks.named('yamlRestTest').configure {
-  it.onlyIf("snapshot build") { buildParams.snapshotBuild }
-}
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
 
 esplugin {
   description = 'A test module that allows to delay aggregations on shards with a configurable time'
@@ -23,3 +19,8 @@ restResources {
     include '_common', 'indices', 'index', 'cluster', 'search'
   }
 }
+
+tasks.named('yamlRestTest') {
+  def isSnapshot = buildParams.snapshotBuild
+  it.onlyIf("snapshot build") { isSnapshot }
+}
diff --git a/test/external-modules/delayed-aggs/src/yamlRestTest/java/org/elasticsearch/search/aggregations/DelayedShardAggregationClientYamlTestSuiteIT.java b/test/external-modules/delayed-aggs/src/yamlRestTest/java/org/elasticsearch/search/aggregations/DelayedShardAggregationClientYamlTestSuiteIT.java
@@ -12,10 +12,16 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.ClassRule;
 
 public class DelayedShardAggregationClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("test-delayed-aggs").build();
+
     public DelayedShardAggregationClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
@@ -24,4 +30,9 @@ public DelayedShardAggregationClientYamlTestSuiteIT(@Name("yaml") ClientYamlTest
     public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
 }
diff --git a/test/external-modules/error-query/build.gradle b/test/external-modules/error-query/build.gradle
@@ -7,11 +7,10 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
-tasks.named('yamlRestTest').configure {
-  it.onlyIf("snapshot build") { buildParams.snapshotBuild }
-}
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.internal-java-rest-test'
 
 esplugin {
   description = 'A test module that exposes a way to simulate search shard failures and warnings'
@@ -23,3 +22,14 @@ restResources {
     include '_common', 'indices', 'index', 'cluster', 'search'
   }
 }
+
+dependencies {
+  clusterModules project(':x-pack:plugin:esql')
+  clusterModules project(':x-pack:plugin:autoscaling')
+  clusterModules project(':x-pack:plugin:ilm')
+}
+
+tasks.withType(StandaloneRestIntegTestTask).configureEach {
+  def isSnapshot = buildParams.snapshotBuild
+  onlyIf("snapshot build") { isSnapshot }
+}
diff --git a/test/external-modules/error-query/src/yamlRestTest/java/org/elasticsearch/search/query/ErrorQueryClientYamlTestSuiteIT.java b/test/external-modules/error-query/src/yamlRestTest/java/org/elasticsearch/search/query/ErrorQueryClientYamlTestSuiteIT.java
@@ -12,10 +12,16 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.ClassRule;
 
 public class ErrorQueryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("test-error-query").build();
+
     public ErrorQueryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
@@ -24,4 +30,9 @@ public ErrorQueryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate tes
     public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
 }
diff --git a/test/yaml-rest-runner/build.gradle b/test/yaml-rest-runner/build.gradle
@@ -1,6 +1,6 @@
 apply plugin: 'elasticsearch.build'
-apply plugin: 'elasticsearch.legacy-yaml-rest-test'
-apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.yaml-rest-compat-test'
 
 dependencies {
   api project(':test:framework')
diff --git a/test/yaml-rest-runner/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseFailLogIT.java b/test/yaml-rest-runner/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseFailLogIT.java
@@ -13,12 +13,17 @@
 
 import org.apache.logging.log4j.Level;
 import org.elasticsearch.test.MockLog;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.junit.ClassRule;
 
 import java.io.IOException;
 
 public class ESClientYamlSuiteTestCaseFailLogIT extends ESClientYamlSuiteTestCase {
 
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local().build();
+
     public ESClientYamlSuiteTestCaseFailLogIT(final ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
@@ -65,4 +70,9 @@ public void test() throws IOException {
             mockLog.assertAllExpectationsMatched();
         }
     }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
 }
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/RestDeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/RestDeprecationInfoAction.java
@@ -16,11 +16,14 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Set;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestDeprecationInfoAction extends BaseRestHandler {
 
+    private static final Set<String> SUPPORTED_CAPABILITIES = Set.of("data_streams", "ilm_policies", "templates");
+
     @Override
     public List<Route> routes() {
         return List.of(new Route(GET, "/_migration/deprecations"), new Route(GET, "/{index}/_migration/deprecations"));
@@ -47,4 +50,9 @@ private static RestChannelConsumer handleGet(final RestRequest request, NodeClie
         );
         return channel -> client.execute(DeprecationInfoAction.INSTANCE, infoRequest, new RestToXContentListener<>(channel));
     }
+
+    @Override
+    public Set<String> supportedCapabilities() {
+        return SUPPORTED_CAPABILITIES;
+    }
 }
diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java
@@ -29,7 +29,6 @@
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.util.Version;
 import org.elasticsearch.core.UpdateForV10;
-import org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.BWCLucene70Codec;
 import org.elasticsearch.xpack.lucene.bwc.codecs.lucene80.BWCLucene80Codec;
 import org.elasticsearch.xpack.lucene.bwc.codecs.lucene84.BWCLucene84Codec;
 import org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.BWCLucene86Codec;
@@ -229,7 +228,6 @@ private static Codec getBackwardCompatibleCodec(Codec codec) {
         if (codec == null) return null;
 
         return switch (codec.getClass().getSimpleName()) {
-            case "Lucene70Codec" -> new BWCLucene70Codec();
             case "Lucene80Codec" -> new BWCLucene80Codec();
             case "Lucene84Codec" -> new BWCLucene84Codec();
             case "Lucene86Codec" -> new BWCLucene86Codec();
diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java
@@ -86,8 +86,16 @@ public MetadataOnlyBKDReader(IndexInput metaIn, boolean isVersionPost86) throws
         pointCount = metaIn.readVLong();
         docCount = metaIn.readVInt();
 
-        // This code has been introduced to process IndexInput created with Lucene86Codec+. This is not necessary
-        // in the read-only version for older formats.
+        // The pre-8.6 code does not read the following fields that its standard Lucene counterpart does. After experimenting with the
+        // code, we got to the conclusion that these are the last fields being read, which are not needed in the metadata-only reader, and
+        // we can safely ignore them when loading the file. Although by coincidence, nothing breaks if we read a couple of VLongs, as long
+        // as some bytes are available to read.
+        //
+        // The extra reads have been introduced to process IndexInput created with Lucene86Codec+, where a new BKD format has been
+        // introduced. We have stricter checks around the header and footer starting from the 86 formats hence we do need to
+        // consume all the data input there but not in previous formats.
+        //
+        // For correctness, we added version checking here. If and only if, the version is 8.6 or higher, we read the additional fields.
         if (isVersionPost86) {
             metaIn.readVInt();
             metaIn.readLong();
diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java
@@ -89,7 +89,7 @@ public static class Builder extends FieldMapper.Builder {
             }
 
             return XContentMapValues.nodeIntegerValue(o);
-        }, m -> toType(m).fieldType().dims, XContentBuilder::field, Object::toString).setSerializerCheck((id, ic, v) -> v != null)
+        }, m -> toType(m).fieldType().dims, XContentBuilder::field, Objects::toString).setSerializerCheck((id, ic, v) -> v != null)
             .setMergeValidator((previous, current, c) -> previous == null || Objects.equals(previous, current))
             .addValidator(dims -> {
                 if (dims == null) {
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/deprecation/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/deprecation/10_basic.yml
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rank_vectors/rank_vectors.yml
