Merge branch 'main' into fix_invalid_geo

Author: craigtaverner

libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java

@@ -43,14 +43,14 @@ public record BootstrapArgs(
         Function<Class<?>, PolicyManager.PolicyScope> scopeResolver,
         PathLookup pathLookup,
         Map<String, Path> sourcePaths,
-        Set<Class<?>> suppressFailureLogClasses
+        Set<Package> suppressFailureLogPackages
     ) {
         public BootstrapArgs {
             requireNonNull(pluginPolicies);
             requireNonNull(scopeResolver);
             requireNonNull(pathLookup);
             requireNonNull(sourcePaths);
-            requireNonNull(suppressFailureLogClasses);
+            requireNonNull(suppressFailureLogPackages);
         }
     }
 
@@ -78,7 +78,7 @@ public static BootstrapArgs bootstrapArgs() {
      * @param tempDir        the temp directory for Elasticsearch
      * @param logsDir        the log directory for Elasticsearch
      * @param pidFile        path to a pid file for Elasticsearch, or {@code null} if one was not specified
-     * @param suppressFailureLogClasses   classes for which we do not need or want to log Entitlements failures
+     * @param suppressFailureLogPackages   packages for which we do not need or want to log Entitlements failures
      */
     public static void bootstrap(
         Policy serverPolicyPatch,
@@ -95,7 +95,7 @@ public static void bootstrap(
         Path logsDir,
         Path tempDir,
         Path pidFile,
-        Set<Class<?>> suppressFailureLogClasses
+        Set<Package> suppressFailureLogPackages
     ) {
         logger.debug("Loading entitlement agent");
         if (EntitlementBootstrap.bootstrapArgs != null) {
@@ -119,7 +119,7 @@ public static void bootstrap(
                 settingResolver
             ),
             sourcePaths,
-            suppressFailureLogClasses
+            suppressFailureLogPackages
         );
         exportInitializationToAgent();
         loadAgent(findAgentJar());

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -90,7 +90,7 @@ private static PolicyManager createPolicyManager() {
             EntitlementBootstrap.bootstrapArgs().sourcePaths(),
             ENTITLEMENTS_MODULE,
             pathLookup,
-            bootstrapArgs.suppressFailureLogClasses()
+            bootstrapArgs.suppressFailureLogPackages()
         );
     }
 

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java

@@ -256,7 +256,7 @@ ModuleEntitlements policyEntitlements(String componentName, Path componentPath,
     private final Map<String, Map<String, List<Entitlement>>> pluginsEntitlements;
     private final Function<Class<?>, PolicyScope> scopeResolver;
     private final PathLookup pathLookup;
-    private final Set<Class<?>> mutedClasses;
+    private final Set<Package> suppressFailureLogPackages;
 
     public static final String ALL_UNNAMED = "ALL-UNNAMED";
 
@@ -311,7 +311,7 @@ public PolicyManager(
         Map<String, Path> sourcePaths,
         Module entitlementsModule,
         PathLookup pathLookup,
-        Set<Class<?>> suppressFailureLogClasses
+        Set<Package> suppressFailureLogPackages
     ) {
         this.serverEntitlements = buildScopeEntitlementsMap(requireNonNull(serverPolicy));
         this.apmAgentEntitlements = apmAgentEntitlements;
@@ -322,7 +322,7 @@ public PolicyManager(
         this.sourcePaths = sourcePaths;
         this.entitlementsModule = entitlementsModule;
         this.pathLookup = requireNonNull(pathLookup);
-        this.mutedClasses = suppressFailureLogClasses;
+        this.suppressFailureLogPackages = suppressFailureLogPackages;
 
         List<ExclusiveFileEntitlement> exclusiveFileEntitlements = new ArrayList<>();
         for (var e : serverEntitlements.entrySet()) {
@@ -688,8 +688,8 @@ public void checkWriteProperty(Class<?> callerClass, String property) {
 
     private void notEntitled(String message, Class<?> callerClass, ModuleEntitlements entitlements) {
         var exception = new NotEntitledException(message);
-        // Don't emit a log for muted classes, e.g. classes containing self tests
-        if (mutedClasses.contains(callerClass) == false) {
+        // Don't emit a log for suppressed packages, e.g. packages containing self tests
+        if (suppressFailureLogPackages.contains(callerClass.getPackage()) == false) {
             entitlements.logger().warn("Not entitled: {}", message, exception);
         }
         throw exception;

muted-tests.yml

@@ -438,9 +438,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
   method: test {lookup-join.MvJoinKeyOnTheLookupIndex ASYNC}
   issue: https://github.com/elastic/elasticsearch/issues/128030
-- class: org.elasticsearch.compute.aggregation.PercentileIntGroupingAggregatorFunctionTests
-  method: testManyInitialManyPartialFinalRunner
-  issue: https://github.com/elastic/elasticsearch/issues/128092
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test042KeystorePermissionsAreCorrect
   issue: https://github.com/elastic/elasticsearch/issues/128018

rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.repository_verify_integrity.json

@@ -5,7 +5,7 @@
       "description":"Verifies the integrity of the contents of a snapshot repository"
     },
     "stability":"experimental",
-    "visibility":"private",
+    "visibility":"public",
     "headers": {
       "accept": [
         "application/json"

server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java

@@ -264,7 +264,7 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException {
             nodeEnv.logsDir(),
             nodeEnv.tmpDir(),
             args.pidFile(),
-            Set.of(EntitlementSelfTester.class)
+            Set.of(EntitlementSelfTester.class.getPackage())
         );
         EntitlementSelfTester.entitlementSelfTest();
 

server/src/main/java/org/elasticsearch/index/codec/Elasticsearch900Lucene101Codec.java

@@ -28,6 +28,8 @@
  */
 public class Elasticsearch900Lucene101Codec extends CodecService.DeduplicateFieldInfosCodec {
 
+    static final PostingsFormat DEFAULT_POSTINGS_FORMAT = new Lucene101PostingsFormat();
+
     private final StoredFieldsFormat storedFieldsFormat;
 
     private final PostingsFormat defaultPostingsFormat;
@@ -66,7 +68,7 @@ public Elasticsearch900Lucene101Codec() {
     public Elasticsearch900Lucene101Codec(Zstd814StoredFieldsFormat.Mode mode) {
         super("Elasticsearch900Lucene101", new Lucene101Codec());
         this.storedFieldsFormat = mode.getFormat();
-        this.defaultPostingsFormat = new Lucene101PostingsFormat();
+        this.defaultPostingsFormat = DEFAULT_POSTINGS_FORMAT;
         this.defaultDVFormat = new Lucene90DocValuesFormat();
         this.defaultKnnVectorsFormat = new Lucene99HnswVectorsFormat();
     }

server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java

@@ -12,7 +12,6 @@
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat;
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
 import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
 import org.elasticsearch.common.util.BigArrays;
@@ -34,13 +33,12 @@
  * vectors.
  */
 public class PerFieldFormatSupplier {
-    public static final FeatureFlag USE_LUCENE101_POSTINGS_FORMAT = new FeatureFlag("use_lucene101_postings_format");
+    public static final FeatureFlag USE_DEFAULT_LUCENE_POSTINGS_FORMAT = new FeatureFlag("use_default_lucene_postings_format");
 
     private static final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat();
     private static final KnnVectorsFormat knnVectorsFormat = new Lucene99HnswVectorsFormat();
     private static final ES819TSDBDocValuesFormat tsdbDocValuesFormat = new ES819TSDBDocValuesFormat();
     private static final ES812PostingsFormat es812PostingsFormat = new ES812PostingsFormat();
-    private static final Lucene101PostingsFormat lucene101PostingsFormat = new Lucene101PostingsFormat();
     private static final PostingsFormat completionPostingsFormat = PostingsFormat.forName("Completion101");
 
     private final ES87BloomFilterPostingsFormat bloomFilterPostingsFormat;
@@ -53,10 +51,10 @@ public PerFieldFormatSupplier(MapperService mapperService, BigArrays bigArrays)
         this.bloomFilterPostingsFormat = new ES87BloomFilterPostingsFormat(bigArrays, this::internalGetPostingsFormatForField);
 
         if (mapperService != null
-            && USE_LUCENE101_POSTINGS_FORMAT.isEnabled()
+            && USE_DEFAULT_LUCENE_POSTINGS_FORMAT.isEnabled()
             && mapperService.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.USE_LUCENE101_POSTINGS_FORMAT)
             && mapperService.getIndexSettings().getMode() == IndexMode.STANDARD) {
-            defaultPostingsFormat = lucene101PostingsFormat;
+            defaultPostingsFormat = Elasticsearch900Lucene101Codec.DEFAULT_POSTINGS_FORMAT;
         } else {
             // our own posting format using PFOR
             defaultPostingsFormat = es812PostingsFormat;

server/src/main/java/org/elasticsearch/index/store/LuceneFilesExtensions.java

@@ -83,7 +83,11 @@ public enum LuceneFilesExtensions {
     VEMQ("vemq", "Scalar Quantized Vector Metadata", true, false),
     VEQ("veq", "Scalar Quantized Vector Data", false, true),
     VEMB("vemb", "Binarized Vector Metadata", true, false),
-    VEB("veb", "Binarized Vector Data", false, true);
+    VEB("veb", "Binarized Vector Data", false, true),
+    // ivf vectors format
+    MIVF("mivf", "IVF Metadata", true, false),
+    CENIVF("cenivf", "IVF Centroid Data", false, true),
+    CLIVF("clivf", "IVF Cluster Data", false, true);
 
     /**
      * Allow plugin developers of custom codecs to opt out of the assertion in {@link #fromExtension}

server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java

@@ -94,7 +94,7 @@ public void testUseBloomFilter() throws IOException {
         assertThat(perFieldMapperCodec.getPostingsFormatForField("_id"), instanceOf(ES87BloomFilterPostingsFormat.class));
         assertThat(perFieldMapperCodec.useBloomFilter("another_field"), is(false));
 
-        Class<? extends PostingsFormat> expectedPostingsFormat = PerFieldFormatSupplier.USE_LUCENE101_POSTINGS_FORMAT.isEnabled()
+        Class<? extends PostingsFormat> expectedPostingsFormat = PerFieldFormatSupplier.USE_DEFAULT_LUCENE_POSTINGS_FORMAT.isEnabled()
             && timeSeries == false ? Lucene101PostingsFormat.class : ES812PostingsFormat.class;
         assertThat(perFieldMapperCodec.getPostingsFormatForField("another_field"), instanceOf(expectedPostingsFormat));
     }
@@ -110,7 +110,7 @@ public void testUseBloomFilterWithTimestampFieldEnabled() throws IOException {
     public void testUseBloomFilterWithTimestampFieldEnabled_noTimeSeriesMode() throws IOException {
         PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, false, false);
         assertThat(perFieldMapperCodec.useBloomFilter("_id"), is(false));
-        Class<? extends PostingsFormat> expectedPostingsFormat = PerFieldFormatSupplier.USE_LUCENE101_POSTINGS_FORMAT.isEnabled()
+        Class<? extends PostingsFormat> expectedPostingsFormat = PerFieldFormatSupplier.USE_DEFAULT_LUCENE_POSTINGS_FORMAT.isEnabled()
             ? Lucene101PostingsFormat.class
             : ES812PostingsFormat.class;
         assertThat(perFieldMapperCodec.getPostingsFormatForField("_id"), instanceOf(expectedPostingsFormat));