Merge remote-tracking branch 'origin/main' into es-108907-fix

Author: ywangd

docs/changelog/128405.yaml

@@ -0,0 +1,5 @@
+pr: 128405
+summary: Modify the mechanism to pause indexing
+area: Distributed
+type: bug
+issues: []

libs/entitlement/src/main/java/module-info.java

@@ -20,11 +20,13 @@
     requires static org.elasticsearch.entitlement.bridge; // At runtime, this will be in java.base
 
     exports org.elasticsearch.entitlement.runtime.api;
-    exports org.elasticsearch.entitlement.runtime.policy;
-    exports org.elasticsearch.entitlement.runtime.policy.entitlements to org.elasticsearch.server;
     exports org.elasticsearch.entitlement.instrumentation;
     exports org.elasticsearch.entitlement.bootstrap to org.elasticsearch.server;
     exports org.elasticsearch.entitlement.initialization to java.base;
 
+    // TODO: Most of the things in the policy package should be internal implementation details that are not exported.
+    exports org.elasticsearch.entitlement.runtime.policy;
+    exports org.elasticsearch.entitlement.runtime.policy.entitlements to org.elasticsearch.server;
+
     uses org.elasticsearch.entitlement.instrumentation.InstrumentationService;
 }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java

@@ -14,11 +14,9 @@
 import com.sun.tools.attach.AttachNotSupportedException;
 import com.sun.tools.attach.VirtualMachine;
 
-import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.entitlement.initialization.EntitlementInitialization;
-import org.elasticsearch.entitlement.runtime.policy.PathLookup;
 import org.elasticsearch.entitlement.runtime.policy.PathLookupImpl;
 import org.elasticsearch.entitlement.runtime.policy.Policy;
 import org.elasticsearch.entitlement.runtime.policy.PolicyManager;
@@ -33,35 +31,11 @@
 import java.util.function.Function;
 import java.util.stream.Stream;
 
-import static java.util.Objects.requireNonNull;
-
 public class EntitlementBootstrap {
 
-    public record BootstrapArgs(
-        @Nullable Policy serverPolicyPatch,
-        Map<String, Policy> pluginPolicies,
-        Function<Class<?>, PolicyManager.PolicyScope> scopeResolver,
-        PathLookup pathLookup,
-        Map<String, Path> sourcePaths,
-        Set<Package> suppressFailureLogPackages
-    ) {
-        public BootstrapArgs {
-            requireNonNull(pluginPolicies);
-            requireNonNull(scopeResolver);
-            requireNonNull(pathLookup);
-            requireNonNull(sourcePaths);
-            requireNonNull(suppressFailureLogPackages);
-        }
-    }
-
-    private static BootstrapArgs bootstrapArgs;
-
-    public static BootstrapArgs bootstrapArgs() {
-        return bootstrapArgs;
-    }
-
     /**
-     * Activates entitlement checking. Once this method returns, calls to methods protected by Entitlements from classes without a valid
+     * Main entry point that activates entitlement checking. Once this method returns,
+     * calls to methods protected by entitlements from classes without a valid
      * policy will throw {@link org.elasticsearch.entitlement.runtime.api.NotEntitledException}.
      *
      * @param serverPolicyPatch a policy with additional entitlements to patch the embedded server layer policy
@@ -98,10 +72,10 @@ public static void bootstrap(
         Set<Package> suppressFailureLogPackages
     ) {
         logger.debug("Loading entitlement agent");
-        if (EntitlementBootstrap.bootstrapArgs != null) {
-            throw new IllegalStateException("plugin data is already set");
+        if (EntitlementInitialization.initializeArgs != null) {
+            throw new IllegalStateException("initialization data is already set");
         }
-        EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(
+        EntitlementInitialization.initializeArgs = new EntitlementInitialization.InitializeArgs(
             serverPolicyPatch,
             pluginPolicies,
             scopeResolver,

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -10,9 +10,9 @@
 package org.elasticsearch.entitlement.initialization;
 
 import org.elasticsearch.core.Booleans;
-import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.entitlement.bridge.EntitlementChecker;
-import org.elasticsearch.entitlement.runtime.api.ElasticsearchEntitlementChecker;
+import org.elasticsearch.entitlement.runtime.policy.ElasticsearchEntitlementChecker;
 import org.elasticsearch.entitlement.runtime.policy.PathLookup;
 import org.elasticsearch.entitlement.runtime.policy.Policy;
 import org.elasticsearch.entitlement.runtime.policy.PolicyChecker;
@@ -22,8 +22,12 @@
 import java.lang.instrument.Instrumentation;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
+import java.nio.file.Path;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
+
+import static java.util.Objects.requireNonNull;
 
 /**
  * Called by the agent during {@code agentmain} to configure the entitlement system,
@@ -36,6 +40,7 @@ public class EntitlementInitialization {
 
     private static final Module ENTITLEMENTS_MODULE = PolicyManager.class.getModule();
 
+    public static InitializeArgs initializeArgs;
     private static ElasticsearchEntitlementChecker checker;
 
     // Note: referenced by bridge reflectively
@@ -63,32 +68,60 @@ public static EntitlementChecker checker() {
      * @param inst the JVM instrumentation class instance
      */
     public static void initialize(Instrumentation inst) throws Exception {
-        checker = initChecker(inst, createPolicyManager());
+        // the checker _MUST_ be set before _any_ instrumentation is done
+        checker = initChecker(createPolicyManager());
+        initInstrumentation(inst);
+    }
+
+    /**
+     * Arguments to {@link #initialize}. Since that's called in a static context from the agent,
+     * we have no way to pass arguments directly, so we stuff them in here.
+     *
+     * @param serverPolicyPatch
+     * @param pluginPolicies
+     * @param scopeResolver
+     * @param pathLookup
+     * @param sourcePaths
+     * @param suppressFailureLogPackages
+     */
+    public record InitializeArgs(
+        @Nullable Policy serverPolicyPatch,
+        Map<String, Policy> pluginPolicies,
+        Function<Class<?>, PolicyManager.PolicyScope> scopeResolver,
+        PathLookup pathLookup,
+        Map<String, Path> sourcePaths,
+        Set<Package> suppressFailureLogPackages
+    ) {
+        public InitializeArgs {
+            requireNonNull(pluginPolicies);
+            requireNonNull(scopeResolver);
+            requireNonNull(pathLookup);
+            requireNonNull(sourcePaths);
+            requireNonNull(suppressFailureLogPackages);
+        }
     }
 
     private static PolicyCheckerImpl createPolicyChecker(PolicyManager policyManager) {
-        EntitlementBootstrap.BootstrapArgs bootstrapArgs = EntitlementBootstrap.bootstrapArgs();
         return new PolicyCheckerImpl(
-            bootstrapArgs.suppressFailureLogPackages(),
+            initializeArgs.suppressFailureLogPackages(),
             ENTITLEMENTS_MODULE,
             policyManager,
-            bootstrapArgs.pathLookup()
+            initializeArgs.pathLookup()
         );
     }
 
     private static PolicyManager createPolicyManager() {
-        EntitlementBootstrap.BootstrapArgs bootstrapArgs = EntitlementBootstrap.bootstrapArgs();
-        Map<String, Policy> pluginPolicies = bootstrapArgs.pluginPolicies();
-        PathLookup pathLookup = bootstrapArgs.pathLookup();
+        Map<String, Policy> pluginPolicies = initializeArgs.pluginPolicies();
+        PathLookup pathLookup = initializeArgs.pathLookup();
 
         FilesEntitlementsValidation.validate(pluginPolicies, pathLookup);
 
         return new PolicyManager(
-            HardcodedEntitlements.serverPolicy(pathLookup.pidFile(), bootstrapArgs.serverPolicyPatch()),
+            HardcodedEntitlements.serverPolicy(pathLookup.pidFile(), initializeArgs.serverPolicyPatch()),
             HardcodedEntitlements.agentEntitlements(),
             pluginPolicies,
-            EntitlementBootstrap.bootstrapArgs().scopeResolver(),
-            EntitlementBootstrap.bootstrapArgs().sourcePaths(),
+            initializeArgs.scopeResolver(),
+            initializeArgs.sourcePaths(),
             pathLookup
         );
     }
@@ -115,7 +148,7 @@ private static void ensureClassesSensitiveToVerificationAreInitialized() {
         }
     }
 
-    static ElasticsearchEntitlementChecker initChecker(Instrumentation inst, PolicyManager policyManager) throws Exception {
+    static ElasticsearchEntitlementChecker initChecker(PolicyManager policyManager) {
         final PolicyChecker policyChecker = createPolicyChecker(policyManager);
         final Class<?> clazz = EntitlementCheckerUtils.getVersionSpecificCheckerClass(
             ElasticsearchEntitlementChecker.class,
@@ -136,17 +169,20 @@ static ElasticsearchEntitlementChecker initChecker(Instrumentation inst, PolicyM
             throw new AssertionError(e);
         }
 
+        return checker;
+    }
+
+    static void initInstrumentation(Instrumentation instrumentation) throws Exception {
         var verifyBytecode = Booleans.parseBoolean(System.getProperty("es.entitlements.verify_bytecode", "false"));
         if (verifyBytecode) {
             ensureClassesSensitiveToVerificationAreInitialized();
         }
 
         DynamicInstrumentation.initialize(
-            inst,
+            instrumentation,
             EntitlementCheckerUtils.getVersionSpecificCheckerClass(EntitlementChecker.class, Runtime.version().feature()),
             verifyBytecode
         );
 
-        return checker;
     }
 }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/package-info.java

@@ -189,7 +189,7 @@
  * <h2>Checks</h2>
  * <p>
  * The injected prologue calls a {@code check$} method on {@link org.elasticsearch.entitlement.bridge.EntitlementChecker}; its
- * implementation (normally on {@link org.elasticsearch.entitlement.runtime.api.ElasticsearchEntitlementChecker}, unless it is a
+ * implementation (normally on {@link org.elasticsearch.entitlement.runtime.policy.ElasticsearchEntitlementChecker}, unless it is a
  * version-specific method) calls the appropriate methods on {@link org.elasticsearch.entitlement.runtime.policy.PolicyManager},
  * forwarding the caller class and a specific set of arguments. These methods all start with check, roughly matching an entitlement type
  * (e.g. {@link org.elasticsearch.entitlement.runtime.policy.PolicyChecker#checkInboundNetworkAccess},

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/package-info.java

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+/**
+ * The public API for the Entitlements system.
+ * All other packages are implementation details that should use selective exports.
+ */
+package org.elasticsearch.entitlement.runtime.api;

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java renamed to libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ElasticsearchEntitlementChecker.java

@@ -7,13 +7,12 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.entitlement.runtime.api;
+package org.elasticsearch.entitlement.runtime.policy;
 
 import jdk.nio.Channels;
 
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.entitlement.bridge.EntitlementChecker;
-import org.elasticsearch.entitlement.runtime.policy.PolicyChecker;
 
 import java.io.File;
 import java.io.FileDescriptor;

libs/entitlement/src/main23/java/org/elasticsearch/entitlement/runtime/api/Java23ElasticsearchEntitlementChecker.java renamed to libs/entitlement/src/main23/java/org/elasticsearch/entitlement/runtime/policy/Java23ElasticsearchEntitlementChecker.java

@@ -7,10 +7,9 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.entitlement.runtime.api;
+package org.elasticsearch.entitlement.runtime.policy;
 
 import org.elasticsearch.entitlement.bridge.Java23EntitlementChecker;
-import org.elasticsearch.entitlement.runtime.policy.PolicyChecker;
 
 public class Java23ElasticsearchEntitlementChecker extends ElasticsearchEntitlementChecker implements Java23EntitlementChecker {
 

muted-tests.yml

@@ -417,9 +417,6 @@ tests:
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test025SyncPluginsUsingProxy
   issue: https://github.com/elastic/elasticsearch/issues/127138
-- class: org.elasticsearch.indices.stats.IndexStatsIT
-  method: testThrottleStats
-  issue: https://github.com/elastic/elasticsearch/issues/126359
 - class: org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityRestIT
   method: testTaskCancellation
   issue: https://github.com/elastic/elasticsearch/issues/128009
@@ -495,9 +492,12 @@ tests:
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test085EnvironmentVariablesAreRespectedUnderDockerExec
   issue: https://github.com/elastic/elasticsearch/issues/128115
-- class: org.elasticsearch.test.apmintegration.TracesApmIT
-  method: testApmIntegration
-  issue: https://github.com/elastic/elasticsearch/issues/128651
+- class: org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLikeTests
+  method: testEvaluateInManyThreads {TestCase=100 random code points matches self case insensitive with keyword}
+  issue: https://github.com/elastic/elasticsearch/issues/128676
+- class: org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLikeTests
+  method: testEvaluateInManyThreads {TestCase=100 random code points matches self case insensitive with text}
+  issue: https://github.com/elastic/elasticsearch/issues/128677
 
 # Examples:
 #

rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java

@@ -38,6 +38,7 @@ public class ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
         .feature(FeatureFlag.DOC_VALUES_SKIPPER)
         .feature(FeatureFlag.USE_LUCENE101_POSTINGS_FORMAT)
+        .feature(FeatureFlag.IVF_FORMAT)
         .build();
 
     public ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {