Fix entitlements in internalClusterTest (#131539) (#132067)

mosche · web-flow · commit 2479ec430770 · 2025-07-29T16:46:52.000+02:00
Previously, entitlement checks got disabled when resetting the policy manager (which defaults to inactive). This change makes sure entitlements are correctly enabled during tests.

Due to the lack of entitlement delegation (and usage of server's FileSystemUtils and similar in test code), there's a few remaining issues:
- various tests have to run without entitlements
- node base dirs cannot be removed immediately when shutting down the node due to pending cleanups (wipePendingDataDirectories)

Due to Netty dependency issues (ES-12435), azure and inference tests have to run without entitlements.
diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java
@@ -367,9 +367,7 @@ private ModuleEntitlements getModuleScopeEntitlements(
      * @return true if permission is granted regardless of the entitlement
      */
     boolean isTriviallyAllowed(Class<?> requestingClass) {
-        if (generalLogger.isTraceEnabled()) {
-            generalLogger.trace("Stack trace for upcoming trivially-allowed check", new Exception());
-        }
+        // note: do not log exceptions in here, this could interfere with loading of additionally necessary classes such as ThrowableProxy
         if (requestingClass == null) {
             generalLogger.debug("Entitlement trivially allowed: no caller frames outside the entitlement library");
             return true;
diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java
@@ -41,6 +41,7 @@
 import org.elasticsearch.telemetry.Measurement;
 import org.elasticsearch.telemetry.TestTelemetryPlugin;
 import org.elasticsearch.test.BackgroundIndexer;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.ByteArrayInputStream;
@@ -73,6 +74,7 @@
 import static org.hamcrest.Matchers.is;
 
 @SuppressForbidden(reason = "this test uses a HttpServer to emulate an Azure endpoint")
+@ESTestCase.WithoutEntitlements // due to dependency issue ES-12435
 public class AzureBlobStoreRepositoryTests extends ESMockAPIBasedRepositoryIntegTestCase {
 
     protected static final String DEFAULT_ACCOUNT_NAME = "account";
diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureRepositoryMissingCredentialsIT.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureRepositoryMissingCredentialsIT.java
@@ -18,12 +18,14 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.repositories.RepositoryVerificationException;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESTestCase;
 
 import java.util.Collection;
 
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
 
+@ESTestCase.WithoutEntitlements // due to dependency issue ES-12435
 public class AzureRepositoryMissingCredentialsIT extends ESIntegTestCase {
 
     @Override
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.plugins.ReloadablePlugin;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESTestCase;
 import org.junit.BeforeClass;
 
 import java.io.InputStream;
@@ -47,6 +48,7 @@
 import static org.hamcrest.Matchers.nullValue;
 
 @ESIntegTestCase.ClusterScope(minNumDataNodes = 2)
+@ESTestCase.WithoutEntitlements // requires entitlement delegation ES-10920
 public class ReloadSecureSettingsIT extends ESIntegTestCase {
 
     private static final String VALID_SECURE_SETTING_NAME = "some.setting.that.exists";
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.InternalTestCluster;
 
 import java.util.Arrays;
@@ -22,6 +23,7 @@
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_READ_ONLY;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE;
 
+@ESTestCase.WithoutEntitlements // requires entitlement delegation ES-10920
 public class PendingTasksBlocksIT extends ESIntegTestCase {
 
     public void testPendingTasksWithIndexBlocks() {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommandIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommandIT.java
@@ -18,12 +18,14 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESTestCase;
 
 import java.util.Map;
 
 import static org.hamcrest.Matchers.containsString;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
+@ESTestCase.WithoutEntitlements // commands don't run with entitlements enforced
 public class RemoveCustomsCommandIT extends ESIntegTestCase {
 
     public void testRemoveCustomsAbortedByUser() throws Exception {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommandIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommandIT.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESTestCase;
 
 import java.util.Collection;
 import java.util.List;
@@ -31,6 +32,7 @@
 import static org.hamcrest.Matchers.not;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
+@ESTestCase.WithoutEntitlements // commands don't run with entitlements enforced
 public class RemoveIndexSettingsCommandIT extends ESIntegTestCase {
 
     static final Setting<Integer> FOO = Setting.intSetting("index.foo", 1, Setting.Property.IndexScope, Setting.Property.Dynamic);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveSettingsCommandIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveSettingsCommandIT.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESTestCase;
 
 import java.util.Map;
 
@@ -27,6 +28,7 @@
 import static org.hamcrest.Matchers.not;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
+@ESTestCase.WithoutEntitlements // commands don't run with entitlements enforced
 public class RemoveSettingsCommandIT extends ESIntegTestCase {
 
     public void testRemoveSettingsAbortedByUser() throws Exception {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java
@@ -58,6 +58,7 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.CorruptionUtils;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.engine.MockEngineSupport;
@@ -93,6 +94,7 @@
 import static org.hamcrest.Matchers.startsWith;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
+@ESTestCase.WithoutEntitlements // commands don't run with entitlements enforced
 public class RemoveCorruptedShardDataCommandIT extends ESIntegTestCase {
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/entitlement/bootstrap/TestEntitlementBootstrap.java b/test/framework/src/main/java/org/elasticsearch/entitlement/bootstrap/TestEntitlementBootstrap.java
@@ -9,6 +9,7 @@
 
 package org.elasticsearch.entitlement.bootstrap;
 
+import org.apache.lucene.tests.mockfile.FilterPath;
 import org.elasticsearch.bootstrap.TestBuildInfo;
 import org.elasticsearch.bootstrap.TestBuildInfoParser;
 import org.elasticsearch.bootstrap.TestScopeResolver;
@@ -38,6 +39,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -83,34 +85,56 @@ public static void registerNodeBaseDirs(Settings settings, Path configPath) {
         if (policyManager == null) {
             return;
         }
-        Path homeDir = absolutePath(PATH_HOME_SETTING.get(settings));
-        Path configDir = configPath != null ? configPath : homeDir.resolve("config");
+
+        Path homeDir = homeDir(settings);
+        Path configDir = configDir(configPath, homeDir);
         Collection<Path> dataDirs = dataDirs(settings, homeDir);
         Collection<Path> sharedDataDir = sharedDataDir(settings);
         Collection<Path> repoDirs = repoDirs(settings);
-        logger.debug("Registering node dirs: config [{}], dataDirs [{}], repoDirs [{}]", configDir, dataDirs, repoDirs);
+        logger.debug(
+            "Registering node dirs: config [{}], dataDirs [{}], sharedDataDir [{}], repoDirs [{}]",
+            configDir,
+            dataDirs,
+            sharedDataDir,
+            repoDirs
+        );
         baseDirPaths.compute(BaseDir.CONFIG, baseDirModifier(paths -> paths.add(configDir)));
         baseDirPaths.compute(BaseDir.DATA, baseDirModifier(paths -> paths.addAll(dataDirs)));
         baseDirPaths.compute(BaseDir.SHARED_DATA, baseDirModifier(paths -> paths.addAll(sharedDataDir)));
         baseDirPaths.compute(BaseDir.SHARED_REPO, baseDirModifier(paths -> paths.addAll(repoDirs)));
-        policyManager.reset();
+        policyManager.clearModuleEntitlementsCache();
     }
 
     public static void unregisterNodeBaseDirs(Settings settings, Path configPath) {
         if (policyManager == null) {
             return;
         }
-        Path homeDir = absolutePath(PATH_HOME_SETTING.get(settings));
-        Path configDir = configPath != null ? configPath : homeDir.resolve("config");
+
+        Path homeDir = homeDir(settings);
+        Path configDir = configDir(configPath, homeDir);
         Collection<Path> dataDirs = dataDirs(settings, homeDir);
         Collection<Path> sharedDataDir = sharedDataDir(settings);
         Collection<Path> repoDirs = repoDirs(settings);
-        logger.debug("Unregistering node dirs: config [{}], dataDirs [{}], repoDirs [{}]", configDir, dataDirs, repoDirs);
+        logger.debug(
+            "Unregistering node dirs: config [{}], dataDirs [{}], sharedDataDir [{}], repoDirs [{}]",
+            configDir,
+            dataDirs,
+            sharedDataDir,
+            repoDirs
+        );
         baseDirPaths.compute(BaseDir.CONFIG, baseDirModifier(paths -> paths.remove(configDir)));
         baseDirPaths.compute(BaseDir.DATA, baseDirModifier(paths -> paths.removeAll(dataDirs)));
         baseDirPaths.compute(BaseDir.SHARED_DATA, baseDirModifier(paths -> paths.removeAll(sharedDataDir)));
         baseDirPaths.compute(BaseDir.SHARED_REPO, baseDirModifier(paths -> paths.removeAll(repoDirs)));
-        policyManager.reset();
+        policyManager.clearModuleEntitlementsCache();
+    }
+
+    private static Path homeDir(Settings settings) {
+        return absolutePath(PATH_HOME_SETTING.get(settings));
+    }
+
+    private static Path configDir(Path configDir, Path homeDir) {
+        return configDir != null ? unwrapFilterPath(configDir) : homeDir.resolve("config");
     }
 
     private static Collection<Path> dataDirs(Settings settings, Path homeDir) {
@@ -130,15 +154,21 @@ private static Collection<Path> repoDirs(Settings settings) {
     }
 
     private static BiFunction<BaseDir, Collection<Path>, Collection<Path>> baseDirModifier(Consumer<Collection<Path>> consumer) {
+        // always return a new unmodifiable copy
         return (BaseDir baseDir, Collection<Path> paths) -> {
-            if (paths == null) {
-                paths = new HashSet<>();
-            }
+            paths = paths == null ? new HashSet<>() : new HashSet<>(paths);
             consumer.accept(paths);
-            return paths;
+            return Collections.unmodifiableCollection(paths);
         };
     }
 
+    private static Path unwrapFilterPath(Path path) {
+        while (path instanceof FilterPath fPath) {
+            path = fPath.getDelegate();
+        }
+        return path;
+    }
+
     @SuppressForbidden(reason = "must be resolved using the default file system, rather then the mocked test file system")
     private static Path absolutePath(String path) {
         return Paths.get(path).toAbsolutePath().normalize();
@@ -168,9 +198,11 @@ public static void setEntitledTestPackages(String[] entitledTestPackages) {
         policyManager.setEntitledTestPackages(entitledTestPackages);
     }
 
-    public static void reset() {
+    public static void resetAfterTest() {
+        // reset all base dirs except TEMP, which is initialized just once statically
+        baseDirPaths.keySet().retainAll(List.of(TEMP));
         if (policyManager != null) {
-            policyManager.reset();
+            policyManager.resetAfterTest();
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/entitlement/runtime/policy/TestPolicyManager.java b/test/framework/src/main/java/org/elasticsearch/entitlement/runtime/policy/TestPolicyManager.java
@@ -53,7 +53,7 @@ public TestPolicyManager(
         super(serverPolicy, apmAgentEntitlements, pluginPolicies, scopeResolver, name -> classpath, pathLookup);
         this.classpath = classpath;
         this.testOnlyClasspath = testOnlyClasspath;
-        reset();
+        resetAfterTest();
     }
 
     public void setActive(boolean newValue) {
@@ -77,13 +77,21 @@ public void setEntitledTestPackages(String... entitledTestPackages) {
     /**
      * Called between tests so each test is not affected by prior tests
      */
-    public final void reset() {
-        assert moduleEntitlementsMap.isEmpty() : "We're not supposed to be using moduleEntitlementsMap in tests";
-        classEntitlementsMap.clear();
+    public final void resetAfterTest() {
+        clearModuleEntitlementsCache();
         isActive = false;
         isTriviallyAllowingTestCode = true;
     }
 
+    /**
+     * Clear cached module entitlements.
+     * This is required after updating entries in {@link TestPathLookup}.
+     */
+    public final void clearModuleEntitlementsCache() {
+        assert moduleEntitlementsMap.isEmpty() : "We're not supposed to be using moduleEntitlementsMap in tests";
+        classEntitlementsMap.clear();
+    }
+
     @Override
     protected boolean isTrustedSystemClass(Class<?> requestingClass) {
         ClassLoader loader = requestingClass.getClassLoader();
diff --git a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java
@@ -54,11 +54,11 @@
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.TransportSettings;
 
-import java.io.IOException;
 import java.nio.file.Path;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import java.util.function.LongSupplier;
 
@@ -282,11 +282,12 @@ private static Environment prepareEnvironment(final Settings settings, final Pat
     }
 
     @Override
-    public synchronized void close() throws IOException {
+    public synchronized boolean awaitClose(long timeout, TimeUnit timeUnit) throws InterruptedException {
         try {
-            super.close();
+            return super.awaitClose(timeout, timeUnit);
         } finally {
-            TestEntitlementBootstrap.unregisterNodeBaseDirs(getEnvironment().settings(), getEnvironment().configDir());
+            // wipePendingDataDirectories requires entitlement delegation to work due to this using FileSystemUtils ES-10920
+            // TestEntitlementBootstrap.unregisterNodeBaseDirs(settings(), getEnvironment().configDir());
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -546,7 +546,7 @@ public static void setupEntitlementsForClass() {
 
     @AfterClass
     public static void resetEntitlements() {
-        TestEntitlementBootstrap.reset();
+        TestEntitlementBootstrap.resetAfterTest();
     }
 
     // setup mock filesystems for this test run. we change PathUtils
diff --git a/test/framework/src/test/java/org/elasticsearch/entitlement/runtime/policy/TestPolicyManagerTests.java b/test/framework/src/test/java/org/elasticsearch/entitlement/runtime/policy/TestPolicyManagerTests.java
@@ -41,13 +41,13 @@ public void setupPolicyManager() {
         policyManager.setActive(true);
     }
 
-    public void testReset() {
+    public void testResetAfterTest() {
         assertTrue(policyManager.classEntitlementsMap.isEmpty());
         assertEquals("example-plugin1", policyManager.getEntitlements(getClass()).componentName());
         assertEquals("example-plugin1", policyManager.getEntitlements(getClass()).componentName());
         assertFalse(policyManager.classEntitlementsMap.isEmpty());
 
-        policyManager.reset();
+        policyManager.resetAfterTest();
 
         assertTrue(policyManager.classEntitlementsMap.isEmpty());
         assertEquals("example-plugin2", policyManager.getEntitlements(getClass()).componentName());
diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterBasicLicenseIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterBasicLicenseIT.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.license.LicenseSettings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.inference.LocalStateInferencePlugin;
 import org.elasticsearch.xpack.inference.Utils;
@@ -42,6 +43,7 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.instanceOf;
 
+@ESTestCase.WithoutEntitlements // due to dependency issue ES-12435
 public class ShardBulkInferenceActionFilterBasicLicenseIT extends ESIntegTestCase {
     public static final String INDEX_NAME = "test-index";
 
diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.xpack.inference.InferenceIndex;
 import org.elasticsearch.xpack.inference.LocalStateInferencePlugin;
@@ -56,6 +57,7 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
+@ESTestCase.WithoutEntitlements // due to dependency issue ES-12435
 public class ShardBulkInferenceActionFilterIT extends ESIntegTestCase {
     public static final String INDEX_NAME = "test-index";
 
diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/InferenceRevokeDefaultEndpointsIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/InferenceRevokeDefaultEndpointsIT.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.reindex.ReindexPlugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.http.MockResponse;
 import org.elasticsearch.test.http.MockWebServer;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -47,6 +48,7 @@
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.mockito.Mockito.mock;
 
+@ESTestCase.WithoutEntitlements // due to dependency issue ES-12435
 public class InferenceRevokeDefaultEndpointsIT extends ESSingleNodeTestCase {
     private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS);
 
diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java
diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticTextIndexOptionsIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticTextIndexOptionsIT.java
diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticTextIndexVersionIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticTextIndexVersionIT.java
diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListenerTests.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListenerTests.java
