diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java index 9fbba42d09ad3..da72315521423 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java @@ -34,9 +34,11 @@ import java.io.File; import java.util.List; import java.util.Map; +import java.util.stream.Stream; import javax.inject.Inject; +import static java.util.stream.Collectors.joining; import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; import static org.elasticsearch.gradle.util.FileUtils.mkdirs; import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure; @@ -173,6 +175,16 @@ public void execute(Task t) { // we use 'temp' relative to CWD since this is per JVM and tests are forbidden from writing to CWD nonInputProperties.systemProperty("java.io.tmpdir", test.getWorkingDir().toPath().resolve("temp")); + SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); + SourceSet mainSourceSet = sourceSets.findByName(SourceSet.MAIN_SOURCE_SET_NAME); + SourceSet testSourceSet = sourceSets.findByName(SourceSet.TEST_SOURCE_SET_NAME); + if ("test".equals(test.getName()) && mainSourceSet != null && testSourceSet != null) { + FileCollection mainRuntime = mainSourceSet.getRuntimeClasspath(); + FileCollection testRuntime = testSourceSet.getRuntimeClasspath(); + FileCollection testOnlyFiles = testRuntime.minus(mainRuntime); + test.doFirst(task -> test.environment("es.entitlement.testOnlyPath", testOnlyFiles.getAsPath())); + } + test.systemProperties(getProviderFactory().systemPropertiesPrefixedBy("tests.").get()); test.systemProperties(getProviderFactory().systemPropertiesPrefixedBy("es.").get()); @@ -205,46 +217,122 @@ public void execute(Task t) { } /* - * If this project builds a shadow JAR than any unit tests should test against that artifact instead of + * If this project builds a shadow JAR then any unit tests should test against that artifact instead of * compiled class output and dependency jars. This better emulates the runtime environment of consumers. */ project.getPluginManager().withPlugin("com.gradleup.shadow", p -> { if (test.getName().equals(JavaPlugin.TEST_TASK_NAME)) { // Remove output class files and any other dependencies from the test classpath, since the shadow JAR includes these - SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); - FileCollection mainRuntime = sourceSets.getByName(SourceSet.MAIN_SOURCE_SET_NAME).getRuntimeClasspath(); // Add any "shadow" dependencies. These are dependencies that are *not* bundled into the shadow JAR Configuration shadowConfig = project.getConfigurations().getByName(ShadowBasePlugin.CONFIGURATION_NAME); // Add the shadow JAR artifact itself FileCollection shadowJar = project.files(project.getTasks().named("shadowJar")); - FileCollection testRuntime = sourceSets.getByName(SourceSet.TEST_SOURCE_SET_NAME).getRuntimeClasspath(); + FileCollection mainRuntime = mainSourceSet.getRuntimeClasspath(); + FileCollection testRuntime = testSourceSet.getRuntimeClasspath(); test.setClasspath(testRuntime.minus(mainRuntime).plus(shadowConfig).plus(shadowJar)); } }); }); - configureImmutableCollectionsPatch(project); + configureJavaBaseModuleOptions(project); + configureEntitlements(project); + } + + /** + * Computes and sets the {@code --patch-module=java.base} and {@code --add-opens=java.base} JVM command line options. + */ + private void configureJavaBaseModuleOptions(Project project) { + project.getTasks().withType(Test.class).matching(task -> task.getName().equals("test")).configureEach(test -> { + FileCollection patchedImmutableCollections = patchedImmutableCollections(project); + if (patchedImmutableCollections != null) { + test.getInputs().files(patchedImmutableCollections); + test.systemProperty("tests.hackImmutableCollections", "true"); + } + + FileCollection entitlementBridge = entitlementBridge(project); + if (entitlementBridge != null) { + test.getInputs().files(entitlementBridge); + } + + test.getJvmArgumentProviders().add(() -> { + String javaBasePatch = Stream.concat( + singleFilePath(patchedImmutableCollections).map(str -> str + "/java.base"), + singleFilePath(entitlementBridge) + ).collect(joining(File.pathSeparator)); + + return javaBasePatch.isEmpty() + ? List.of() + : List.of("--patch-module=java.base=" + javaBasePatch, "--add-opens=java.base/java.util=ALL-UNNAMED"); + }); + }); } - private void configureImmutableCollectionsPatch(Project project) { + private Stream singleFilePath(FileCollection collection) { + return Stream.ofNullable(collection).filter(fc -> fc.isEmpty() == false).map(FileCollection::getSingleFile).map(File::toString); + } + + private static FileCollection patchedImmutableCollections(Project project) { String patchProject = ":test:immutable-collections-patch"; if (project.findProject(patchProject) == null) { - return; // build tests may not have this project, just skip + return null; // build tests may not have this project, just skip } String configurationName = "immutableCollectionsPatch"; FileCollection patchedFileCollection = project.getConfigurations() .create(configurationName, config -> config.setCanBeConsumed(false)); var deps = project.getDependencies(); deps.add(configurationName, deps.project(Map.of("path", patchProject, "configuration", "patch"))); - project.getTasks().withType(Test.class).matching(task -> task.getName().equals("test")).configureEach(test -> { - test.getInputs().files(patchedFileCollection); - test.systemProperty("tests.hackImmutableCollections", "true"); - test.getJvmArgumentProviders() - .add( - () -> List.of( - "--patch-module=java.base=" + patchedFileCollection.getSingleFile() + "/java.base", - "--add-opens=java.base/java.util=ALL-UNNAMED" - ) + return patchedFileCollection; + } + + private static FileCollection entitlementBridge(Project project) { + return project.getConfigurations().findByName("entitlementBridge"); + } + + /** + * Sets the required JVM options and system properties to enable entitlement enforcement on tests. + *

+ * One command line option is set in {@link #configureJavaBaseModuleOptions} out of necessity, + * since the command line can have only one {@code --patch-module} option for a given module. + */ + private static void configureEntitlements(Project project) { + Configuration agentConfig = project.getConfigurations().create("entitlementAgent"); + Project agent = project.findProject(":libs:entitlement:agent"); + if (agent != null) { + agentConfig.defaultDependencies( + deps -> { deps.add(project.getDependencies().project(Map.of("path", ":libs:entitlement:agent"))); } + ); + } + FileCollection agentFiles = agentConfig; + + Configuration bridgeConfig = project.getConfigurations().create("entitlementBridge"); + Project bridge = project.findProject(":libs:entitlement:bridge"); + if (bridge != null) { + bridgeConfig.defaultDependencies( + deps -> { deps.add(project.getDependencies().project(Map.of("path", ":libs:entitlement:bridge"))); } + ); + } + FileCollection bridgeFiles = bridgeConfig; + + project.getTasks().withType(Test.class).configureEach(test -> { + // See also SystemJvmOptions.maybeAttachEntitlementAgent. + + // Agent + if (agentFiles.isEmpty() == false) { + test.getInputs().files(agentFiles); + test.systemProperty("es.entitlement.agentJar", agentFiles.getAsPath()); + test.systemProperty("jdk.attach.allowAttachSelf", true); + } + + // Bridge + if (bridgeFiles.isEmpty() == false) { + String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net"; + test.getInputs().files(bridgeFiles); + // Tests may not be modular, but the JDK still is + test.jvmArgs( + "--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED," + + modulesContainingEntitlementInstrumentation ); + } }); } + } diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java index c967e8cb8b9fb..380d4fbf8414d 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java @@ -188,7 +188,7 @@ private ChangelogEntry makeHighlightsEntry(int pr, boolean notable) { } private String getResource(String name) throws Exception { - return Files.readString(Paths.get(Objects.requireNonNull(this.getClass().getResource(name)).toURI()), StandardCharsets.UTF_8); + return Files.readString(Paths.get(Objects.requireNonNull(this.getClass().getResource(name)).toURI()), StandardCharsets.UTF_8).replace("\r", ""); } private void writeResource(String name, String contents) throws Exception { diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java index 3cab57a333d2c..ed20d40582f57 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java @@ -18,8 +18,11 @@ import org.gradle.api.provider.ProviderFactory; import org.gradle.api.tasks.SourceSet; import org.gradle.api.tasks.SourceSetContainer; +import org.gradle.api.tasks.testing.Test; import org.gradle.language.jvm.tasks.ProcessResources; +import java.util.List; + import javax.inject.Inject; /** @@ -53,5 +56,11 @@ public void apply(Project project) { project.getTasks().withType(ProcessResources.class).named("processResources").configure(task -> { task.into("META-INF", copy -> copy.from(testBuildInfoTask)); }); + + if (project.getRootProject().getName().equals("elasticsearch")) { + project.getTasks().withType(Test.class).matching(test -> List.of("test").contains(test.getName())).configureEach(test -> { + test.systemProperty("es.entitlement.enableForTests", "true"); + }); + } } } diff --git a/libs/build.gradle b/libs/build.gradle index efd2329ca2b5e..b39ddaab98c2d 100644 --- a/libs/build.gradle +++ b/libs/build.gradle @@ -45,4 +45,14 @@ configure(childProjects.values()) { */ apply plugin: 'elasticsearch.build' } + + // This is for any code potentially included in the server at runtime. + // Omit oddball libraries that aren't in server. + def nonServerLibs = ['plugin-scanner'] + if (false == nonServerLibs.contains(project.name)) { + project.getTasks().withType(Test.class).matching(test -> ['test'].contains(test.name)).configureEach(test -> { + test.systemProperty('es.entitlement.enableForTests', 'true') + }) + } + } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyCheckerImpl.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyCheckerImpl.java index 5ea477c177740..2c3374f594847 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyCheckerImpl.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyCheckerImpl.java @@ -135,7 +135,7 @@ private void neverEntitled(Class callerClass, Supplier operationDescr Strings.format( "component [%s], module [%s], class [%s], operation [%s]", entitlements.componentName(), - PolicyCheckerImpl.getModuleName(requestingClass), + entitlements.moduleName(), requestingClass, operationDescription.get() ), @@ -247,7 +247,7 @@ public void checkFileRead(Class callerClass, Path path, boolean followLinks) Strings.format( "component [%s], module [%s], class [%s], entitlement [file], operation [read], path [%s]", entitlements.componentName(), - PolicyCheckerImpl.getModuleName(requestingClass), + entitlements.moduleName(), requestingClass, realPath == null ? path : Strings.format("%s -> %s", path, realPath) ), @@ -279,7 +279,7 @@ public void checkFileWrite(Class callerClass, Path path) { Strings.format( "component [%s], module [%s], class [%s], entitlement [file], operation [write], path [%s]", entitlements.componentName(), - PolicyCheckerImpl.getModuleName(requestingClass), + entitlements.moduleName(), requestingClass, path ), @@ -383,7 +383,7 @@ public void checkWriteProperty(Class callerClass, String property) { () -> Strings.format( "Entitled: component [%s], module [%s], class [%s], entitlement [write_system_properties], property [%s]", entitlements.componentName(), - PolicyCheckerImpl.getModuleName(requestingClass), + entitlements.moduleName(), requestingClass, property ) @@ -394,7 +394,7 @@ public void checkWriteProperty(Class callerClass, String property) { Strings.format( "component [%s], module [%s], class [%s], entitlement [write_system_properties], property [%s]", entitlements.componentName(), - PolicyCheckerImpl.getModuleName(requestingClass), + entitlements.moduleName(), requestingClass, property ), @@ -447,7 +447,7 @@ private void checkFlagEntitlement( Strings.format( "component [%s], module [%s], class [%s], entitlement [%s]", classEntitlements.componentName(), - PolicyCheckerImpl.getModuleName(requestingClass), + classEntitlements.moduleName(), requestingClass, PolicyParser.buildEntitlementNameFromClass(entitlementClass) ), @@ -460,7 +460,7 @@ private void checkFlagEntitlement( () -> Strings.format( "Entitled: component [%s], module [%s], class [%s], entitlement [%s]", classEntitlements.componentName(), - PolicyCheckerImpl.getModuleName(requestingClass), + classEntitlements.moduleName(), requestingClass, PolicyParser.buildEntitlementNameFromClass(entitlementClass) ) diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java index d05d9ad5858cd..f7d465adee631 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java @@ -118,8 +118,9 @@ public enum ComponentKind { * * @param componentName the plugin name or else one of the special component names like "(server)". */ - record ModuleEntitlements( + protected record ModuleEntitlements( String componentName, + String moduleName, Map, List> entitlementsByType, FileAccessTree fileAccess, Logger logger @@ -148,7 +149,13 @@ private FileAccessTree getDefaultFileAccess(Collection componentPaths) { // pkg private for testing ModuleEntitlements defaultEntitlements(String componentName, Collection componentPaths, String moduleName) { - return new ModuleEntitlements(componentName, Map.of(), getDefaultFileAccess(componentPaths), getLogger(componentName, moduleName)); + return new ModuleEntitlements( + componentName, + moduleName, + Map.of(), + getDefaultFileAccess(componentPaths), + getLogger(componentName, moduleName) + ); } // pkg private for testing @@ -166,6 +173,7 @@ ModuleEntitlements policyEntitlements( } return new ModuleEntitlements( componentName, + moduleName, entitlements.stream().collect(groupingBy(Entitlement::getClass)), FileAccessTree.of(componentName, moduleName, filesEntitlement, pathLookup, componentPaths, exclusivePaths), getLogger(componentName, moduleName) @@ -293,11 +301,11 @@ private static Logger getLogger(String componentName, String moduleName) { */ private static final ConcurrentHashMap MODULE_LOGGERS = new ConcurrentHashMap<>(); - ModuleEntitlements getEntitlements(Class requestingClass) { + protected ModuleEntitlements getEntitlements(Class requestingClass) { return moduleEntitlementsMap.computeIfAbsent(requestingClass.getModule(), m -> computeEntitlements(requestingClass)); } - private ModuleEntitlements computeEntitlements(Class requestingClass) { + protected final ModuleEntitlements computeEntitlements(Class requestingClass) { var policyScope = scopeResolver.apply(requestingClass); var componentName = policyScope.componentName(); var moduleName = policyScope.moduleName(); @@ -336,8 +344,7 @@ private ModuleEntitlements computeEntitlements(Class requestingClass) { } } - // pkg private for testing - static Collection getComponentPathsFromClass(Class requestingClass) { + protected Collection getComponentPathsFromClass(Class requestingClass) { var codeSource = requestingClass.getProtectionDomain().getCodeSource(); if (codeSource == null) { return List.of(); diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java index a8e66ffae0feb..21197fc6bd942 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java @@ -89,7 +89,6 @@ public void testGetEntitlements() { AtomicReference policyScope = new AtomicReference<>(); // A common policy with a variety of entitlements to test - Collection thisSourcePaths = PolicyManager.getComponentPathsFromClass(getClass()); var plugin1SourcePaths = List.of(Path.of("modules", "plugin1")); var policyManager = new PolicyManager( new Policy("server", List.of(new Scope("org.example.httpclient", List.of(new OutboundNetworkEntitlement())))), @@ -99,6 +98,7 @@ public void testGetEntitlements() { Map.of("plugin1", plugin1SourcePaths), TEST_PATH_LOOKUP ); + Collection thisSourcePaths = policyManager.getComponentPathsFromClass(getClass()); // "Unspecified" below means that the module is not named in the policy diff --git a/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java b/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java index 4916f578903ce..f0f4e870cb211 100644 --- a/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java +++ b/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.ingest.Processor; import org.elasticsearch.ingest.RandomDocumentPicks; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.ESTestCase.WithoutEntitlements; import org.junit.Before; import java.io.InputStream; @@ -38,6 +39,7 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +@WithoutEntitlements // ES-12084 public class AttachmentProcessorTests extends ESTestCase { private Processor processor; diff --git a/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java b/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java index c17570f3df0da..5545a5e1a8899 100644 --- a/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java +++ b/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java @@ -14,6 +14,7 @@ import org.apache.tika.metadata.Metadata; import org.elasticsearch.core.PathUtils; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.ESTestCase.WithoutEntitlements; import java.nio.file.DirectoryStream; import java.nio.file.Files; @@ -25,6 +26,7 @@ * comes back and no exception. */ @SuppressFileSystems("ExtrasFS") // don't try to parse extraN +@WithoutEntitlements // ES-12084 public class TikaDocTests extends ESTestCase { /** some test files from tika test suite, zipped up */ diff --git a/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaImplTests.java b/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaImplTests.java index baf51484245d4..094dd47dcb2e1 100644 --- a/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaImplTests.java +++ b/modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaImplTests.java @@ -9,7 +9,9 @@ package org.elasticsearch.ingest.attachment; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.ESTestCase.WithoutEntitlements; +@WithoutEntitlements // ES-12084 public class TikaImplTests extends ESTestCase { public void testTikaLoads() throws Exception { diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java index 1dc7e87004caf..f2624400d4812 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.ESTestCase.WithoutEntitlements; import org.junit.Before; import java.util.HashMap; @@ -18,6 +19,7 @@ import static org.hamcrest.Matchers.equalTo; +@WithoutEntitlements // ES-12084 public class RegisteredDomainProcessorFactoryTests extends ESTestCase { private RegisteredDomainProcessor.Factory factory; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java index b9fe870af2385..68255ec0ca5e4 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.ingest.TestIngestDocument; import org.elasticsearch.ingest.common.RegisteredDomainProcessor.DomainInfo; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.ESTestCase.WithoutEntitlements; import java.util.Collections; import java.util.Map; @@ -30,6 +31,7 @@ * Effective TLDs (eTLDs) are not the same as DNS TLDs. Uses for eTLDs are listed here: * https://publicsuffix.org/learn/ */ +@WithoutEntitlements // ES-12084 public class RegisteredDomainProcessorTests extends ESTestCase { public void testGetRegisteredDomain() { diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index 5e3c5d026f283..e38f2f88d4ad9 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -267,11 +267,18 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException { args.pidFile(), Set.of(EntitlementSelfTester.class.getPackage()) ); - EntitlementSelfTester.entitlementSelfTest(); + entitlementSelfTest(); bootstrap.setPluginsLoader(pluginsLoader); } + /** + * @throws IllegalStateException if entitlements aren't functioning properly. + */ + static void entitlementSelfTest() { + EntitlementSelfTester.entitlementSelfTest(); + } + private static void logSystemInfo() { final Logger logger = LogManager.getLogger(Elasticsearch.class); logger.info( diff --git a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java index bfa897173a368..dba50a794a780 100644 --- a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java @@ -217,7 +217,9 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) { }; Files.walkFileTree(startPath, visitor); - final Path testStartPath = PathUtils.get(ExceptionSerializationTests.class.getResource(path).toURI()); + final Path testStartPath = PathUtils.get( + ElasticsearchExceptionTests.class.getProtectionDomain().getCodeSource().getLocation().toURI() + ).resolve("org").resolve("elasticsearch"); Files.walkFileTree(testStartPath, visitor); assertTrue(notRegistered.remove(TestException.class)); assertTrue(notRegistered.remove(UnknownHeaderException.class)); diff --git a/server/src/test/java/org/elasticsearch/bootstrap/EntitlementMetaTests.java b/server/src/test/java/org/elasticsearch/bootstrap/EntitlementMetaTests.java new file mode 100644 index 0000000000000..04e59e5476f3b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/bootstrap/EntitlementMetaTests.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.bootstrap; + +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.entitlement.bootstrap.TestEntitlementBootstrap; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.ESTestCase.WithEntitlementsOnTestCode; + +import java.io.IOException; +import java.nio.file.Path; + +/** + * Ensures that unit tests are subject to entitlement checks. + * This is a "meta test" because it tests that the tests are working: + * if these tests fail, it means other tests won't be correctly detecting + * entitlement enforcement errors. + *

+ * It may seem strange to have this test where it is, rather than in the entitlement library. + * There's a reason for that. + *

+ * To exercise entitlement enforcement, we must attempt an operation that should be denied. + * This necessitates some operation that fails the entitlement check, + * and it must be in production code (or else we'd also need {@link WithEntitlementsOnTestCode}, + * and we don't want to require that here). + * Naturally, there are very few candidates, because most code doesn't fail entitlement checks: + * really just the entitlement self-test we do at startup. Hence, that's what we use here. + *

+ * Since we want to call the self-test, which is in the server, we can't call it + * from a place like the entitlement library tests, because those deliberately do not + * have a dependency on the server code. Hence, this test lives here in the server tests. + * + * @see WithoutEntitlementsMetaTests + * @see WithEntitlementsOnTestCodeMetaTests + */ +public class EntitlementMetaTests extends ESTestCase { + public void testSelfTestPasses() { + assumeTrue("Not yet working in serverless", TestEntitlementBootstrap.isEnabledForTest()); + Elasticsearch.entitlementSelfTest(); + } + + /** + * Unless {@link WithEntitlementsOnTestCode} is specified, sensitive methods can + * be called from test code. + */ + @SuppressForbidden(reason = "Testing that a forbidden API is allowed under these circumstances") + public void testForbiddenActionAllowedInTestCode() throws IOException { + // If entitlements were enforced, this would throw. + Path.of(".").toRealPath(); + } +} diff --git a/server/src/test/java/org/elasticsearch/bootstrap/WithEntitlementsOnTestCodeMetaTests.java b/server/src/test/java/org/elasticsearch/bootstrap/WithEntitlementsOnTestCodeMetaTests.java new file mode 100644 index 0000000000000..c126490922e48 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/bootstrap/WithEntitlementsOnTestCodeMetaTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.bootstrap; + +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.entitlement.bootstrap.TestEntitlementBootstrap; +import org.elasticsearch.entitlement.runtime.api.NotEntitledException; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.ESTestCase.WithEntitlementsOnTestCode; + +import java.nio.file.Path; + +/** + * A version of {@link EntitlementMetaTests} that tests {@link WithEntitlementsOnTestCode}. + * + * @see EntitlementMetaTests + * @see WithoutEntitlementsMetaTests + */ +@WithEntitlementsOnTestCode +public class WithEntitlementsOnTestCodeMetaTests extends ESTestCase { + /** + * {@link WithEntitlementsOnTestCode} should not affect this, since the sensitive method + * is called from server code. The self-test should pass as usual. + */ + public void testSelfTestPasses() { + assumeTrue("Not yet working in serverless", TestEntitlementBootstrap.isEnabledForTest()); + Elasticsearch.entitlementSelfTest(); + } + + @SuppressForbidden(reason = "Testing that a forbidden API is disallowed") + public void testForbiddenActionDenied() { + assumeTrue("Not yet working in serverless", TestEntitlementBootstrap.isEnabledForTest()); + assertThrows(NotEntitledException.class, () -> Path.of(".").toRealPath()); + } +} diff --git a/server/src/test/java/org/elasticsearch/bootstrap/WithoutEntitlementsMetaTests.java b/server/src/test/java/org/elasticsearch/bootstrap/WithoutEntitlementsMetaTests.java new file mode 100644 index 0000000000000..8ec9116a97ab8 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/bootstrap/WithoutEntitlementsMetaTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.bootstrap; + +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.ESTestCase.WithoutEntitlements; + +import java.io.IOException; +import java.nio.file.Path; + +/** + * A version of {@link EntitlementMetaTests} that tests {@link WithoutEntitlements}. + * + * @see EntitlementMetaTests + * @see WithEntitlementsOnTestCodeMetaTests + */ +@WithoutEntitlements +public class WithoutEntitlementsMetaTests extends ESTestCase { + /** + * Without enforcement of entitlements, {@link Elasticsearch#entitlementSelfTest} will fail and throw. + */ + public void testSelfTestFails() { + assertThrows(IllegalStateException.class, Elasticsearch::entitlementSelfTest); + } + + /** + * A forbidden action called from test code should be allowed, + * with or without {@link WithoutEntitlements}. + */ + @SuppressForbidden(reason = "Testing that a forbidden API is allowed under these circumstances") + public void testForbiddenActionAllowed() throws IOException { + // If entitlements were enforced, this would throw + Path.of(".").toRealPath(); + } +} diff --git a/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java b/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java index 47026fe713c5c..253abcf93dace 100644 --- a/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.ESTestCase.WithoutEntitlements; import org.elasticsearch.transport.TransportSettings; import org.mockito.Mockito; @@ -48,6 +49,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; +@WithoutEntitlements // Entitlement logging interferes public class ScopedSettingsTests extends ESTestCase { public void testResetSetting() { diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 680f6fca9652e..a41a5dd9394b4 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -2534,11 +2534,11 @@ public void testIndexWriterInfoStream() throws IllegalAccessException, IOExcepti MockAppender mockAppender = new MockAppender("testIndexWriterInfoStream"); mockAppender.start(); - Logger rootLogger = LogManager.getRootLogger(); - Level savedLevel = rootLogger.getLevel(); - Loggers.addAppender(rootLogger, mockAppender); - Loggers.setLevel(rootLogger, Level.DEBUG); - rootLogger = LogManager.getRootLogger(); + Logger theLogger = LogManager.getLogger("org.elasticsearch.index"); + Level savedLevel = theLogger.getLevel(); + Loggers.addAppender(theLogger, mockAppender); + Loggers.setLevel(theLogger, Level.DEBUG); + theLogger = LogManager.getLogger("org.elasticsearch.index"); try { // First, with DEBUG, which should NOT log IndexWriter output: @@ -2548,15 +2548,15 @@ public void testIndexWriterInfoStream() throws IllegalAccessException, IOExcepti assertFalse(mockAppender.sawIndexWriterMessage); // Again, with TRACE, which should log IndexWriter output: - Loggers.setLevel(rootLogger, Level.TRACE); + Loggers.setLevel(theLogger, Level.TRACE); engine.index(indexForDoc(doc)); engine.flush(); assertTrue(mockAppender.sawIndexWriterMessage); engine.close(); } finally { - Loggers.removeAppender(rootLogger, mockAppender); + Loggers.removeAppender(theLogger, mockAppender); mockAppender.stop(); - Loggers.setLevel(rootLogger, savedLevel); + Loggers.setLevel(theLogger, savedLevel); } } @@ -2596,10 +2596,10 @@ public void testMergeThreadLogging() throws Exception { final MockMergeThreadAppender mockAppender = new MockMergeThreadAppender("testMergeThreadLogging"); mockAppender.start(); - Logger rootLogger = LogManager.getRootLogger(); - Level savedLevel = rootLogger.getLevel(); - Loggers.addAppender(rootLogger, mockAppender); - Loggers.setLevel(rootLogger, Level.TRACE); + Logger theLogger = LogManager.getLogger("org.elasticsearch.index"); + Level savedLevel = theLogger.getLevel(); + Loggers.addAppender(theLogger, mockAppender); + Loggers.setLevel(theLogger, Level.TRACE); try { LogMergePolicy lmp = newLogMergePolicy(); lmp.setMergeFactor(2); @@ -2632,12 +2632,12 @@ public void testMergeThreadLogging() throws Exception { assertThat(mockAppender.mergeCompleted(), is(true)); }); - Loggers.setLevel(rootLogger, savedLevel); + Loggers.setLevel(theLogger, savedLevel); engine.close(); } } finally { - Loggers.setLevel(rootLogger, savedLevel); - Loggers.removeAppender(rootLogger, mockAppender); + Loggers.setLevel(theLogger, savedLevel); + Loggers.removeAppender(theLogger, mockAppender); mockAppender.stop(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java index 318f2ce863173..be709eaf5f43c 100644 --- a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java +++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java @@ -14,7 +14,9 @@ import org.elasticsearch.common.network.IfConfig; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Booleans; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.PathUtils; +import org.elasticsearch.entitlement.bootstrap.TestEntitlementBootstrap; import org.elasticsearch.jdk.JarHell; import java.io.IOException; @@ -71,6 +73,21 @@ public class BootstrapForTesting { // Log ifconfig output before SecurityManager is installed IfConfig.logIfNecessary(); + + // Fire up entitlements + try { + TestEntitlementBootstrap.bootstrap(javaTmpDir, maybePath(System.getProperty("tests.config"))); + } catch (IOException e) { + throw new IllegalStateException(e.getClass().getSimpleName() + " while initializing entitlements for tests", e); + } + } + + private static @Nullable Path maybePath(String str) { + if (str == null) { + return null; + } else { + return PathUtils.get(str); + } } // does nothing, just easy way to make sure the class is loaded. diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/TestScopeResolver.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/TestScopeResolver.java index c29bd84d1fda9..3a42485822f3c 100644 --- a/test/framework/src/main/java/org/elasticsearch/bootstrap/TestScopeResolver.java +++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/TestScopeResolver.java @@ -16,11 +16,15 @@ import java.net.MalformedURLException; import java.net.URL; -import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.TreeMap; import java.util.function.Function; +import static org.elasticsearch.entitlement.runtime.policy.PolicyManager.ALL_UNNAMED; +import static org.elasticsearch.entitlement.runtime.policy.PolicyManager.ComponentKind.PLUGIN; + public record TestScopeResolver(Map scopeMap) { private static final Logger logger = LogManager.getLogger(TestScopeResolver.class); @@ -31,6 +35,13 @@ PolicyManager.PolicyScope getScope(Class callerClass) { var location = callerCodeSource.getLocation().toString(); var scope = scopeMap.get(location); + if (scope == null) { + // Special cases for libraries not handled by our automatically-generated scopeMap + if (callerClass.getPackageName().startsWith("org.bouncycastle")) { + scope = new PolicyManager.PolicyScope(PLUGIN, "security", ALL_UNNAMED); + logger.debug("Assuming bouncycastle is part of the security plugin"); + } + } if (scope == null) { logger.warn("Cannot identify a scope for class [{}], location [{}]", callerClass.getName(), location); return PolicyManager.PolicyScope.unknown(location); @@ -40,20 +51,22 @@ PolicyManager.PolicyScope getScope(Class callerClass) { public static Function, PolicyManager.PolicyScope> createScopeResolver( TestBuildInfo serverBuildInfo, - List pluginsBuildInfo + List pluginsBuildInfo, + Set modularPlugins ) { - - Map scopeMap = new HashMap<>(); + Map scopeMap = new TreeMap<>(); // Sorted to make it easier to read during debugging for (var pluginBuildInfo : pluginsBuildInfo) { + boolean isModular = modularPlugins.contains(pluginBuildInfo.component()); for (var location : pluginBuildInfo.locations()) { var codeSource = TestScopeResolver.class.getClassLoader().getResource(location.representativeClass()); if (codeSource == null) { throw new IllegalArgumentException("Cannot locate class [" + location.representativeClass() + "]"); } try { + String module = isModular ? location.module() : ALL_UNNAMED; scopeMap.put( getCodeSource(codeSource, location.representativeClass()), - PolicyManager.PolicyScope.plugin(pluginBuildInfo.component(), location.module()) + PolicyManager.PolicyScope.plugin(pluginBuildInfo.component(), module) ); } catch (MalformedURLException e) { throw new IllegalArgumentException("Cannot locate class [" + location.representativeClass() + "]", e); @@ -64,7 +77,8 @@ public static Function, PolicyManager.PolicyScope> createScopeResolver( for (var location : serverBuildInfo.locations()) { var classUrl = TestScopeResolver.class.getClassLoader().getResource(location.representativeClass()); if (classUrl == null) { - throw new IllegalArgumentException("Cannot locate class [" + location.representativeClass() + "]"); + logger.debug("Representative class is unavailable; proceeding without {}", location); + continue; } try { scopeMap.put(getCodeSource(classUrl, location.representativeClass()), PolicyManager.PolicyScope.server(location.module())); diff --git a/test/framework/src/main/java/org/elasticsearch/entitlement/bootstrap/TestEntitlementBootstrap.java b/test/framework/src/main/java/org/elasticsearch/entitlement/bootstrap/TestEntitlementBootstrap.java index f4573f5061cc9..c2d56a1acea74 100644 --- a/test/framework/src/main/java/org/elasticsearch/entitlement/bootstrap/TestEntitlementBootstrap.java +++ b/test/framework/src/main/java/org/elasticsearch/entitlement/bootstrap/TestEntitlementBootstrap.java @@ -12,13 +12,16 @@ import org.elasticsearch.bootstrap.TestBuildInfo; import org.elasticsearch.bootstrap.TestBuildInfoParser; import org.elasticsearch.bootstrap.TestScopeResolver; +import org.elasticsearch.core.Booleans; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.PathUtils; import org.elasticsearch.core.Strings; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.entitlement.initialization.EntitlementInitialization; import org.elasticsearch.entitlement.runtime.policy.PathLookup; import org.elasticsearch.entitlement.runtime.policy.Policy; -import org.elasticsearch.entitlement.runtime.policy.PolicyManager; import org.elasticsearch.entitlement.runtime.policy.PolicyParser; +import org.elasticsearch.entitlement.runtime.policy.TestPathLookup; import org.elasticsearch.entitlement.runtime.policy.TestPolicyManager; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; @@ -26,78 +29,127 @@ import java.io.IOException; import java.io.InputStream; +import java.net.URI; import java.net.URL; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Stream; +import java.util.TreeSet; + +import static java.util.stream.Collectors.toCollection; +import static java.util.stream.Collectors.toMap; +import static java.util.stream.Collectors.toSet; +import static org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir.CONFIG; +import static org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir.TEMP; public class TestEntitlementBootstrap { private static final Logger logger = LogManager.getLogger(TestEntitlementBootstrap.class); + private static TestPolicyManager policyManager; + /** * Activates entitlement checking in tests. */ - public static void bootstrap() throws IOException { - TestPathLookup pathLookup = new TestPathLookup(); - EntitlementInitialization.initializeArgs = new EntitlementInitialization.InitializeArgs( - pathLookup, - Set.of(), - createPolicyManager(pathLookup) - ); + public static void bootstrap(@Nullable Path tempDir, @Nullable Path configDir) throws IOException { + if (isEnabledForTest() == false) { + return; + } + TestPathLookup pathLookup = new TestPathLookup(Map.of(TEMP, zeroOrOne(tempDir), CONFIG, zeroOrOne(configDir))); + policyManager = createPolicyManager(pathLookup); + EntitlementInitialization.initializeArgs = new EntitlementInitialization.InitializeArgs(pathLookup, Set.of(), policyManager); logger.debug("Loading entitlement agent"); EntitlementBootstrap.loadAgent(EntitlementBootstrap.findAgentJar(), EntitlementInitialization.class.getName()); } - private record TestPathLookup() implements PathLookup { - @Override - public Path pidFile() { - return null; + private static List zeroOrOne(T item) { + if (item == null) { + return List.of(); + } else { + return List.of(item); } + } - @Override - public Stream getBaseDirPaths(BaseDir baseDir) { - return Stream.empty(); - } + public static boolean isEnabledForTest() { + return Booleans.parseBoolean(System.getProperty("es.entitlement.enableForTests", "false")); + } - @Override - public Stream resolveSettingPaths(BaseDir baseDir, String settingName) { - return Stream.empty(); - } + public static void setActive(boolean newValue) { + policyManager.setActive(newValue); + } + public static void setTriviallyAllowingTestCode(boolean newValue) { + policyManager.setTriviallyAllowingTestCode(newValue); } - private static PolicyManager createPolicyManager(PathLookup pathLookup) throws IOException { + public static void reset() { + if (policyManager != null) { + policyManager.reset(); + } + } + private static TestPolicyManager createPolicyManager(PathLookup pathLookup) throws IOException { var pluginsTestBuildInfo = TestBuildInfoParser.parseAllPluginTestBuildInfo(); var serverTestBuildInfo = TestBuildInfoParser.parseServerTestBuildInfo(); - var scopeResolver = TestScopeResolver.createScopeResolver(serverTestBuildInfo, pluginsTestBuildInfo); List pluginNames = pluginsTestBuildInfo.stream().map(TestBuildInfo::component).toList(); var pluginDescriptors = parsePluginsDescriptors(pluginNames); + Set modularPlugins = pluginDescriptors.stream() + .filter(PluginDescriptor::isModular) + .map(PluginDescriptor::getName) + .collect(toSet()); + var scopeResolver = TestScopeResolver.createScopeResolver(serverTestBuildInfo, pluginsTestBuildInfo, modularPlugins); var pluginsData = pluginDescriptors.stream() .map(descriptor -> new TestPluginData(descriptor.getName(), descriptor.isModular(), false)) .toList(); Map pluginPolicies = parsePluginsPolicies(pluginsData); + String separator = System.getProperty("path.separator"); + + // In productions, plugins would have access to their respective bundle directories, + // and so they'd be able to read from their jars. In testing, we approximate this + // by considering the entire classpath to be "source paths" of all plugins. This + // also has the effect of granting read access to everything on the test-only classpath, + // which is fine, because any entitlement errors there could only be false positives. + String classPathProperty = System.getProperty("java.class.path"); + + Set classPathEntries; + if (classPathProperty == null) { + classPathEntries = Set.of(); + } else { + classPathEntries = Arrays.stream(classPathProperty.split(separator)).map(PathUtils::get).collect(toCollection(TreeSet::new)); + } + Map> pluginSourcePaths = pluginNames.stream().collect(toMap(n -> n, n -> classPathEntries)); + FilesEntitlementsValidation.validate(pluginPolicies, pathLookup); + String testOnlyPathString = System.getenv("es.entitlement.testOnlyPath"); + Set testOnlyClassPath; + if (testOnlyPathString == null) { + testOnlyClassPath = Set.of(); + } else { + testOnlyClassPath = Arrays.stream(testOnlyPathString.split(separator)) + .map(PathUtils::get) + .map(Path::toUri) + .collect(toCollection(TreeSet::new)); + } + return new TestPolicyManager( HardcodedEntitlements.serverPolicy(null, null), HardcodedEntitlements.agentEntitlements(), pluginPolicies, scopeResolver, - Map.of(), - pathLookup + pluginSourcePaths, + pathLookup, + testOnlyClassPath ); } - private record TestPluginData(String pluginName, boolean isModular, boolean isExternalPlugin) {} - private static Map parsePluginsPolicies(List pluginsData) { Map policies = new HashMap<>(); for (var pluginData : pluginsData) { @@ -137,4 +189,6 @@ private static InputStream getStream(URL resource) throws IOException { return resource.openStream(); } + private record TestPluginData(String pluginName, boolean isModular, boolean isExternalPlugin) {} + } diff --git a/test/framework/src/main/java/org/elasticsearch/entitlement/runtime/policy/TestPathLookup.java b/test/framework/src/main/java/org/elasticsearch/entitlement/runtime/policy/TestPathLookup.java index 915f14c9d1ab8..be99d8187f95e 100644 --- a/test/framework/src/main/java/org/elasticsearch/entitlement/runtime/policy/TestPathLookup.java +++ b/test/framework/src/main/java/org/elasticsearch/entitlement/runtime/policy/TestPathLookup.java @@ -10,9 +10,18 @@ package org.elasticsearch.entitlement.runtime.policy; import java.nio.file.Path; +import java.util.Collection; +import java.util.List; +import java.util.Map; import java.util.stream.Stream; public class TestPathLookup implements PathLookup { + final Map> baseDirPaths; + + public TestPathLookup(Map> baseDirPaths) { + this.baseDirPaths = baseDirPaths; + } + @Override public Path pidFile() { return null; @@ -20,7 +29,7 @@ public Path pidFile() { @Override public Stream getBaseDirPaths(BaseDir baseDir) { - return Stream.empty(); + return baseDirPaths.getOrDefault(baseDir, List.of()).stream(); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/entitlement/runtime/policy/TestPolicyManager.java b/test/framework/src/main/java/org/elasticsearch/entitlement/runtime/policy/TestPolicyManager.java index 2acb31182c1f8..2acf549231950 100644 --- a/test/framework/src/main/java/org/elasticsearch/entitlement/runtime/policy/TestPolicyManager.java +++ b/test/framework/src/main/java/org/elasticsearch/entitlement/runtime/policy/TestPolicyManager.java @@ -10,30 +10,64 @@ package org.elasticsearch.entitlement.runtime.policy; import org.elasticsearch.entitlement.runtime.policy.entitlements.Entitlement; +import org.elasticsearch.test.ESTestCase; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.file.Path; +import java.security.CodeSource; +import java.security.ProtectionDomain; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; +import static java.util.Objects.requireNonNull; + public class TestPolicyManager extends PolicyManager { + + boolean isActive; + boolean isTriviallyAllowingTestCode; + + /** + * We don't have modules in tests, so we can't use the inherited map of entitlements per module. + * We need this larger map per class instead. + */ + final Map, ModuleEntitlements> classEntitlementsMap = new ConcurrentHashMap<>(); + + final Collection testOnlyClasspath; + public TestPolicyManager( Policy serverPolicy, List apmAgentEntitlements, Map pluginPolicies, Function, PolicyScope> scopeResolver, Map> pluginSourcePaths, - PathLookup pathLookup + PathLookup pathLookup, + Collection testOnlyClasspath ) { super(serverPolicy, apmAgentEntitlements, pluginPolicies, scopeResolver, pluginSourcePaths, pathLookup); + this.testOnlyClasspath = testOnlyClasspath; + reset(); + } + + public void setActive(boolean newValue) { + this.isActive = newValue; + } + + public void setTriviallyAllowingTestCode(boolean newValue) { + this.isTriviallyAllowingTestCode = newValue; } /** * Called between tests so each test is not affected by prior tests */ - public void reset() { - super.moduleEntitlementsMap.clear(); + public final void reset() { + assert moduleEntitlementsMap.isEmpty() : "We're not supposed to be using moduleEntitlementsMap in tests"; + classEntitlementsMap.clear(); + isActive = false; + isTriviallyAllowingTestCode = true; } @Override @@ -44,7 +78,31 @@ protected boolean isTrustedSystemClass(Class requestingClass) { @Override boolean isTriviallyAllowed(Class requestingClass) { - return isTestFrameworkClass(requestingClass) || isEntitlementClass(requestingClass) || super.isTriviallyAllowed(requestingClass); + if (isActive == false) { + return true; + } + if (isEntitlementClass(requestingClass)) { + return true; + } + if (isTestFrameworkClass(requestingClass)) { + return true; + } + if ("org.elasticsearch.jdk".equals(requestingClass.getPackageName())) { + // PluginsLoaderTests, PluginsServiceTests, PluginsUtilsTests + return true; + } + if ("org.elasticsearch.nativeaccess".equals(requestingClass.getPackageName())) { + // UberModuleClassLoaderTests + return true; + } + if (requestingClass.getPackageName().startsWith("org.elasticsearch.plugins")) { + // PluginsServiceTests, NamedComponentReaderTests + return true; + } + if (isTriviallyAllowingTestCode && isTestCode(requestingClass)) { + return true; + } + return super.isTriviallyAllowed(requestingClass); } private boolean isEntitlementClass(Class requestingClass) { @@ -52,8 +110,59 @@ private boolean isEntitlementClass(Class requestingClass) { && (requestingClass.getName().contains("Test") == false); } + @Deprecated // TODO: reevaluate whether we want this. + // If we can simply check for dependencies the gradle worker has that aren't + // declared in the gradle config (namely org.gradle) that would be simpler. private boolean isTestFrameworkClass(Class requestingClass) { String packageName = requestingClass.getPackageName(); - return packageName.startsWith("org.junit") || packageName.startsWith("org.gradle"); + for (String prefix : TEST_FRAMEWORK_PACKAGE_PREFIXES) { + if (packageName.startsWith(prefix)) { + return true; + } + } + return false; + } + + private boolean isTestCode(Class requestingClass) { + // TODO: Cache this? It's expensive + for (Class candidate = requireNonNull(requestingClass); candidate != null; candidate = candidate.getDeclaringClass()) { + if (ESTestCase.class.isAssignableFrom(candidate)) { + return true; + } + } + ProtectionDomain protectionDomain = requestingClass.getProtectionDomain(); + CodeSource codeSource = protectionDomain.getCodeSource(); + if (codeSource == null) { + // This can happen for JDK classes + return false; + } + URI needle; + try { + needle = codeSource.getLocation().toURI(); + } catch (URISyntaxException e) { + throw new IllegalStateException(e); + } + boolean result = testOnlyClasspath.contains(needle); + return result; + } + + private static final String[] TEST_FRAMEWORK_PACKAGE_PREFIXES = { + "org.gradle", + + "org.jcodings", // A library loaded with SPI that tries to create a CharsetProvider + "com.google.common.jimfs", // Used on Windows + + // We shouldn't really need the rest of these. They should be discovered on the testOnlyClasspath. + "com.carrotsearch.randomizedtesting", + "com.sun.tools.javac", + "org.apache.lucene.tests", // Interferes with SSLErrorMessageFileTests.testMessageForPemCertificateOutsideConfigDir + "org.junit", + "org.mockito", + "net.bytebuddy", // Mockito uses this + }; + + @Override + protected ModuleEntitlements getEntitlements(Class requestingClass) { + return classEntitlementsMap.computeIfAbsent(requestingClass, this::computeEntitlements); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index afbabe110aa4e..884c1caf85488 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -285,6 +285,7 @@ * */ @LuceneTestCase.SuppressFileSystems("ExtrasFS") // doesn't work with potential multi data path from test cluster yet +@ESTestCase.WithoutEntitlements // ES-12042 public abstract class ESIntegTestCase extends ESTestCase { /** node names of the corresponding clusters will start with these prefixes */ diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index 7ebc5765bda63..f7d272e793e0f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -90,6 +90,7 @@ * A test that keep a singleton node started for all tests that can be used to get * references to Guice injectors in unit tests. */ +@ESTestCase.WithoutEntitlements // ES-12042 public abstract class ESSingleNodeTestCase extends ESTestCase { private static Node NODE = null; diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 7183a43c6f731..ffc0e00909c81 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -111,6 +111,7 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; +import org.elasticsearch.entitlement.bootstrap.TestEntitlementBootstrap; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.TestEnvironment; @@ -164,6 +165,11 @@ import java.io.IOException; import java.io.InputStream; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; import java.lang.invoke.MethodHandles; import java.math.BigInteger; import java.net.InetAddress; @@ -492,6 +498,44 @@ protected void afterIfFailed(List errors) {} /** called after a test is finished, but only if successful */ protected void afterIfSuccessful() throws Exception {} + /** + * Marks a test suite or a test method that should run without checking for entitlements. + */ + @Retention(RetentionPolicy.RUNTIME) + @Target(ElementType.TYPE) + @Inherited + public @interface WithoutEntitlements { + } + + /** + * Marks a test suite or a test method that enforce entitlements on the test code itself. + * Useful for testing the enforcement of entitlements; for any other test cases, this probably isn't what you want. + */ + @Retention(RetentionPolicy.RUNTIME) + @Target(ElementType.TYPE) + @Inherited + public @interface WithEntitlementsOnTestCode { + } + + @BeforeClass + public static void setupEntitlementsForClass() { + boolean withoutEntitlements = getTestClass().isAnnotationPresent(WithoutEntitlements.class); + boolean withEntitlementsOnTestCode = getTestClass().isAnnotationPresent(WithEntitlementsOnTestCode.class); + if (TestEntitlementBootstrap.isEnabledForTest()) { + TestEntitlementBootstrap.setActive(false == withoutEntitlements); + TestEntitlementBootstrap.setTriviallyAllowingTestCode(false == withEntitlementsOnTestCode); + } else if (withEntitlementsOnTestCode) { + throw new AssertionError( + "Cannot use WithEntitlementsOnTestCode on tests that are not configured to use entitlements for testing" + ); + } + } + + @AfterClass + public static void resetEntitlements() { + TestEntitlementBootstrap.reset(); + } + // setup mock filesystems for this test run. we change PathUtils // so that all accesses are plumbed thru any mock wrappers diff --git a/test/framework/src/test/java/org/elasticsearch/bootstrap/TestScopeResolverTests.java b/test/framework/src/test/java/org/elasticsearch/bootstrap/TestScopeResolverTests.java index 24d8f26342797..e489d8dc7a17c 100644 --- a/test/framework/src/test/java/org/elasticsearch/bootstrap/TestScopeResolverTests.java +++ b/test/framework/src/test/java/org/elasticsearch/bootstrap/TestScopeResolverTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.test.ESTestCase; import java.util.List; +import java.util.Set; import static org.hamcrest.Matchers.is; @@ -23,7 +24,7 @@ public void testScopeResolverServerClass() { "server", List.of(new TestBuildInfoLocation("org/elasticsearch/Build.class", "org.elasticsearch.server")) ); - var resolver = TestScopeResolver.createScopeResolver(testBuildInfo, List.of()); + var resolver = TestScopeResolver.createScopeResolver(testBuildInfo, List.of(), Set.of()); var scope = resolver.apply(Plugin.class); assertThat(scope.componentName(), is("(server)")); @@ -39,7 +40,7 @@ public void testScopeResolverInternalClass() { "test-component", List.of(new TestBuildInfoLocation("org/elasticsearch/bootstrap/TestBuildInfoParserTests.class", "test-module-name")) ); - var resolver = TestScopeResolver.createScopeResolver(testBuildInfo, List.of(testOwnBuildInfo)); + var resolver = TestScopeResolver.createScopeResolver(testBuildInfo, List.of(testOwnBuildInfo), Set.of("test-component")); var scope = resolver.apply(this.getClass()); assertThat(scope.componentName(), is("test-component")); diff --git a/test/framework/src/test/java/org/elasticsearch/entitlement/runtime/policy/TestPolicyManagerTests.java b/test/framework/src/test/java/org/elasticsearch/entitlement/runtime/policy/TestPolicyManagerTests.java index 1efe76f82c8b5..bf59174410948 100644 --- a/test/framework/src/test/java/org/elasticsearch/entitlement/runtime/policy/TestPolicyManagerTests.java +++ b/test/framework/src/test/java/org/elasticsearch/entitlement/runtime/policy/TestPolicyManagerTests.java @@ -31,22 +31,24 @@ public void setupPolicyManager() { Map.of(), c -> new PolicyScope(PLUGIN, "example-plugin" + scopeCounter.incrementAndGet(), "org.example.module"), Map.of(), - new TestPathLookup() + new TestPathLookup(Map.of()), + List.of() ); + policyManager.setActive(true); } public void testReset() { - assertTrue(policyManager.moduleEntitlementsMap.isEmpty()); + assertTrue(policyManager.classEntitlementsMap.isEmpty()); assertEquals("example-plugin1", policyManager.getEntitlements(getClass()).componentName()); assertEquals("example-plugin1", policyManager.getEntitlements(getClass()).componentName()); - assertFalse(policyManager.moduleEntitlementsMap.isEmpty()); + assertFalse(policyManager.classEntitlementsMap.isEmpty()); policyManager.reset(); - assertTrue(policyManager.moduleEntitlementsMap.isEmpty()); + assertTrue(policyManager.classEntitlementsMap.isEmpty()); assertEquals("example-plugin2", policyManager.getEntitlements(getClass()).componentName()); assertEquals("example-plugin2", policyManager.getEntitlements(getClass()).componentName()); - assertFalse(policyManager.moduleEntitlementsMap.isEmpty()); + assertFalse(policyManager.classEntitlementsMap.isEmpty()); } public void testIsTriviallyAllowed() { @@ -54,6 +56,8 @@ public void testIsTriviallyAllowed() { assertTrue(policyManager.isTriviallyAllowed(org.junit.Before.class)); assertTrue(policyManager.isTriviallyAllowed(PolicyManager.class)); + assertTrue(policyManager.isTriviallyAllowed(getClass())); + policyManager.setTriviallyAllowingTestCode(false); assertFalse(policyManager.isTriviallyAllowed(getClass())); } } diff --git a/x-pack/plugin/core/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/core/src/main/plugin-metadata/entitlement-policy.yaml index 5b089eba8b5b1..60d08fef7cef0 100644 --- a/x-pack/plugin/core/src/main/plugin-metadata/entitlement-policy.yaml +++ b/x-pack/plugin/core/src/main/plugin-metadata/entitlement-policy.yaml @@ -18,6 +18,7 @@ org.apache.httpcomponents.httpasyncclient: - manage_threads unboundid.ldapsdk: - set_https_connection_properties # TODO: review if we need this once we have proper test coverage + - inbound_network # For com.unboundid.ldap.listener.LDAPListener - outbound_network - manage_threads - write_system_properties: diff --git a/x-pack/plugin/monitoring/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/monitoring/src/main/plugin-metadata/entitlement-policy.yaml index 27ff2988cdcbe..ba7219af73c42 100644 --- a/x-pack/plugin/monitoring/src/main/plugin-metadata/entitlement-policy.yaml +++ b/x-pack/plugin/monitoring/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,5 +1,6 @@ ALL-UNNAMED: - set_https_connection_properties # potentially required by apache.httpcomponents + - manage_threads # For org.elasticsearch.client.snif.Sniffer # the original policy has java.net.SocketPermission "*", "accept,connect" # but a comment stating it was "needed for multiple server implementations used in tests" # TODO: this is likely not needed, but including here to be on the safe side until