[9.1] Bootstrap entitlements for testing (#129268)

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -34,9 +34,11 @@
 import java.io.File;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Stream;
 
 import javax.inject.Inject;
 
+import static java.util.stream.Collectors.joining;
 import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams;
 import static org.elasticsearch.gradle.util.FileUtils.mkdirs;
 import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure;
@@ -173,6 +175,16 @@ public void execute(Task t) {
             // we use 'temp' relative to CWD since this is per JVM and tests are forbidden from writing to CWD
             nonInputProperties.systemProperty("java.io.tmpdir", test.getWorkingDir().toPath().resolve("temp"));
 
+            SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+            SourceSet mainSourceSet = sourceSets.findByName(SourceSet.MAIN_SOURCE_SET_NAME);
+            SourceSet testSourceSet = sourceSets.findByName(SourceSet.TEST_SOURCE_SET_NAME);
+            if ("test".equals(test.getName()) && mainSourceSet != null && testSourceSet != null) {
+                FileCollection mainRuntime = mainSourceSet.getRuntimeClasspath();
+                FileCollection testRuntime = testSourceSet.getRuntimeClasspath();
+                FileCollection testOnlyFiles = testRuntime.minus(mainRuntime);
+                test.doFirst(task -> test.environment("es.entitlement.testOnlyPath", testOnlyFiles.getAsPath()));
+            }
+
             test.systemProperties(getProviderFactory().systemPropertiesPrefixedBy("tests.").get());
             test.systemProperties(getProviderFactory().systemPropertiesPrefixedBy("es.").get());
 
@@ -205,46 +217,122 @@ public void execute(Task t) {
             }
 
             /*
-             *  If this project builds a shadow JAR than any unit tests should test against that artifact instead of
+             *  If this project builds a shadow JAR then any unit tests should test against that artifact instead of
              *  compiled class output and dependency jars. This better emulates the runtime environment of consumers.
              */
             project.getPluginManager().withPlugin("com.gradleup.shadow", p -> {
                 if (test.getName().equals(JavaPlugin.TEST_TASK_NAME)) {
                     // Remove output class files and any other dependencies from the test classpath, since the shadow JAR includes these
-                    SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-                    FileCollection mainRuntime = sourceSets.getByName(SourceSet.MAIN_SOURCE_SET_NAME).getRuntimeClasspath();
                     // Add any "shadow" dependencies. These are dependencies that are *not* bundled into the shadow JAR
                     Configuration shadowConfig = project.getConfigurations().getByName(ShadowBasePlugin.CONFIGURATION_NAME);
                     // Add the shadow JAR artifact itself
                     FileCollection shadowJar = project.files(project.getTasks().named("shadowJar"));
-                    FileCollection testRuntime = sourceSets.getByName(SourceSet.TEST_SOURCE_SET_NAME).getRuntimeClasspath();
+                    FileCollection mainRuntime = mainSourceSet.getRuntimeClasspath();
+                    FileCollection testRuntime = testSourceSet.getRuntimeClasspath();
                     test.setClasspath(testRuntime.minus(mainRuntime).plus(shadowConfig).plus(shadowJar));
                 }
             });
         });
-        configureImmutableCollectionsPatch(project);
+        configureJavaBaseModuleOptions(project);
+        configureEntitlements(project);
+    }
+
+    /**
+     * Computes and sets the {@code --patch-module=java.base} and {@code --add-opens=java.base} JVM command line options.
+     */
+    private void configureJavaBaseModuleOptions(Project project) {
+        project.getTasks().withType(Test.class).matching(task -> task.getName().equals("test")).configureEach(test -> {
+            FileCollection patchedImmutableCollections = patchedImmutableCollections(project);
+            if (patchedImmutableCollections != null) {
+                test.getInputs().files(patchedImmutableCollections);
+                test.systemProperty("tests.hackImmutableCollections", "true");
+            }
+
+            FileCollection entitlementBridge = entitlementBridge(project);
+            if (entitlementBridge != null) {
+                test.getInputs().files(entitlementBridge);
+            }
+
+            test.getJvmArgumentProviders().add(() -> {
+                String javaBasePatch = Stream.concat(
+                    singleFilePath(patchedImmutableCollections).map(str -> str + "/java.base"),
+                    singleFilePath(entitlementBridge)
+                ).collect(joining(File.pathSeparator));
+
+                return javaBasePatch.isEmpty()
+                    ? List.of()
+                    : List.of("--patch-module=java.base=" + javaBasePatch, "--add-opens=java.base/java.util=ALL-UNNAMED");
+            });
+        });
     }
 
-    private void configureImmutableCollectionsPatch(Project project) {
+    private Stream<String> singleFilePath(FileCollection collection) {
+        return Stream.ofNullable(collection).filter(fc -> fc.isEmpty() == false).map(FileCollection::getSingleFile).map(File::toString);
+    }
+
+    private static FileCollection patchedImmutableCollections(Project project) {
         String patchProject = ":test:immutable-collections-patch";
         if (project.findProject(patchProject) == null) {
-            return; // build tests may not have this project, just skip
+            return null; // build tests may not have this project, just skip
         }
         String configurationName = "immutableCollectionsPatch";
         FileCollection patchedFileCollection = project.getConfigurations()
             .create(configurationName, config -> config.setCanBeConsumed(false));
         var deps = project.getDependencies();
         deps.add(configurationName, deps.project(Map.of("path", patchProject, "configuration", "patch")));
-        project.getTasks().withType(Test.class).matching(task -> task.getName().equals("test")).configureEach(test -> {
-            test.getInputs().files(patchedFileCollection);
-            test.systemProperty("tests.hackImmutableCollections", "true");
-            test.getJvmArgumentProviders()
-                .add(
-                    () -> List.of(
-                        "--patch-module=java.base=" + patchedFileCollection.getSingleFile() + "/java.base",
-                        "--add-opens=java.base/java.util=ALL-UNNAMED"
-                    )
+        return patchedFileCollection;
+    }
+
+    private static FileCollection entitlementBridge(Project project) {
+        return project.getConfigurations().findByName("entitlementBridge");
+    }
+
+    /**
+     * Sets the required JVM options and system properties to enable entitlement enforcement on tests.
+     * <p>
+     * One command line option is set in {@link #configureJavaBaseModuleOptions} out of necessity,
+     * since the command line can have only one {@code --patch-module} option for a given module.
+     */
+    private static void configureEntitlements(Project project) {
+        Configuration agentConfig = project.getConfigurations().create("entitlementAgent");
+        Project agent = project.findProject(":libs:entitlement:agent");
+        if (agent != null) {
+            agentConfig.defaultDependencies(
+                deps -> { deps.add(project.getDependencies().project(Map.of("path", ":libs:entitlement:agent"))); }
+            );
+        }
+        FileCollection agentFiles = agentConfig;
+
+        Configuration bridgeConfig = project.getConfigurations().create("entitlementBridge");
+        Project bridge = project.findProject(":libs:entitlement:bridge");
+        if (bridge != null) {
+            bridgeConfig.defaultDependencies(
+                deps -> { deps.add(project.getDependencies().project(Map.of("path", ":libs:entitlement:bridge"))); }
+            );
+        }
+        FileCollection bridgeFiles = bridgeConfig;
+
+        project.getTasks().withType(Test.class).configureEach(test -> {
+            // See also SystemJvmOptions.maybeAttachEntitlementAgent.
+
+            // Agent
+            if (agentFiles.isEmpty() == false) {
+                test.getInputs().files(agentFiles);
+                test.systemProperty("es.entitlement.agentJar", agentFiles.getAsPath());
+                test.systemProperty("jdk.attach.allowAttachSelf", true);
+            }
+
+            // Bridge
+            if (bridgeFiles.isEmpty() == false) {
+                String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
+                test.getInputs().files(bridgeFiles);
+                // Tests may not be modular, but the JDK still is
+                test.jvmArgs(
+                    "--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
+                        + modulesContainingEntitlementInstrumentation
                 );
+            }
         });
     }
+
 }

build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java

@@ -188,7 +188,7 @@ private ChangelogEntry makeHighlightsEntry(int pr, boolean notable) {
     }
 
     private String getResource(String name) throws Exception {
-        return Files.readString(Paths.get(Objects.requireNonNull(this.getClass().getResource(name)).toURI()), StandardCharsets.UTF_8);
+        return Files.readString(Paths.get(Objects.requireNonNull(this.getClass().getResource(name)).toURI()), StandardCharsets.UTF_8).replace("\r", "");
     }
 
     private void writeResource(String name, String contents) throws Exception {

build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java

@@ -18,8 +18,11 @@
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.SourceSetContainer;
+import org.gradle.api.tasks.testing.Test;
 import org.gradle.language.jvm.tasks.ProcessResources;
 
+import java.util.List;
+
 import javax.inject.Inject;
 
 /**
@@ -53,5 +56,11 @@ public void apply(Project project) {
         project.getTasks().withType(ProcessResources.class).named("processResources").configure(task -> {
             task.into("META-INF", copy -> copy.from(testBuildInfoTask));
         });
+
+        if (project.getRootProject().getName().equals("elasticsearch")) {
+            project.getTasks().withType(Test.class).matching(test -> List.of("test").contains(test.getName())).configureEach(test -> {
+                test.systemProperty("es.entitlement.enableForTests", "true");
+            });
+        }
     }
 }

libs/build.gradle

@@ -45,4 +45,14 @@ configure(childProjects.values()) {
      */
     apply plugin: 'elasticsearch.build'
   }
+
+  // This is for any code potentially included in the server at runtime.
+  // Omit oddball libraries that aren't in server.
+  def nonServerLibs = ['plugin-scanner']
+  if (false == nonServerLibs.contains(project.name)) {
+    project.getTasks().withType(Test.class).matching(test -> ['test'].contains(test.name)).configureEach(test -> {
+      test.systemProperty('es.entitlement.enableForTests', 'true')
+    })
+  }
+
 }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyCheckerImpl.java

@@ -135,7 +135,7 @@ private void neverEntitled(Class<?> callerClass, Supplier<String> operationDescr
             Strings.format(
                 "component [%s], module [%s], class [%s], operation [%s]",
                 entitlements.componentName(),
-                PolicyCheckerImpl.getModuleName(requestingClass),
+                entitlements.moduleName(),
                 requestingClass,
                 operationDescription.get()
             ),
@@ -247,7 +247,7 @@ public void checkFileRead(Class<?> callerClass, Path path, boolean followLinks)
                 Strings.format(
                     "component [%s], module [%s], class [%s], entitlement [file], operation [read], path [%s]",
                     entitlements.componentName(),
-                    PolicyCheckerImpl.getModuleName(requestingClass),
+                    entitlements.moduleName(),
                     requestingClass,
                     realPath == null ? path : Strings.format("%s -> %s", path, realPath)
                 ),
@@ -279,7 +279,7 @@ public void checkFileWrite(Class<?> callerClass, Path path) {
                 Strings.format(
                     "component [%s], module [%s], class [%s], entitlement [file], operation [write], path [%s]",
                     entitlements.componentName(),
-                    PolicyCheckerImpl.getModuleName(requestingClass),
+                    entitlements.moduleName(),
                     requestingClass,
                     path
                 ),
@@ -383,7 +383,7 @@ public void checkWriteProperty(Class<?> callerClass, String property) {
                     () -> Strings.format(
                         "Entitled: component [%s], module [%s], class [%s], entitlement [write_system_properties], property [%s]",
                         entitlements.componentName(),
-                        PolicyCheckerImpl.getModuleName(requestingClass),
+                        entitlements.moduleName(),
                         requestingClass,
                         property
                     )
@@ -394,7 +394,7 @@ public void checkWriteProperty(Class<?> callerClass, String property) {
             Strings.format(
                 "component [%s], module [%s], class [%s], entitlement [write_system_properties], property [%s]",
                 entitlements.componentName(),
-                PolicyCheckerImpl.getModuleName(requestingClass),
+                entitlements.moduleName(),
                 requestingClass,
                 property
             ),
@@ -447,7 +447,7 @@ private void checkFlagEntitlement(
                 Strings.format(
                     "component [%s], module [%s], class [%s], entitlement [%s]",
                     classEntitlements.componentName(),
-                    PolicyCheckerImpl.getModuleName(requestingClass),
+                    classEntitlements.moduleName(),
                     requestingClass,
                     PolicyParser.buildEntitlementNameFromClass(entitlementClass)
                 ),
@@ -460,7 +460,7 @@ private void checkFlagEntitlement(
                 () -> Strings.format(
                     "Entitled: component [%s], module [%s], class [%s], entitlement [%s]",
                     classEntitlements.componentName(),
-                    PolicyCheckerImpl.getModuleName(requestingClass),
+                    classEntitlements.moduleName(),
                     requestingClass,
                     PolicyParser.buildEntitlementNameFromClass(entitlementClass)
                 )

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java

@@ -118,8 +118,9 @@ public enum ComponentKind {
      *
      * @param componentName the plugin name or else one of the special component names like "(server)".
      */
-    record ModuleEntitlements(
+    protected record ModuleEntitlements(
         String componentName,
+        String moduleName,
         Map<Class<? extends Entitlement>, List<Entitlement>> entitlementsByType,
         FileAccessTree fileAccess,
         Logger logger
@@ -148,7 +149,13 @@ private FileAccessTree getDefaultFileAccess(Collection<Path> componentPaths) {
 
     // pkg private for testing
     ModuleEntitlements defaultEntitlements(String componentName, Collection<Path> componentPaths, String moduleName) {
-        return new ModuleEntitlements(componentName, Map.of(), getDefaultFileAccess(componentPaths), getLogger(componentName, moduleName));
+        return new ModuleEntitlements(
+            componentName,
+            moduleName,
+            Map.of(),
+            getDefaultFileAccess(componentPaths),
+            getLogger(componentName, moduleName)
+        );
     }
 
     // pkg private for testing
@@ -166,6 +173,7 @@ ModuleEntitlements policyEntitlements(
         }
         return new ModuleEntitlements(
             componentName,
+            moduleName,
             entitlements.stream().collect(groupingBy(Entitlement::getClass)),
             FileAccessTree.of(componentName, moduleName, filesEntitlement, pathLookup, componentPaths, exclusivePaths),
             getLogger(componentName, moduleName)
@@ -293,11 +301,11 @@ private static Logger getLogger(String componentName, String moduleName) {
      */
     private static final ConcurrentHashMap<String, Logger> MODULE_LOGGERS = new ConcurrentHashMap<>();
 
-    ModuleEntitlements getEntitlements(Class<?> requestingClass) {
+    protected ModuleEntitlements getEntitlements(Class<?> requestingClass) {
         return moduleEntitlementsMap.computeIfAbsent(requestingClass.getModule(), m -> computeEntitlements(requestingClass));
     }
 
-    private ModuleEntitlements computeEntitlements(Class<?> requestingClass) {
+    protected final ModuleEntitlements computeEntitlements(Class<?> requestingClass) {
         var policyScope = scopeResolver.apply(requestingClass);
         var componentName = policyScope.componentName();
         var moduleName = policyScope.moduleName();
@@ -336,8 +344,7 @@ private ModuleEntitlements computeEntitlements(Class<?> requestingClass) {
         }
     }
 
-    // pkg private for testing
-    static Collection<Path> getComponentPathsFromClass(Class<?> requestingClass) {
+    protected Collection<Path> getComponentPathsFromClass(Class<?> requestingClass) {
         var codeSource = requestingClass.getProtectionDomain().getCodeSource();
         if (codeSource == null) {
             return List.of();

libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java

@@ -89,7 +89,6 @@ public void testGetEntitlements() {
         AtomicReference<PolicyScope> policyScope = new AtomicReference<>();
 
         // A common policy with a variety of entitlements to test
-        Collection<Path> thisSourcePaths = PolicyManager.getComponentPathsFromClass(getClass());
         var plugin1SourcePaths = List.of(Path.of("modules", "plugin1"));
         var policyManager = new PolicyManager(
             new Policy("server", List.of(new Scope("org.example.httpclient", List.of(new OutboundNetworkEntitlement())))),
@@ -99,6 +98,7 @@ public void testGetEntitlements() {
             Map.of("plugin1", plugin1SourcePaths),
             TEST_PATH_LOOKUP
         );
+        Collection<Path> thisSourcePaths = policyManager.getComponentPathsFromClass(getClass());
 
         // "Unspecified" below means that the module is not named in the policy
 

modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ESTestCase.WithoutEntitlements;
 import org.junit.Before;
 
 import java.io.InputStream;
@@ -38,6 +39,7 @@
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
+@WithoutEntitlements // ES-12084
 public class AttachmentProcessorTests extends ESTestCase {
 
     private Processor processor;

modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java

@@ -14,6 +14,7 @@
 import org.apache.tika.metadata.Metadata;
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ESTestCase.WithoutEntitlements;
 
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
@@ -25,6 +26,7 @@
  * comes back and no exception.
  */
 @SuppressFileSystems("ExtrasFS") // don't try to parse extraN
+@WithoutEntitlements // ES-12084
 public class TikaDocTests extends ESTestCase {
 
     /** some test files from tika test suite, zipped up */

modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaImplTests.java

@@ -9,7 +9,9 @@
 package org.elasticsearch.ingest.attachment;
 
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ESTestCase.WithoutEntitlements;
 
+@WithoutEntitlements // ES-12084
 public class TikaImplTests extends ESTestCase {
 
     public void testTikaLoads() throws Exception {