Dynamic entitlement agent (#116125)

* Refactor: treat "maybe" JVM options uniformly

* WIP

* Get entitlement running with bridge all the way through, with qualified
exports

* Cosmetic changes to SystemJvmOptions

* Disable entitlements by default

* Bridge module comments

* Fixup forbidden APIs

* spotless

* Rename EntitlementChecker

* Fixup InstrumenterTests

* exclude recursive dep

* Fix some compliance stuff

* Rename asm-provider

* Stop using bridge in InstrumenterTests

* Generalize readme for asm-provider

* InstrumenterTests doesn't need EntitlementCheckerHandle

* Better javadoc

* Call parseBoolean

* Add entitlement to internal module list

* Docs as requested by Lorenzo

* Changes from Jack

* Rename ElasticsearchEntitlementChecker

* Remove logging javadoc

* exportInitializationToAgent should reference EntitlementInitialization, not EntitlementBootstrap.

They're currently in the same module, but if that ever changes, this code would have become wrong.

* Some suggestions from Mark

---------

Co-authored-by: Ryan Ernst <[email protected]>

Author: prdoyle

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java

@@ -48,10 +48,11 @@ public class InternalDistributionModuleCheckTaskProvider {
     /** ES jars in the lib directory that are not modularized. For now, es-log4j is the only one. */
     private static final List<String> ES_JAR_EXCLUDES = List.of("elasticsearch-log4j");
 
-    /** List of the current Elasticsearch Java Modules, by name. */
+    /** List of the current Elasticsearch Java Modules, alphabetically by name. */
     private static final List<String> EXPECTED_ES_SERVER_MODULES = List.of(
         "org.elasticsearch.base",
         "org.elasticsearch.cli",
+        "org.elasticsearch.entitlement",
         "org.elasticsearch.geo",
         "org.elasticsearch.grok",
         "org.elasticsearch.logging",

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java

@@ -42,6 +42,7 @@ public abstract class RunTask extends DefaultTestClustersTask {
 
     private Boolean debug = false;
     private Boolean cliDebug = false;
+    private Boolean entitlementsEnabled = false;
     private Boolean apmServerEnabled = false;
 
     private Boolean preserveData = false;
@@ -69,6 +70,14 @@ public void setCliDebug(boolean enabled) {
         this.cliDebug = enabled;
     }
 
+    @Option(
+        option = "entitlements",
+        description = "Use the Entitlements agent system in place of SecurityManager to enforce sandbox policies."
+    )
+    public void setEntitlementsEnabled(boolean enabled) {
+        this.entitlementsEnabled = enabled;
+    }
+
     @Input
     public Boolean getDebug() {
         return debug;
@@ -79,6 +88,11 @@ public Boolean getCliDebug() {
         return cliDebug;
     }
 
+    @Input
+    public Boolean getEntitlementsEnabled() {
+        return entitlementsEnabled;
+    }
+
     @Input
     public Boolean getApmServerEnabled() {
         return apmServerEnabled;
@@ -226,6 +240,9 @@ else if (node.getSettingKeys().contains("telemetry.metrics.enabled") == false) {
         if (cliDebug) {
             enableCliDebug();
         }
+        if (entitlementsEnabled) {
+            enableEntitlements();
+        }
     }
 
     @TaskAction

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java

@@ -74,4 +74,12 @@ default void enableCliDebug() {
             }
         }
     }
+
+    default void enableEntitlements() {
+        for (ElasticsearchCluster cluster : getClusters()) {
+            for (ElasticsearchNode node : cluster.getNodes()) {
+                node.cliJvmArgs("-Des.entitlements.enabled=true");
+            }
+        }
+    }
 }

distribution/build.gradle

@@ -262,7 +262,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
    *             Properties to expand when copying packaging files             *
    *****************************************************************************/
   configurations {
-    ['libs', 'libsVersionChecker', 'libsCliLauncher', 'libsServerCli', 'libsWindowsServiceCli', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli', 'libsGeoIpCli', 'libsAnsiConsole', 'libsNative'].each {
+    ['libs', 'libsVersionChecker', 'libsCliLauncher', 'libsServerCli', 'libsWindowsServiceCli', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli', 'libsGeoIpCli', 'libsAnsiConsole', 'libsNative', 'libsEntitlementAgent', 'libsEntitlementBridge'].each {
       create(it) {
         canBeConsumed = false
         canBeResolved = true
@@ -292,6 +292,8 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
     libsSecurityCli project(':x-pack:plugin:security:cli')
     libsGeoIpCli project(':distribution:tools:geoip-cli')
     libsNative project(':libs:native:native-libraries')
+    libsEntitlementAgent project(':libs:entitlement:agent')
+    libsEntitlementBridge project(':libs:entitlement:bridge')
   }
 
   project.ext {
@@ -336,6 +338,12 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
             include (os + '-' + architecture + '/*')
           }
         }
+        into('entitlement-agent') {
+          from(configurations.libsEntitlementAgent)
+        }
+        into('entitlement-bridge') {
+          from(configurations.libsEntitlementBridge)
+        }
       }
     }
 

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -12,18 +12,20 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 final class SystemJvmOptions {
 
     static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, String> sysprops) {
         String distroType = sysprops.get("es.distribution.type");
         boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot");
-
-        return Stream.concat(
+        boolean useEntitlements = Boolean.parseBoolean(sysprops.getOrDefault("es.entitlements.enabled", "false"));
+        return Stream.of(
             Stream.of(
                 /*
                  * Cache ttl in seconds for positive DNS lookups noting that this overrides the JDK security property
@@ -35,8 +37,6 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                  * networkaddress.cache.negative ttl; set to -1 to cache forever.
                  */
                 "-Des.networkaddress.cache.negative.ttl=10",
-                // Allow to set the security manager.
-                "-Djava.security.manager=allow",
                 // pre-touch JVM emory pages during initialization
                 "-XX:+AlwaysPreTouch",
                 // explicitly set the stack size
@@ -61,15 +61,17 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                 "-Dlog4j2.disable.jmx=true",
                 "-Dlog4j2.formatMsgNoLookups=true",
                 "-Djava.locale.providers=CLDR",
-                maybeEnableNativeAccess(),
-                maybeOverrideDockerCgroup(distroType),
-                maybeSetActiveProcessorCount(nodeSettings),
-                setReplayFile(distroType, isHotspot),
                 // Pass through distribution type
                 "-Des.distribution.type=" + distroType
             ),
-            maybeWorkaroundG1Bug()
-        ).filter(e -> e.isEmpty() == false).collect(Collectors.toList());
+            maybeEnableNativeAccess(),
+            maybeOverrideDockerCgroup(distroType),
+            maybeSetActiveProcessorCount(nodeSettings),
+            maybeSetReplayFile(distroType, isHotspot),
+            maybeWorkaroundG1Bug(),
+            maybeAllowSecurityManager(),
+            maybeAttachEntitlementAgent(useEntitlements)
+        ).flatMap(s -> s).toList();
     }
 
     /*
@@ -86,42 +88,42 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
      * that cgroup statistics are available for the container this process
      * will run in.
      */
-    private static String maybeOverrideDockerCgroup(String distroType) {
+    private static Stream<String> maybeOverrideDockerCgroup(String distroType) {
         if ("docker".equals(distroType)) {
-            return "-Des.cgroups.hierarchy.override=/";
+            return Stream.of("-Des.cgroups.hierarchy.override=/");
         }
-        return "";
+        return Stream.empty();
     }
 
-    private static String setReplayFile(String distroType, boolean isHotspot) {
+    private static Stream<String> maybeSetReplayFile(String distroType, boolean isHotspot) {
         if (isHotspot == false) {
             // the replay file option is only guaranteed for hotspot vms
-            return "";
+            return Stream.empty();
         }
         String replayDir = "logs";
         if ("rpm".equals(distroType) || "deb".equals(distroType)) {
             replayDir = "/var/log/elasticsearch";
         }
-        return "-XX:ReplayDataFile=" + replayDir + "/replay_pid%p.log";
+        return Stream.of("-XX:ReplayDataFile=" + replayDir + "/replay_pid%p.log");
     }
 
     /*
      * node.processors determines thread pool sizes for Elasticsearch. When it
      * is set, we need to also tell the JVM to respect a different value
      */
-    private static String maybeSetActiveProcessorCount(Settings nodeSettings) {
+    private static Stream<String> maybeSetActiveProcessorCount(Settings nodeSettings) {
         if (EsExecutors.NODE_PROCESSORS_SETTING.exists(nodeSettings)) {
             int allocated = EsExecutors.allocatedProcessors(nodeSettings);
-            return "-XX:ActiveProcessorCount=" + allocated;
+            return Stream.of("-XX:ActiveProcessorCount=" + allocated);
         }
-        return "";
+        return Stream.empty();
     }
 
-    private static String maybeEnableNativeAccess() {
+    private static Stream<String> maybeEnableNativeAccess() {
         if (Runtime.version().feature() >= 21) {
-            return "--enable-native-access=org.elasticsearch.nativeaccess,org.apache.lucene.core";
+            return Stream.of("--enable-native-access=org.elasticsearch.nativeaccess,org.apache.lucene.core");
         }
-        return "";
+        return Stream.empty();
     }
 
     /*
@@ -134,4 +136,37 @@ private static Stream<String> maybeWorkaroundG1Bug() {
         }
         return Stream.of();
     }
+
+    private static Stream<String> maybeAllowSecurityManager() {
+        // Will become conditional on useEntitlements once entitlements can run without SM
+        return Stream.of("-Djava.security.manager=allow");
+    }
+
+    private static Stream<String> maybeAttachEntitlementAgent(boolean useEntitlements) {
+        if (useEntitlements == false) {
+            return Stream.empty();
+        }
+
+        Path dir = Path.of("lib", "entitlement-bridge");
+        if (Files.exists(dir) == false) {
+            throw new IllegalStateException("Directory for entitlement bridge jar does not exist: " + dir);
+        }
+        String bridgeJar;
+        try (var s = Files.list(dir)) {
+            var candidates = s.limit(2).toList();
+            if (candidates.size() != 1) {
+                throw new IllegalStateException("Expected one jar in " + dir + "; found " + candidates.size());
+            }
+            bridgeJar = candidates.get(0).toString();
+        } catch (IOException e) {
+            throw new IllegalStateException("Failed to list entitlement jars in: " + dir, e);
+        }
+        return Stream.of(
+            "-Des.entitlements.enabled=true",
+            "-XX:+EnableDynamicAgentLoading",
+            "-Djdk.attach.allowAttachSelf=true",
+            "--patch-module=java.base=" + bridgeJar,
+            "--add-exports=java.base/org.elasticsearch.entitlement.bridge=org.elasticsearch.entitlement"
+        );
+    }
 }

libs/core/src/main/java/module-info.java

@@ -19,7 +19,7 @@
         to
             org.elasticsearch.xcontent,
             org.elasticsearch.nativeaccess,
-            org.elasticsearch.entitlement.agent;
+            org.elasticsearch.entitlement;
 
     uses ModuleQualifiedExportsService;
 }

libs/entitlement/README.md

@@ -1,7 +1,7 @@
-### Entitlement runtime
+### Entitlement library
 
 This module implements mechanisms to grant and check permissions under the _entitlements_ system.
 
 The entitlements system provides an alternative to the legacy `SecurityManager` system, which is deprecated for removal.
-The `entitlement-agent` tool instruments sensitive class library methods with calls to this module, in order to enforce the controls.
+The `entitlement-agent` instruments sensitive class library methods with calls to this module, in order to enforce the controls.
 

libs/entitlement/agent/README.md

@@ -5,6 +5,6 @@ This is a java agent that instruments sensitive class library methods with calls
 The entitlements system provides an alternative to the legacy `SecurityManager` system, which is deprecated for removal.
 With this agent, the Elasticsearch server can retain some control over which class library methods can be invoked by which callers.
 
-This module is responsible for inserting the appropriate bytecode to achieve enforcement of the rules governed by the `entitlement-runtime` module.
+This module is responsible for inserting the appropriate bytecode to achieve enforcement of the rules governed by the main `entitlement` module.
 
-It is not responsible for permission granting or checking logic. That responsibility lies with `entitlement-runtime`.
+It is not responsible for permission granting or checking logic. That responsibility lies with the main `entitlement` module.

libs/entitlement/agent/build.gradle

@@ -6,51 +6,18 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
-
-import static java.util.stream.Collectors.joining
-
 apply plugin: 'elasticsearch.build'
-apply plugin: 'elasticsearch.embedded-providers'
-
-embeddedProviders {
-  impl 'entitlement-agent', project(':libs:entitlement:agent:impl')
-}
-
-configurations {
-  entitlementBridge
-}
 
 dependencies {
-  entitlementBridge project(":libs:entitlement:bridge")
   compileOnly project(":libs:core")
   compileOnly project(":libs:entitlement")
-  testImplementation project(":test:framework")
-  testImplementation project(":libs:entitlement:bridge")
-  testImplementation project(":libs:entitlement:agent:impl")
-}
-
-tasks.named('test').configure {
-  systemProperty "tests.security.manager", "false"
-  dependsOn('jar')
-
-  // Register an argument provider to avoid eager resolution of configurations
-  jvmArgumentProviders.add(new CommandLineArgumentProvider() {
-    @Override
-    Iterable<String> asArguments() {
-      return ["-javaagent:${tasks.jar.archiveFile.get()}", "-Des.entitlements.bridgeJar=${configurations.entitlementBridge.singleFile}"]
-    }
-  })
-
-
-  // The Elasticsearch build plugin automatically adds all compileOnly deps as testImplementation.
-  // We must not add the bridge this way because it is also on the boot classpath, and that would lead to jar hell.
-  classpath -= files(configurations.entitlementBridge)
+  compileOnly project(":libs:entitlement:bridge")
 }
 
 tasks.named('jar').configure {
   manifest {
     attributes(
-      'Premain-Class': 'org.elasticsearch.entitlement.agent.EntitlementAgent'
+      'Agent-Class': 'org.elasticsearch.entitlement.agent.EntitlementAgent'
       , 'Can-Retransform-Classes': 'true'
     )
   }