Backport entitlement work up to elastic#116473 to 8.x (elastic#116613)

* Add initial entitlement policy parsing (elastic#114448)

This change adds entitlement policy parsing with the following design:
* YAML file for readability and re-use of our x-content parsers
* hierarchical structure to group entitlements under a single scope
* no general entitlements without a scope or for the entire project

* Avoid double instrumentation via class annotation (elastic#115398)

* Move entitlement jars to libs (elastic#115883)

The distribution tools are meant to be CLIs. This commit moves the
entitlements jar projects to the libs dir, under a single
libs/entitlement root directory to keep the related jars together.

* Entitlement tools: SecurityManager scanner (elastic#116020)

* Dynamic entitlement agent (elastic#116125)

* Refactor: treat "maybe" JVM options uniformly

* WIP

* Get entitlement running with bridge all the way through, with qualified
exports

* Cosmetic changes to SystemJvmOptions

* Disable entitlements by default

* Bridge module comments

* Fixup forbidden APIs

* spotless

* Rename EntitlementChecker

* Fixup InstrumenterTests

* exclude recursive dep

* Fix some compliance stuff

* Rename asm-provider

* Stop using bridge in InstrumenterTests

* Generalize readme for asm-provider

* InstrumenterTests doesn't need EntitlementCheckerHandle

* Better javadoc

* Call parseBoolean

* Add entitlement to internal module list

* Docs as requested by Lorenzo

* Changes from Jack

* Rename ElasticsearchEntitlementChecker

* Remove logging javadoc

* exportInitializationToAgent should reference EntitlementInitialization, not EntitlementBootstrap.

They're currently in the same module, but if that ever changes, this code would have become wrong.

* Some suggestions from Mark

---------

Co-authored-by: Ryan Ernst <[email protected]>

* Remove unused EntitlementInternals (elastic#116473)

* Revert "Entitlement tools: SecurityManager scanner (elastic#116020)"

This reverts commit 023fb66.

---------

Co-authored-by: Jack Conradson <[email protected]>
Co-authored-by: Lorenzo Dematté <[email protected]>
Co-authored-by: Ryan Ernst <[email protected]>

Author: 4 people

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java

@@ -48,10 +48,11 @@ public class InternalDistributionModuleCheckTaskProvider {
     /** ES jars in the lib directory that are not modularized. For now, es-log4j is the only one. */
     private static final List<String> ES_JAR_EXCLUDES = List.of("elasticsearch-log4j");
 
-    /** List of the current Elasticsearch Java Modules, by name. */
+    /** List of the current Elasticsearch Java Modules, alphabetically by name. */
     private static final List<String> EXPECTED_ES_SERVER_MODULES = List.of(
         "org.elasticsearch.base",
         "org.elasticsearch.cli",
+        "org.elasticsearch.entitlement",
         "org.elasticsearch.geo",
         "org.elasticsearch.grok",
         "org.elasticsearch.logging",

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java

@@ -42,6 +42,7 @@ public abstract class RunTask extends DefaultTestClustersTask {
 
     private Boolean debug = false;
     private Boolean cliDebug = false;
+    private Boolean entitlementsEnabled = false;
     private Boolean apmServerEnabled = false;
 
     private Boolean preserveData = false;
@@ -69,6 +70,14 @@ public void setCliDebug(boolean enabled) {
         this.cliDebug = enabled;
     }
 
+    @Option(
+        option = "entitlements",
+        description = "Use the Entitlements agent system in place of SecurityManager to enforce sandbox policies."
+    )
+    public void setEntitlementsEnabled(boolean enabled) {
+        this.entitlementsEnabled = enabled;
+    }
+
     @Input
     public Boolean getDebug() {
         return debug;
@@ -79,6 +88,11 @@ public Boolean getCliDebug() {
         return cliDebug;
     }
 
+    @Input
+    public Boolean getEntitlementsEnabled() {
+        return entitlementsEnabled;
+    }
+
     @Input
     public Boolean getApmServerEnabled() {
         return apmServerEnabled;
@@ -226,6 +240,9 @@ else if (node.getSettingKeys().contains("telemetry.metrics.enabled") == false) {
         if (cliDebug) {
             enableCliDebug();
         }
+        if (entitlementsEnabled) {
+            enableEntitlements();
+        }
     }
 
     @TaskAction

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java

@@ -74,4 +74,12 @@ default void enableCliDebug() {
             }
         }
     }
+
+    default void enableEntitlements() {
+        for (ElasticsearchCluster cluster : getClusters()) {
+            for (ElasticsearchNode node : cluster.getNodes()) {
+                node.cliJvmArgs("-Des.entitlements.enabled=true");
+            }
+        }
+    }
 }

distribution/build.gradle

@@ -262,7 +262,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
    *             Properties to expand when copying packaging files             *
    *****************************************************************************/
   configurations {
-    ['libs', 'libsVersionChecker', 'libsCliLauncher', 'libsServerCli', 'libsWindowsServiceCli', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli', 'libsGeoIpCli', 'libsAnsiConsole', 'libsNative'].each {
+    ['libs', 'libsVersionChecker', 'libsCliLauncher', 'libsServerCli', 'libsWindowsServiceCli', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli', 'libsGeoIpCli', 'libsAnsiConsole', 'libsNative', 'libsEntitlementAgent', 'libsEntitlementBridge'].each {
       create(it) {
         canBeConsumed = false
         canBeResolved = true
@@ -292,6 +292,8 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
     libsSecurityCli project(':x-pack:plugin:security:cli')
     libsGeoIpCli project(':distribution:tools:geoip-cli')
     libsNative project(':libs:native:native-libraries')
+    libsEntitlementAgent project(':libs:entitlement:agent')
+    libsEntitlementBridge project(':libs:entitlement:bridge')
   }
 
   project.ext {
@@ -336,6 +338,12 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
             include (os + '-' + architecture + '/*')
           }
         }
+        into('entitlement-agent') {
+          from(configurations.libsEntitlementAgent)
+        }
+        into('entitlement-bridge') {
+          from(configurations.libsEntitlementBridge)
+        }
       }
     }