[Entitlements] Allow policy overrides via system properties

libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAllowedViaOverrideIT.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.qa;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.core.Strings;
+import org.junit.ClassRule;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.util.Base64;
+import java.util.function.BiConsumer;
+import java.util.function.Function;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.entitlement.qa.EntitlementsTestRule.ENTITLEMENT_QA_TEST_MODULE_NAME;
+import static org.elasticsearch.entitlement.qa.EntitlementsTestRule.ENTITLEMENT_TEST_PLUGIN_NAME;
+
+public class EntitlementsAllowedViaOverrideIT extends AbstractEntitlementsIT {
+
+    private static void addPolicyOverrideSystemProperties(BiConsumer<String, Function<Path, String>> adder) {
+        adder.accept("es.entitlements.policy." + ENTITLEMENT_TEST_PLUGIN_NAME, tempDir -> {
+            String policyOverride = Strings.format("""
+                policy:
+                  %s:
+                    - load_native_libraries
+                    - files:
+                        - path: %s
+                          mode: read
+                """, ENTITLEMENT_QA_TEST_MODULE_NAME, tempDir.resolve("read_dir"));
+            return new String(Base64.getEncoder().encode(policyOverride.getBytes(StandardCharsets.UTF_8)));
+        });
+    }
+
+    @ClassRule
+    public static EntitlementsTestRule testRule = new EntitlementsTestRule(
+        true,
+        null,
+        EntitlementsAllowedViaOverrideIT::addPolicyOverrideSystemProperties
+    );
+
+    public EntitlementsAllowedViaOverrideIT(@Name("actionName") String actionName) {
+        super(actionName, true);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> data() {
+        return Stream.of("runtime_load_library", "fileList").map(action -> new Object[] { action }).toList();
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        return testRule.cluster.getHttpAddresses();
+    }
+}

libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsTestRule.java

@@ -28,6 +28,8 @@
 import java.nio.file.Path;
 import java.util.List;
 import java.util.Map;
+import java.util.function.BiConsumer;
+import java.util.function.Function;
 
 class EntitlementsTestRule implements TestRule {
 
@@ -48,17 +50,27 @@ class EntitlementsTestRule implements TestRule {
             )
         );
     };
+    public static final String ENTITLEMENT_QA_TEST_MODULE_NAME = "org.elasticsearch.entitlement.qa.test";
+    public static final String ENTITLEMENT_TEST_PLUGIN_NAME = "entitlement-test-plugin";
 
     interface PolicyBuilder {
         void build(XContentBuilder builder, Path tempDir) throws IOException;
     }
 
+    interface SystemPropertyBuilder {
+        void build(BiConsumer<String, Function<Path, String>> adder);
+    }
+
     final TemporaryFolder testDir;
     final ElasticsearchCluster cluster;
     final TestRule ruleChain;
 
-    @SuppressWarnings("this-escape")
     EntitlementsTestRule(boolean modular, PolicyBuilder policyBuilder) {
+        this(modular, policyBuilder, (adder) -> {});
+    }
+
+    @SuppressWarnings("this-escape")
+    EntitlementsTestRule(boolean modular, PolicyBuilder policyBuilder, SystemPropertyBuilder systemPropertyBuilder) {
         testDir = new TemporaryFolder();
         var tempDirSetup = new ExternalResource() {
             @Override
@@ -70,15 +82,16 @@ protected void before() throws Throwable {
                 Files.writeString(testPath.resolve("read_write_file"), "");
             }
         };
-        cluster = ElasticsearchCluster.local()
+        var clusterBuilder = ElasticsearchCluster.local()
             .module("entitled", spec -> buildEntitlements(spec, "org.elasticsearch.entitlement.qa.entitled", ENTITLED_POLICY))
-            .module("entitlement-test-plugin", spec -> setupEntitlements(spec, modular, policyBuilder))
+            .module(ENTITLEMENT_TEST_PLUGIN_NAME, spec -> setupEntitlements(spec, modular, policyBuilder))
             .systemProperty("es.entitlements.enabled", "true")
             .systemProperty("es.entitlements.testdir", () -> testDir.getRoot().getAbsolutePath())
-            .setting("xpack.security.enabled", "false")
-            // Logs in libs/entitlement/qa/build/test-results/javaRestTest/TEST-org.elasticsearch.entitlement.qa.EntitlementsXXX.xml
-            // .setting("logger.org.elasticsearch.entitlement", "DEBUG")
-            .build();
+            .setting("xpack.security.enabled", "false");
+        // Logs in libs/entitlement/qa/build/test-results/javaRestTest/TEST-org.elasticsearch.entitlement.qa.EntitlementsXXX.xml
+        // .setting("logger.org.elasticsearch.entitlement", "DEBUG")
+        systemPropertyBuilder.build((k, v) -> clusterBuilder.systemProperty(k, () -> v.apply(testDir.getRoot().toPath())));
+        cluster = clusterBuilder.build();
         ruleChain = RuleChain.outerRule(testDir).around(tempDirSetup).around(cluster);
     }
 
@@ -108,14 +121,14 @@ private void buildEntitlements(PluginInstallSpec spec, String moduleName, Policy
     }
 
     private void setupEntitlements(PluginInstallSpec spec, boolean modular, PolicyBuilder policyBuilder) {
-        String moduleName = modular ? "org.elasticsearch.entitlement.qa.test" : "ALL-UNNAMED";
+        String moduleName = modular ? ENTITLEMENT_QA_TEST_MODULE_NAME : "ALL-UNNAMED";
         if (policyBuilder != null) {
             buildEntitlements(spec, moduleName, policyBuilder);
         }
 
         if (modular == false) {
             spec.withPropertiesOverride(old -> {
-                String props = old.replace("modulename=org.elasticsearch.entitlement.qa.test", "");
+                String props = old.replace("modulename=" + ENTITLEMENT_QA_TEST_MODULE_NAME, "");
                 System.out.println("Using plugin properties:\n" + props);
                 return Resource.fromString(props);
             });

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java

@@ -33,10 +33,12 @@
 import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
@@ -97,6 +99,58 @@ public PolicyParser(InputStream inputStream, String policyName, boolean isExtern
         this.externalEntitlements = externalEntitlements;
     }
 
+    public VersionedPolicy parseVersionedPolicy() {
+        Set<String> versions = Set.of();
+        Policy policy = emptyPolicy();
+        try {
+            if (policyParser.nextToken() != XContentParser.Token.START_OBJECT) {
+                throw newPolicyParserException("expected object <versioned policy>");
+            }
+
+            while (policyParser.nextToken() != XContentParser.Token.END_OBJECT) {
+                if (policyParser.currentToken() == XContentParser.Token.FIELD_NAME) {
+                    if (policyParser.currentName().equals("versions")) {
+                        versions = parseVersions();
+                    } else if (policyParser.currentName().equals("policy")) {
+                        policy = parsePolicy();
+                    } else {
+                        throw newPolicyParserException("expected either <version> or <policy> field");
+                    }
+                } else {
+                    throw newPolicyParserException("expected either <version> or <policy> field");
+                }
+            }
+
+            return new VersionedPolicy(policy, versions);
+        } catch (IOException ioe) {
+            throw new UncheckedIOException(ioe);
+        }
+    }
+
+    private Policy emptyPolicy() {
+        return new Policy(policyName, List.of());
+    }
+
+    private Set<String> parseVersions() throws IOException {
+        try {
+            if (policyParser.nextToken() != XContentParser.Token.START_ARRAY) {
+                throw newPolicyParserException("expected array of <versions>");
+            }
+            Set<String> versions = new HashSet<>();
+            while (policyParser.nextToken() != XContentParser.Token.END_ARRAY) {
+                if (policyParser.currentToken() == XContentParser.Token.VALUE_STRING) {
+                    String version = policyParser.text();
+                    versions.add(version);
+                } else {
+                    throw newPolicyParserException("expected <version>");
+                }
+            }
+            return versions;
+        } catch (IOException ioe) {
+            throw new UncheckedIOException(ioe);
+        }
+    }
+
     public Policy parsePolicy() {
         try {
             if (policyParser.nextToken() != XContentParser.Token.START_OBJECT) {

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserUtils.java

@@ -10,17 +10,22 @@
 package org.elasticsearch.entitlement.runtime.policy;
 
 import org.elasticsearch.core.Strings;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
 
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.lang.module.ModuleFinder;
 import java.lang.module.ModuleReference;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
+import java.util.Base64;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -29,6 +34,8 @@
 
 public class PolicyParserUtils {
 
+    private static final Logger logger = LogManager.getLogger(PolicyParserUtils.class);
+
     public record PluginData(Path pluginPath, boolean isModular, boolean isExternalPlugin) {
         public PluginData {
             requireNonNull(pluginPath);
@@ -37,41 +44,89 @@ public record PluginData(Path pluginPath, boolean isModular, boolean isExternalP
 
     private static final String POLICY_FILE_NAME = "entitlement-policy.yaml";
 
-    public static Map<String, Policy> createPluginPolicies(Collection<PluginData> pluginData) throws IOException {
+    public static final String POLICY_OVERRIDE_PREFIX = "es.entitlements.policy.";
+
+    public static Map<String, Policy> createPluginPolicies(Collection<PluginData> pluginData, Map<String, String> overrides, String version)
+        throws IOException {
         Map<String, Policy> pluginPolicies = new HashMap<>(pluginData.size());
         for (var entry : pluginData) {
             Path pluginRoot = entry.pluginPath();
             String pluginName = pluginRoot.getFileName().toString();
-
-            final Policy policy = loadPluginPolicy(pluginRoot, entry.isModular(), pluginName, entry.isExternalPlugin());
-
-            pluginPolicies.put(pluginName, policy);
+            final Set<String> moduleNames = getModuleNames(pluginRoot, entry.isModular());
+
+            var overriddenPolicy = parsePolicyOverrideIfExists(overrides, version, entry.isExternalPlugin(), pluginName, moduleNames);
+            if (overriddenPolicy.isPresent()) {
+                pluginPolicies.put(pluginName, overriddenPolicy.get());
+            } else {
+                Path policyFile = pluginRoot.resolve(POLICY_FILE_NAME);
+                var policy = parsePolicyIfExists(pluginName, policyFile, entry.isExternalPlugin());
+                validatePolicyScopes(pluginName, policy, moduleNames, policyFile.toString());
+                pluginPolicies.put(pluginName, policy);
+            }
         }
         return pluginPolicies;
     }
 
-    private static Policy loadPluginPolicy(Path pluginRoot, boolean isModular, String pluginName, boolean isExternalPlugin)
-        throws IOException {
-        Path policyFile = pluginRoot.resolve(POLICY_FILE_NAME);
+    static Optional<Policy> parsePolicyOverrideIfExists(
+        Map<String, String> overrides,
+        String version,
+        boolean externalPlugin,
+        String pluginName,
+        Set<String> moduleNames
+    ) {
+        var policyOverride = overrides.get(pluginName);
+        if (policyOverride != null) {
+            try {
+                var versionedPolicy = decodeOverriddenPluginPolicy(policyOverride, pluginName, externalPlugin);
+                validatePolicyScopes(pluginName, versionedPolicy.policy(), moduleNames, "<override>");
+
+                // Empty versions defaults to "any"
+                if (versionedPolicy.versions().isEmpty() || versionedPolicy.versions().contains(version)) {
+                    logger.info("Using policy override for plugin [{}]", pluginName);
+                    return Optional.of(versionedPolicy.policy());
+                } else {
+                    logger.warn(
+                        "Found a policy override with version mismatch. The override will not be applied. "
+                            + "Plugin [{}]; policy versions [{}]; current version [{}]",
+                        pluginName,
+                        String.join(",", versionedPolicy.versions()),
+                        version
+                    );
+                }
+            } catch (Exception ex) {
+                logger.warn(
+                    Strings.format(
+                        "Found a policy override with invalid content. The override will not be applied. Plugin [%s]",
+                        pluginName
+                    ),
+                    ex
+                );
+            }
+        }
+        return Optional.empty();
+    }
 
-        final Set<String> moduleNames = getModuleNames(pluginRoot, isModular);
-        final Policy policy = parsePolicyIfExists(pluginName, policyFile, isExternalPlugin);
+    static VersionedPolicy decodeOverriddenPluginPolicy(String base64String, String pluginName, boolean isExternalPlugin)
+        throws IOException {
+        byte[] policyDefinition = Base64.getDecoder().decode(base64String);
+        return new PolicyParser(new ByteArrayInputStream(policyDefinition), pluginName, isExternalPlugin).parseVersionedPolicy();
+    }
 
+    private static void validatePolicyScopes(String pluginName, Policy policy, Set<String> moduleNames, String policyLocation) {
         // TODO: should this check actually be part of the parser?
         for (Scope scope : policy.scopes()) {
             if (moduleNames.contains(scope.moduleName()) == false) {
                 throw new IllegalStateException(
                     Strings.format(
-                        "Invalid module name in policy: plugin [%s] does not have module [%s]; available modules [%s]; policy file [%s]",
+                        "Invalid module name in policy: plugin [%s] does not have module [%s]; available modules [%s]; policy path [%s]",
                         pluginName,
                         scope.moduleName(),
                         String.join(", ", moduleNames),
-                        policyFile
+                        policyLocation
                     )
                 );
             }
         }
-        return policy;
     }
 
     private static Policy parsePolicyIfExists(String pluginName, Path policyFile, boolean isExternalPlugin) throws IOException {

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/VersionedPolicy.java

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.runtime.policy;
+
+import java.util.Set;
+
+/**
+ * A Policy and associated versions to which the policy applies
+ */
+public record VersionedPolicy(Policy policy, Set<String> versions) {}