Merge branch 'main' of github.com:elastic/elasticsearch into esql-categorize-multiple-groupings

jan-elastic · jan-elastic · commit 3bccc78b4aa8 · 2024-12-12T15:45:31.000+01:00
diff --git a/libs/entitlement/qa/build.gradle b/libs/entitlement/qa/build.gradle
@@ -13,8 +13,8 @@ apply plugin: 'elasticsearch.internal-test-artifact'
 
 dependencies {
   javaRestTestImplementation project(':libs:entitlement:qa:common')
-  clusterPlugins project(':libs:entitlement:qa:entitlement-allowed')
-  clusterPlugins project(':libs:entitlement:qa:entitlement-allowed-nonmodular')
+  clusterModules project(':libs:entitlement:qa:entitlement-allowed')
+  clusterModules project(':libs:entitlement:qa:entitlement-allowed-nonmodular')
   clusterPlugins project(':libs:entitlement:qa:entitlement-denied')
   clusterPlugins project(':libs:entitlement:qa:entitlement-denied-nonmodular')
 }
diff --git a/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAllowedIT.java b/libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAllowedIT.java
@@ -28,8 +28,8 @@ public class EntitlementsAllowedIT extends ESRestTestCase {
 
     @ClassRule
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
-        .plugin("entitlement-allowed")
-        .plugin("entitlement-allowed-nonmodular")
+        .module("entitlement-allowed")
+        .module("entitlement-allowed-nonmodular")
         .systemProperty("es.entitlements.enabled", "true")
         .setting("xpack.security.enabled", "false")
         .build();
diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java
@@ -15,7 +15,6 @@
 import com.sun.tools.attach.VirtualMachine;
 
 import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.entitlement.initialization.EntitlementInitialization;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
@@ -29,7 +28,9 @@
 
 public class EntitlementBootstrap {
 
-    public record BootstrapArgs(Collection<Tuple<Path, Boolean>> pluginData, Function<Class<?>, String> pluginResolver) {}
+    public record PluginData(Path pluginPath, boolean isModular, boolean isExternalPlugin) {}
+
+    public record BootstrapArgs(Collection<PluginData> pluginData, Function<Class<?>, String> pluginResolver) {}
 
     private static BootstrapArgs bootstrapArgs;
 
@@ -40,11 +41,11 @@ public static BootstrapArgs bootstrapArgs() {
     /**
      * Activates entitlement checking. Once this method returns, calls to methods protected by Entitlements from classes without a valid
      * policy will throw {@link org.elasticsearch.entitlement.runtime.api.NotEntitledException}.
-     * @param pluginData a collection of (plugin path, boolean), that holds the paths of all the installed Elasticsearch modules and
-     *                   plugins, and whether they are Java modular or not.
+     * @param pluginData a collection of (plugin path, boolean, boolean), that holds the paths of all the installed Elasticsearch modules
+     *                   and plugins, whether they are Java modular or not, and whether they are Elasticsearch modules or external plugins.
      * @param pluginResolver a functor to map a Java Class to the plugin it belongs to (the plugin name).
      */
-    public static void bootstrap(Collection<Tuple<Path, Boolean>> pluginData, Function<Class<?>, String> pluginResolver) {
+    public static void bootstrap(Collection<PluginData> pluginData, Function<Class<?>, String> pluginResolver) {
         logger.debug("Loading entitlement agent");
         if (EntitlementBootstrap.bootstrapArgs != null) {
             throw new IllegalStateException("plugin data is already set");
diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java
@@ -9,7 +9,6 @@
 
 package org.elasticsearch.entitlement.initialization;
 
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.core.internal.provider.ProviderLocator;
 import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap;
 import org.elasticsearch.entitlement.bridge.EntitlementChecker;
@@ -96,25 +95,25 @@ private static PolicyManager createPolicyManager() throws IOException {
         return new PolicyManager(serverPolicy, pluginPolicies, EntitlementBootstrap.bootstrapArgs().pluginResolver());
     }
 
-    private static Map<String, Policy> createPluginPolicies(Collection<Tuple<Path, Boolean>> pluginData) throws IOException {
+    private static Map<String, Policy> createPluginPolicies(Collection<EntitlementBootstrap.PluginData> pluginData) throws IOException {
         Map<String, Policy> pluginPolicies = new HashMap<>(pluginData.size());
-        for (Tuple<Path, Boolean> entry : pluginData) {
-            Path pluginRoot = entry.v1();
-            boolean isModular = entry.v2();
-
+        for (var entry : pluginData) {
+            Path pluginRoot = entry.pluginPath();
             String pluginName = pluginRoot.getFileName().toString();
-            final Policy policy = loadPluginPolicy(pluginRoot, isModular, pluginName);
+
+            final Policy policy = loadPluginPolicy(pluginRoot, entry.isModular(), pluginName, entry.isExternalPlugin());
 
             pluginPolicies.put(pluginName, policy);
         }
         return pluginPolicies;
     }
 
-    private static Policy loadPluginPolicy(Path pluginRoot, boolean isModular, String pluginName) throws IOException {
+    private static Policy loadPluginPolicy(Path pluginRoot, boolean isModular, String pluginName, boolean isExternalPlugin)
+        throws IOException {
         Path policyFile = pluginRoot.resolve(POLICY_FILE_NAME);
 
         final Set<String> moduleNames = getModuleNames(pluginRoot, isModular);
-        final Policy policy = parsePolicyIfExists(pluginName, policyFile);
+        final Policy policy = parsePolicyIfExists(pluginName, policyFile, isExternalPlugin);
 
         // TODO: should this check actually be part of the parser?
         for (Scope scope : policy.scopes) {
@@ -125,9 +124,9 @@ private static Policy loadPluginPolicy(Path pluginRoot, boolean isModular, Strin
         return policy;
     }
 
-    private static Policy parsePolicyIfExists(String pluginName, Path policyFile) throws IOException {
+    private static Policy parsePolicyIfExists(String pluginName, Path policyFile, boolean isExternalPlugin) throws IOException {
         if (Files.exists(policyFile)) {
-            return new PolicyParser(Files.newInputStream(policyFile, StandardOpenOption.READ), pluginName).parsePolicy();
+            return new PolicyParser(Files.newInputStream(policyFile, StandardOpenOption.READ), pluginName, isExternalPlugin).parsePolicy();
         }
         return new Policy(pluginName, List.of());
     }
diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java
@@ -33,4 +33,12 @@
      * have to match the parameter names of the constructor.
      */
     String[] parameterNames() default {};
+
+    /**
+     * This flag indicates if this Entitlement can be used in external plugins,
+     * or if it can be used only in Elasticsearch modules ("internal" plugins).
+     * Using an entitlement that is not {@code pluginsAccessible} in an external
+     * plugin policy will throw in exception while parsing.
+     */
+    boolean esModulesOnly() default true;
 }
diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java
@@ -26,7 +26,7 @@ public class FileEntitlement implements Entitlement {
     private final String path;
     private final int actions;
 
-    @ExternalEntitlement(parameterNames = { "path", "actions" })
+    @ExternalEntitlement(parameterNames = { "path", "actions" }, esModulesOnly = false)
     public FileEntitlement(String path, List<String> actionsList) {
         this.path = path;
         int actionsInt = 0;
diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java
@@ -39,6 +39,7 @@ public class PolicyParser {
 
     protected final XContentParser policyParser;
     protected final String policyName;
+    private final boolean isExternalPlugin;
 
     static String getEntitlementTypeName(Class<? extends Entitlement> entitlementClass) {
         var entitlementClassName = entitlementClass.getSimpleName();
@@ -56,9 +57,10 @@ static String getEntitlementTypeName(Class<? extends Entitlement> entitlementCla
             .collect(Collectors.joining("_"));
     }
 
-    public PolicyParser(InputStream inputStream, String policyName) throws IOException {
+    public PolicyParser(InputStream inputStream, String policyName, boolean isExternalPlugin) throws IOException {
         this.policyParser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, Objects.requireNonNull(inputStream));
         this.policyName = policyName;
+        this.isExternalPlugin = isExternalPlugin;
     }
 
     public Policy parsePolicy() {
@@ -125,6 +127,10 @@ protected Entitlement parseEntitlement(String scopeName, String entitlementType)
             throw newPolicyParserException(scopeName, "unknown entitlement type [" + entitlementType + "]");
         }
 
+        if (entitlementMetadata.esModulesOnly() && isExternalPlugin) {
+            throw newPolicyParserException("entitlement type [" + entitlementType + "] is allowed only on modules");
+        }
+
         Class<?>[] parameterTypes = entitlementConstructor.getParameterTypes();
         String[] parametersNames = entitlementMetadata.parameterNames();
 
diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java
@@ -19,7 +19,7 @@ public class PolicyParserFailureTests extends ESTestCase {
     public void testParserSyntaxFailures() {
         PolicyParserException ppe = expectThrows(
             PolicyParserException.class,
-            () -> new PolicyParser(new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml")
+            () -> new PolicyParser(new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false)
                 .parsePolicy()
         );
         assertEquals("[1:1] policy parsing error for [test-failure-policy.yaml]: expected object <scope name>", ppe.getMessage());
@@ -29,7 +29,7 @@ public void testEntitlementDoesNotExist() {
         PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream("""
             entitlement-module-name:
               - does_not_exist: {}
-            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy());
+            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy());
         assertEquals(
             "[2:5] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name]: "
                 + "unknown entitlement type [does_not_exist]",
@@ -41,7 +41,7 @@ public void testEntitlementMissingParameter() {
         PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream("""
             entitlement-module-name:
               - file: {}
-            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy());
+            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy());
         assertEquals(
             "[2:12] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] "
                 + "for entitlement type [file]: missing entitlement parameter [path]",
@@ -52,7 +52,7 @@ public void testEntitlementMissingParameter() {
             entitlement-module-name:
               - file:
                   path: test-path
-            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy());
+            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy());
         assertEquals(
             "[4:1] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] "
                 + "for entitlement type [file]: missing entitlement parameter [actions]",
@@ -68,11 +68,22 @@ public void testEntitlementExtraneousParameter() {
                   actions:
                     - read
                   extra: test
-            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy());
+            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy());
         assertEquals(
             "[7:1] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] "
                 + "for entitlement type [file]: extraneous entitlement parameter(s) {extra=test}",
             ppe.getMessage()
         );
     }
+
+    public void testEntitlementIsNotForExternalPlugins() {
+        PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream("""
+            entitlement-module-name:
+              - create_class_loader
+            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", true).parsePolicy());
+        assertEquals(
+            "[2:5] policy parsing error for [test-failure-policy.yaml]: entitlement type [create_class_loader] is allowed only on modules",
+            ppe.getMessage()
+        );
+    }
 }
diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java
@@ -37,7 +37,17 @@ public void testGetEntitlementTypeName() {
     }
 
     public void testPolicyBuilder() throws IOException {
-        Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml")
+        Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml", false)
+            .parsePolicy();
+        Policy builtPolicy = new Policy(
+            "test-policy.yaml",
+            List.of(new Scope("entitlement-module-name", List.of(new FileEntitlement("test/path/to/file", List.of("read", "write")))))
+        );
+        assertEquals(parsedPolicy, builtPolicy);
+    }
+
+    public void testPolicyBuilderOnExternalPlugin() throws IOException {
+        Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml", true)
             .parsePolicy();
         Policy builtPolicy = new Policy(
             "test-policy.yaml",
@@ -50,7 +60,7 @@ public void testParseCreateClassloader() throws IOException {
         Policy parsedPolicy = new PolicyParser(new ByteArrayInputStream("""
             entitlement-module-name:
               - create_class_loader
-            """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml").parsePolicy();
+            """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", false).parsePolicy();
         Policy builtPolicy = new Policy(
             "test-policy.yaml",
             List.of(new Scope("entitlement-module-name", List.of(new CreateClassLoaderEntitlement())))
diff --git a/muted-tests.yml b/muted-tests.yml
@@ -276,9 +276,6 @@ tests:
 - class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests
   method: testBottomFieldSort
   issue: https://github.com/elastic/elasticsearch/issues/118214
-- class: org.elasticsearch.xpack.esql.action.CrossClustersEnrichIT
-  method: testTopNThenEnrichRemote
-  issue: https://github.com/elastic/elasticsearch/issues/118307
 - class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS1UnavailableRemotesIT
   method: testEsqlRcs1UnavailableRemoteScenarios
   issue: https://github.com/elastic/elasticsearch/issues/118350
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.core.AbstractRefCounted;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexVersion;
@@ -59,6 +58,7 @@
 import java.util.Objects;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.bootstrap.BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING;
 import static org.elasticsearch.nativeaccess.WindowsFunctions.ConsoleCtrlHandler.CTRL_CLOSE_EVENT;
@@ -218,10 +218,14 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException {
         if (Boolean.parseBoolean(System.getProperty("es.entitlements.enabled"))) {
             LogManager.getLogger(Elasticsearch.class).info("Bootstrapping Entitlements");
 
-            List<Tuple<Path, Boolean>> pluginData = pluginsLoader.allBundles()
-                .stream()
-                .map(bundle -> Tuple.tuple(bundle.getDir(), bundle.pluginDescriptor().isModular()))
-                .toList();
+            List<EntitlementBootstrap.PluginData> pluginData = Stream.concat(
+                pluginsLoader.moduleBundles()
+                    .stream()
+                    .map(bundle -> new EntitlementBootstrap.PluginData(bundle.getDir(), bundle.pluginDescriptor().isModular(), false)),
+                pluginsLoader.pluginBundles()
+                    .stream()
+                    .map(bundle -> new EntitlementBootstrap.PluginData(bundle.getDir(), bundle.pluginDescriptor().isModular(), true))
+            ).toList();
 
             EntitlementBootstrap.bootstrap(pluginData, pluginsResolver::resolveClassToPluginName);
         } else if (RuntimeVersionFeature.isSecurityManagerAvailable()) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
@@ -65,7 +65,6 @@ public class DiscoveryNodes implements Iterable<DiscoveryNode>, SimpleDiffable<D
     private final String localNodeId;
     @Nullable
     private final DiscoveryNode localNode;
-    private final Version minNonClientNodeVersion;
     private final Version maxNodeVersion;
     private final Version minNodeVersion;
     private final IndexVersion maxDataNodeCompatibleIndexVersion;
@@ -81,7 +80,6 @@ private DiscoveryNodes(
         Map<String, DiscoveryNode> ingestNodes,
         @Nullable String masterNodeId,
         @Nullable String localNodeId,
-        Version minNonClientNodeVersion,
         Version maxNodeVersion,
         Version minNodeVersion,
         IndexVersion maxDataNodeCompatibleIndexVersion,
@@ -98,7 +96,6 @@ private DiscoveryNodes(
         assert (masterNodeId == null) == (masterNode == null);
         this.localNodeId = localNodeId;
         this.localNode = localNodeId == null ? null : nodes.get(localNodeId);
-        this.minNonClientNodeVersion = minNonClientNodeVersion;
         this.minNodeVersion = minNodeVersion;
         this.maxNodeVersion = maxNodeVersion;
         this.maxDataNodeCompatibleIndexVersion = maxDataNodeCompatibleIndexVersion;
@@ -117,7 +114,6 @@ public DiscoveryNodes withMasterNodeId(@Nullable String masterNodeId) {
             ingestNodes,
             masterNodeId,
             localNodeId,
-            minNonClientNodeVersion,
             maxNodeVersion,
             minNodeVersion,
             maxDataNodeCompatibleIndexVersion,
@@ -346,17 +342,6 @@ public boolean isMixedVersionCluster() {
         return minNodeVersion.equals(maxNodeVersion) == false;
     }
 
-    /**
-     * Returns the version of the node with the oldest version in the cluster that is not a client node
-     *
-     * If there are no non-client nodes, Version.CURRENT will be returned.
-     *
-     * @return the oldest version in the cluster
-     */
-    public Version getSmallestNonClientNodeVersion() {
-        return minNonClientNodeVersion;
-    }
-
     /**
      * Returns the highest index version supported by all data nodes in the cluster
      */
@@ -853,14 +838,12 @@ public DiscoveryNodes build() {
              */
             Version minNodeVersion = null;
             Version maxNodeVersion = null;
-            Version minNonClientNodeVersion = null;
             IndexVersion maxDataNodeCompatibleIndexVersion = null;
             IndexVersion minSupportedIndexVersion = null;
             for (Map.Entry<String, DiscoveryNode> nodeEntry : nodes.entrySet()) {
                 DiscoveryNode discoNode = nodeEntry.getValue();
                 Version version = discoNode.getVersion();
                 if (discoNode.canContainData() || discoNode.isMasterNode()) {
-                    minNonClientNodeVersion = min(minNonClientNodeVersion, version);
                     maxDataNodeCompatibleIndexVersion = min(maxDataNodeCompatibleIndexVersion, discoNode.getMaxIndexVersion());
                 }
                 minNodeVersion = min(minNodeVersion, version);
@@ -894,7 +877,6 @@ public DiscoveryNodes build() {
                 filteredNodes(nodes, DiscoveryNode::isIngestNode),
                 masterNodeId,
                 localNodeId,
-                Objects.requireNonNullElse(minNonClientNodeVersion, Version.CURRENT),
                 Objects.requireNonNullElse(maxNodeVersion, Version.CURRENT),
                 Objects.requireNonNullElse(minNodeVersion, Version.CURRENT.minimumCompatibilityVersion()),
                 Objects.requireNonNullElse(maxDataNodeCompatibleIndexVersion, IndexVersion.current()),
diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java
diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsLoader.java b/server/src/main/java/org/elasticsearch/plugins/PluginsLoader.java
diff --git a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java
diff --git a/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java b/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartTrialAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartTrialAction.java
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalVerifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalVerifier.java
