Merge branch 'main' into fork_n_ary_alt

Author: ioanatia

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -22,11 +22,11 @@ public enum DockerBase {
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:c4e10ecf3d8a21cf4be2fb53a2f522de50e14c80ce1da487e3ffd13f4d48d24d",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:29150cd940cc7f69407d978d5a19c86f4d9e67cf44e4d6ded787a497e8f27c9a",
         "-wolfi",
         "apk"
     ),
-    FIPS("docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7", "-fips", "apk"),
+    FIPS("docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7", "-fips", "apk"),
     // spotless:on
     // Based on WOLFI above, with more extras. We don't set a base image because
     // we programmatically extend from the wolfi image.

build-tools-internal/version.properties

@@ -38,8 +38,8 @@ protobuf          = 3.25.5
 # test dependencies
 randomizedrunner  = 2.8.2
 junit             = 4.13.2
-junit5            = 5.7.1
-hamcrest          = 2.1
+junit5            = 5.12.1
+hamcrest          = 3.0
 mocksocket        = 1.2
 
 # test container dependencies

distribution/tools/plugin-cli/build.gradle

@@ -24,7 +24,8 @@ dependencies {
   compileOnly project(":libs:cli")
   implementation project(":libs:plugin-api")
   implementation project(":libs:plugin-scanner")
-  // TODO: asm is picked up from the plugin scanner, we should consolidate so it is not defined twice
+  implementation project(":libs:entitlement")
+  // TODO: asm is picked up from the plugin scanner and entitlements, we should consolidate so it is not defined twice
   implementation 'org.ow2.asm:asm:9.7.1'
   implementation 'org.ow2.asm:asm-tree:9.7.1'
 

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java

@@ -24,8 +24,6 @@
 import org.bouncycastle.openpgp.operator.jcajce.JcaKeyFingerprintCalculator;
 import org.bouncycastle.openpgp.operator.jcajce.JcaPGPContentVerifierBuilderProvider;
 import org.elasticsearch.Build;
-import org.elasticsearch.bootstrap.PluginPolicyInfo;
-import org.elasticsearch.bootstrap.PolicyUtil;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
@@ -36,9 +34,9 @@
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.entitlement.runtime.policy.PolicyUtils;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.jdk.JarHell;
-import org.elasticsearch.jdk.RuntimeVersionFeature;
 import org.elasticsearch.plugin.scanner.ClassReaders;
 import org.elasticsearch.plugin.scanner.NamedComponentScanner;
 import org.elasticsearch.plugins.Platforms;
@@ -934,13 +932,10 @@ private PluginDescriptor installPlugin(InstallablePlugin descriptor, Path tmpRoo
             );
         }
 
-        if (RuntimeVersionFeature.isSecurityManagerAvailable()) {
-            PluginPolicyInfo pluginPolicy = PolicyUtil.getPluginPolicyInfo(tmpRoot, env.tmpDir());
-            if (pluginPolicy != null) {
-                Set<String> permissions = PluginSecurity.getPermissionDescriptions(pluginPolicy, env.tmpDir());
-                PluginSecurity.confirmPolicyExceptions(terminal, permissions, batch);
-            }
-        }
+        var pluginPolicy = PolicyUtils.parsePolicyIfExists(info.getName(), tmpRoot, true);
+
+        Set<String> entitlements = PolicyUtils.getEntitlementsDescriptions(pluginPolicy);
+        PluginSecurity.confirmPolicyExceptions(terminal, entitlements, batch);
 
         // Validate that the downloaded plugin's ID matches what we expect from the descriptor. The
         // exception is if we install a plugin via `InstallPluginCommand` by specifying a URL or

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginSecurity.java

@@ -9,27 +9,19 @@
 
 package org.elasticsearch.plugins.cli;
 
-import org.elasticsearch.bootstrap.PluginPolicyInfo;
-import org.elasticsearch.bootstrap.PolicyUtil;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.Terminal.Verbosity;
 import org.elasticsearch.cli.UserException;
 
-import java.io.IOException;
-import java.net.URL;
-import java.nio.file.Path;
-import java.security.Permission;
-import java.security.UnresolvedPermission;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
 /**
- * Contains methods for displaying extended plugin permissions to the user, and confirming that
+ * Contains methods for displaying extended plugin entitlements to the user, and confirming that
  * plugin installation can proceed.
  */
 public class PluginSecurity {
@@ -40,37 +32,36 @@ public class PluginSecurity {
     /**
      * prints/confirms policy exceptions with the user
      */
-    static void confirmPolicyExceptions(Terminal terminal, Set<String> permissions, boolean batch) throws UserException {
-        List<String> requested = new ArrayList<>(permissions);
+    static void confirmPolicyExceptions(Terminal terminal, Set<String> entitlements, boolean batch) throws UserException {
+        List<String> requested = new ArrayList<>(entitlements);
         if (requested.isEmpty()) {
-            terminal.println(Verbosity.VERBOSE, "plugin has a policy file with no additional permissions");
+            terminal.println(
+                Verbosity.NORMAL,
+                "WARNING: plugin has a policy file with no additional entitlements. Double check this is intentional."
+            );
         } else {
-            // sort permissions in a reasonable order
+            // sort entitlements in a reasonable order
             Collections.sort(requested);
 
             if (terminal.isHeadless()) {
                 terminal.errorPrintln(
-                    "WARNING: plugin requires additional permissions: ["
+                    "WARNING: plugin requires additional entitlements: ["
                         + requested.stream().map(each -> '\'' + each + '\'').collect(Collectors.joining(", "))
                         + "]"
                 );
                 terminal.errorPrintln(
-                    "See https://docs.oracle.com/javase/8/docs/technotes/guides/security/permissions.html"
-                        + " for descriptions of what these permissions allow and the associated risks."
+                    "See " + ENTITLEMENTS_DESCRIPTION_URL + " for descriptions of what these entitlements allow and the associated risks."
                 );
             } else {
                 terminal.errorPrintln(Verbosity.NORMAL, "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@");
-                terminal.errorPrintln(Verbosity.NORMAL, "@     WARNING: plugin requires additional permissions     @");
+                terminal.errorPrintln(Verbosity.NORMAL, "@     WARNING: plugin requires additional entitlements    @");
                 terminal.errorPrintln(Verbosity.NORMAL, "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@");
-                // print all permissions:
-                for (String permission : requested) {
-                    terminal.errorPrintln(Verbosity.NORMAL, "* " + permission);
+                // print all entitlements:
+                for (String entitlement : requested) {
+                    terminal.errorPrintln(Verbosity.NORMAL, "* " + entitlement);
                 }
-                terminal.errorPrintln(
-                    Verbosity.NORMAL,
-                    "See https://docs.oracle.com/javase/8/docs/technotes/guides/security/permissions.html"
-                );
-                terminal.errorPrintln(Verbosity.NORMAL, "for descriptions of what these permissions allow and the associated risks.");
+                terminal.errorPrintln(Verbosity.NORMAL, "See " + ENTITLEMENTS_DESCRIPTION_URL);
+                terminal.errorPrintln(Verbosity.NORMAL, "for descriptions of what these entitlements allow and the associated risks.");
 
                 if (batch == false) {
                     prompt(terminal);
@@ -86,53 +77,4 @@ private static void prompt(final Terminal terminal) throws UserException {
             throw new UserException(ExitCodes.DATA_ERROR, "installation aborted by user");
         }
     }
-
-    /** Format permission type, name, and actions into a string */
-    static String formatPermission(Permission permission) {
-        StringBuilder sb = new StringBuilder();
-
-        String clazz = null;
-        if (permission instanceof UnresolvedPermission) {
-            clazz = ((UnresolvedPermission) permission).getUnresolvedType();
-        } else {
-            clazz = permission.getClass().getName();
-        }
-        sb.append(clazz);
-
-        String name = null;
-        if (permission instanceof UnresolvedPermission) {
-            name = ((UnresolvedPermission) permission).getUnresolvedName();
-        } else {
-            name = permission.getName();
-        }
-        if (name != null && name.length() > 0) {
-            sb.append(' ');
-            sb.append(name);
-        }
-
-        String actions = null;
-        if (permission instanceof UnresolvedPermission) {
-            actions = ((UnresolvedPermission) permission).getUnresolvedActions();
-        } else {
-            actions = permission.getActions();
-        }
-        if (actions != null && actions.length() > 0) {
-            sb.append(' ');
-            sb.append(actions);
-        }
-        return sb.toString();
-    }
-
-    /**
-     * Extract a unique set of permissions from the plugin's policy file. Each permission is formatted for output to users.
-     */
-    public static Set<String> getPermissionDescriptions(PluginPolicyInfo pluginPolicyInfo, Path tmpDir) throws IOException {
-        Set<Permission> allPermissions = new HashSet<>(PolicyUtil.getPolicyPermissions(null, pluginPolicyInfo.policy(), tmpDir));
-        for (URL jar : pluginPolicyInfo.jars()) {
-            Set<Permission> jarPermissions = PolicyUtil.getPolicyPermissions(jar, pluginPolicyInfo.policy(), tmpDir);
-            allPermissions.addAll(jarPermissions);
-        }
-
-        return allPermissions.stream().map(PluginSecurity::formatPermission).collect(Collectors.toSet());
-    }
 }

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java

@@ -41,14 +41,15 @@
 import org.elasticsearch.common.hash.MessageDigests;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.PathUtilsForTesting;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.entitlement.runtime.policy.PolicyUtils;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
-import org.elasticsearch.jdk.RuntimeVersionFeature;
 import org.elasticsearch.plugin.scanner.NamedComponentScanner;
 import org.elasticsearch.plugins.Platforms;
 import org.elasticsearch.plugins.PluginDescriptor;
@@ -57,6 +58,8 @@
 import org.elasticsearch.test.PosixPermissionsResetter;
 import org.elasticsearch.test.compiler.InMemoryJavaCompiler;
 import org.elasticsearch.test.jar.JarUtils;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.yaml.YamlXContent;
 import org.junit.After;
 import org.junit.Before;
 
@@ -102,6 +105,7 @@
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
+import static org.elasticsearch.entitlement.runtime.policy.PolicyManager.ALL_UNNAMED;
 import static org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase.forEachFileRecursively;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.containsInAnyOrder;
@@ -137,8 +141,6 @@ public class InstallPluginActionTests extends ESTestCase {
 
     @SuppressForbidden(reason = "sets java.io.tmpdir")
     public InstallPluginActionTests(FileSystem fs, Function<String, Path> temp) {
-        assert "false".equals(System.getProperty("tests.security.manager")) : "-Dtests.security.manager=false has to be set";
-
         this.temp = temp;
         this.isPosix = fs.supportedFileAttributeViews().contains("posix");
         this.isReal = fs == PathUtils.getDefaultFileSystem();
@@ -309,15 +311,20 @@ private static String[] pluginProperties(String name, String[] additionalProps,
         ).flatMap(Function.identity()).toArray(String[]::new);
     }
 
-    static void writePluginSecurityPolicy(Path pluginDir, String... permissions) throws IOException {
-        StringBuilder securityPolicyContent = new StringBuilder("grant {\n  ");
-        for (String permission : permissions) {
-            securityPolicyContent.append("permission java.lang.RuntimePermission \"");
-            securityPolicyContent.append(permission);
-            securityPolicyContent.append("\";");
+    static void writePluginEntitlementPolicy(Path pluginDir, String moduleName, CheckedConsumer<XContentBuilder, IOException> policyBuilder)
+        throws IOException {
+        try (var builder = YamlXContent.contentBuilder()) {
+            builder.startObject();
+            builder.field(moduleName);
+            builder.startArray();
+
+            policyBuilder.accept(builder);
+            builder.endArray();
+            builder.endObject();
+
+            String policy = org.elasticsearch.common.Strings.toString(builder);
+            Files.writeString(pluginDir.resolve(PolicyUtils.POLICY_FILE_NAME), policy);
         }
-        securityPolicyContent.append("\n};\n");
-        Files.writeString(pluginDir.resolve("plugin-security.policy"), securityPolicyContent.toString());
     }
 
     static InstallablePlugin createStablePlugin(String name, Path structure, boolean hasNamedComponentFile, String... additionalProps)
@@ -892,9 +899,8 @@ public void testInstallMisspelledOfficialPlugins() {
     }
 
     public void testBatchFlag() throws Exception {
-        assumeTrue("security policy validation only available with SecurityManager", RuntimeVersionFeature.isSecurityManagerAvailable());
         installPlugin(true);
-        assertThat(terminal.getErrorOutput(), containsString("WARNING: plugin requires additional permissions"));
+        assertThat(terminal.getErrorOutput(), containsString("WARNING: plugin requires additional entitlements"));
         assertThat(terminal.getOutput(), containsString("-> Downloading"));
         // No progress bar in batch mode
         assertThat(terminal.getOutput(), not(containsString("100%")));
@@ -942,12 +948,12 @@ public void testPluginHasDifferentNameThatDescriptor() throws Exception {
         assertThat(e.getMessage(), equalTo("Expected downloaded plugin to have ID [other-fake] but found [fake]"));
     }
 
-    private void installPlugin(boolean isBatch, String... additionalProperties) throws Exception {
-        // if batch is enabled, we also want to add a security policy
+    private void installPlugin(boolean isBatch) throws Exception {
+        // if batch is enabled, we also want to add an entitlement policy
         if (isBatch) {
-            writePluginSecurityPolicy(pluginDir, "setFactory");
+            writePluginEntitlementPolicy(pluginDir, ALL_UNNAMED, builder -> builder.value("manage_threads"));
         }
-        InstallablePlugin pluginZip = createPlugin("fake", pluginDir, additionalProperties);
+        InstallablePlugin pluginZip = createPlugin("fake", pluginDir);
         skipJarHellAction.setEnvironment(env.v2());
         skipJarHellAction.setBatch(isBatch);
         skipJarHellAction.execute(List.of(pluginZip));
@@ -1531,11 +1537,13 @@ private void assertPolicyConfirmation(Tuple<Path, Environment> pathEnvironmentTu
     }
 
     public void testPolicyConfirmation() throws Exception {
-        assumeTrue("security policy parsing only available with SecurityManager", RuntimeVersionFeature.isSecurityManagerAvailable());
-        writePluginSecurityPolicy(pluginDir, "getClassLoader", "setFactory");
+        writePluginEntitlementPolicy(pluginDir, "test.plugin.module", builder -> {
+            builder.value("manage_threads");
+            builder.value("outbound_network");
+        });
         InstallablePlugin pluginZip = createPluginZip("fake", pluginDir);
 
-        assertPolicyConfirmation(env, pluginZip, "plugin requires additional permissions");
+        assertPolicyConfirmation(env, pluginZip, "plugin requires additional entitlements");
         assertPlugin("fake", pluginDir, env.v2());
     }
 

docs/changelog/123156.yaml

@@ -0,0 +1,5 @@
+pr: 123156
+summary: Wrap remote errors with cluster name to provide more context
+area: Search
+type: enhancement
+issues: []

docs/changelog/125477.yaml

@@ -0,0 +1,6 @@
+pr: 125477
+summary: Prevent get datafeeds stats API returning an error when local tasks are slow to stop
+area: Machine Learning
+type: bug
+issues:
+  - 104160

docs/changelog/125652.yaml

@@ -0,0 +1,5 @@
+pr: 125652
+summary: Run `TransportGetIndexAction` on local node
+area: Indices APIs
+type: enhancement
+issues: []

docs/changelog/125694.yaml

@@ -0,0 +1,5 @@
+pr: 125694
+summary: LTR score bounding
+area: Ranking
+type: bug
+issues: []