Merge remote-tracking branch 'upstream/main' into test/relax-int4-flat-test

Author: benwtrent

.buildkite/pipelines/intake.template.yml

@@ -96,7 +96,7 @@ steps:
   - trigger: elasticsearch-dra-workflow
     label: Trigger DRA snapshot workflow
     async: true
-    branches: "main 8.* 7.17"
+    branches: "main 9.* 8.* 7.17"
     build:
       branch: "$BUILDKITE_BRANCH"
       commit: "$BUILDKITE_COMMIT"

.buildkite/pipelines/intake.yml

@@ -97,7 +97,7 @@ steps:
   - trigger: elasticsearch-dra-workflow
     label: Trigger DRA snapshot workflow
     async: true
-    branches: "main 8.* 7.17"
+    branches: "main 9.* 8.* 7.17"
     build:
       branch: "$BUILDKITE_BRANCH"
       commit: "$BUILDKITE_COMMIT"

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/EntitlementTest.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.qa.test;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Target(ElementType.METHOD)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface EntitlementTest {
+    enum ExpectedAccess {
+        PLUGINS,
+        ES_MODULES_ONLY,
+        ALWAYS_DENIED
+    }
+
+    ExpectedAccess expectedAccess();
+
+    int fromJavaVersion() default -1;
+}

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java

@@ -22,6 +22,8 @@
 import java.nio.file.attribute.UserPrincipal;
 import java.util.Scanner;
 
+import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS;
+
 @SuppressForbidden(reason = "Explicitly checking APIs that are forbidden")
 class FileCheckActions {
 
@@ -43,38 +45,47 @@ private static Path readWriteFile() {
         return testRootDir.resolve("read_write_file");
     }
 
+    @EntitlementTest(expectedAccess = PLUGINS)
     static void createScannerFile() throws FileNotFoundException {
         new Scanner(readFile().toFile());
     }
 
+    @EntitlementTest(expectedAccess = PLUGINS)
     static void createScannerFileWithCharset() throws IOException {
         new Scanner(readFile().toFile(), StandardCharsets.UTF_8);
     }
 
+    @EntitlementTest(expectedAccess = PLUGINS)
     static void createScannerFileWithCharsetName() throws FileNotFoundException {
         new Scanner(readFile().toFile(), "UTF-8");
     }
 
+    @EntitlementTest(expectedAccess = PLUGINS)
     static void createFileOutputStreamString() throws IOException {
         new FileOutputStream(readWriteFile().toString()).close();
     }
 
+    @EntitlementTest(expectedAccess = PLUGINS)
     static void createFileOutputStreamStringWithAppend() throws IOException {
         new FileOutputStream(readWriteFile().toString(), false).close();
     }
 
+    @EntitlementTest(expectedAccess = PLUGINS)
     static void createFileOutputStreamFile() throws IOException {
         new FileOutputStream(readWriteFile().toFile()).close();
     }
 
+    @EntitlementTest(expectedAccess = PLUGINS)
     static void createFileOutputStreamFileWithAppend() throws IOException {
         new FileOutputStream(readWriteFile().toFile(), false).close();
     }
 
+    @EntitlementTest(expectedAccess = PLUGINS)
     static void filesProbeContentType() throws IOException {
         Files.probeContentType(readFile());
     }
 
+    @EntitlementTest(expectedAccess = PLUGINS)
     static void filesSetOwner() throws IOException {
         UserPrincipal owner = EntitledActions.getFileOwner(readWriteFile());
         Files.setOwner(readWriteFile(), owner); // set to existing owner, just trying to execute the method

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java

@@ -31,10 +31,22 @@
 
 public class EntitlementBootstrap {
 
-    public record BootstrapArgs(Map<String, Policy> pluginPolicies, Function<Class<?>, String> pluginResolver) {
+    public record BootstrapArgs(
+        Map<String, Policy> pluginPolicies,
+        Function<Class<?>, String> pluginResolver,
+        Path[] dataDirs,
+        Path configDir,
+        Path tempDir
+    ) {
         public BootstrapArgs {
             requireNonNull(pluginPolicies);
             requireNonNull(pluginResolver);
+            requireNonNull(dataDirs);
+            if (dataDirs.length == 0) {
+                throw new IllegalArgumentException("must provide at least one data directory");
+            }
+            requireNonNull(configDir);
+            requireNonNull(tempDir);
         }
     }
 
@@ -50,13 +62,22 @@ public static BootstrapArgs bootstrapArgs() {
      *
      * @param pluginPolicies a map holding policies for plugins (and modules), by plugin (or module) name.
      * @param pluginResolver a functor to map a Java Class to the plugin it belongs to (the plugin name).
+     * @param dataDirs data directories for Elasticsearch
+     * @param configDir the config directory for Elasticsearch
+     * @param tempDir the temp directory for Elasticsearch
      */
-    public static void bootstrap(Map<String, Policy> pluginPolicies, Function<Class<?>, String> pluginResolver) {
+    public static void bootstrap(
+        Map<String, Policy> pluginPolicies,
+        Function<Class<?>, String> pluginResolver,
+        Path[] dataDirs,
+        Path configDir,
+        Path tempDir
+    ) {
         logger.debug("Loading entitlement agent");
         if (EntitlementBootstrap.bootstrapArgs != null) {
             throw new IllegalStateException("plugin data is already set");
         }
-        EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(pluginPolicies, pluginResolver);
+        EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(pluginPolicies, pluginResolver, dataDirs, configDir, tempDir);
         exportInitializationToAgent();
         loadAgent(findAgentJar());
         selfTest();

libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java

@@ -9,10 +9,8 @@
 
 package org.elasticsearch.entitlement.runtime.policy;
 
-import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.entitlement.runtime.policy.entitlements.FileEntitlement;
 
-import java.io.File;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -51,20 +49,10 @@ boolean canRead(Path path) {
         return checkPath(normalize(path), readPaths);
     }
 
-    @SuppressForbidden(reason = "Explicitly checking File apis")
-    boolean canRead(File file) {
-        return checkPath(normalize(file.toPath()), readPaths);
-    }
-
     boolean canWrite(Path path) {
         return checkPath(normalize(path), writePaths);
     }
 
-    @SuppressForbidden(reason = "Explicitly checking File apis")
-    boolean canWrite(File file) {
-        return checkPath(normalize(file.toPath()), writePaths);
-    }
-
     private static String normalize(Path path) {
         return path.toAbsolutePath().normalize().toString();
     }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java

@@ -169,23 +169,7 @@ private static void validateEntitlementsPerModule(String sourceName, String modu
     }
 
     public void checkStartProcess(Class<?> callerClass) {
-        neverEntitled(callerClass, "start process");
-    }
-
-    private void neverEntitled(Class<?> callerClass, String operationDescription) {
-        var requestingClass = requestingClass(callerClass);
-        if (isTriviallyAllowed(requestingClass)) {
-            return;
-        }
-
-        throw new NotEntitledException(
-            Strings.format(
-                "Not entitled: caller [%s], module [%s], operation [%s]",
-                callerClass,
-                requestingClass.getModule() == null ? "<none>" : requestingClass.getModule().getName(),
-                operationDescription
-            )
-        );
+        neverEntitled(callerClass, () -> "start process");
     }
 
     /**
@@ -241,31 +225,9 @@ public void checkChangeNetworkHandling(Class<?> callerClass) {
         checkChangeJVMGlobalState(callerClass);
     }
 
-    /**
-     * Check for operations that can access sensitive network information, e.g. secrets, tokens or SSL sessions
-     */
-    public void checkReadSensitiveNetworkInformation(Class<?> callerClass) {
-        neverEntitled(callerClass, "access sensitive network information");
-    }
-
     @SuppressForbidden(reason = "Explicitly checking File apis")
     public void checkFileRead(Class<?> callerClass, File file) {
-        var requestingClass = requestingClass(callerClass);
-        if (isTriviallyAllowed(requestingClass)) {
-            return;
-        }
-
-        ModuleEntitlements entitlements = getEntitlements(requestingClass);
-        if (entitlements.fileAccess().canRead(file) == false) {
-            throw new NotEntitledException(
-                Strings.format(
-                    "Not entitled: caller [%s], module [%s], entitlement [file], operation [read], path [%s]",
-                    callerClass,
-                    requestingClass.getModule(),
-                    file
-                )
-            );
-        }
+        checkFileRead(callerClass, file.toPath());
     }
 
     public void checkFileRead(Class<?> callerClass, Path path) {
@@ -289,22 +251,7 @@ public void checkFileRead(Class<?> callerClass, Path path) {
 
     @SuppressForbidden(reason = "Explicitly checking File apis")
     public void checkFileWrite(Class<?> callerClass, File file) {
-        var requestingClass = requestingClass(callerClass);
-        if (isTriviallyAllowed(requestingClass)) {
-            return;
-        }
-
-        ModuleEntitlements entitlements = getEntitlements(requestingClass);
-        if (entitlements.fileAccess().canWrite(file) == false) {
-            throw new NotEntitledException(
-                Strings.format(
-                    "Not entitled: caller [%s], module [%s], entitlement [file], operation [write], path [%s]",
-                    callerClass,
-                    requestingClass.getModule(),
-                    file
-                )
-            );
-        }
+        checkFileWrite(callerClass, file.toPath());
     }
 
     public void checkFileWrite(Class<?> callerClass, Path path) {

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java

@@ -238,7 +238,6 @@ public void testRequestingClassFastPath() throws IOException, ClassNotFoundExcep
     }
 
     public void testRequestingModuleWithStackWalk() throws IOException, ClassNotFoundException {
-        var agentsClass = new TestAgent();
         var entitlementsClass = makeClassInItsOwnModule();    // A class in the entitlements library itself
         var requestingClass = makeClassInItsOwnModule();      // This guy is always the right answer
         var instrumentedClass = makeClassInItsOwnModule();    // The class that called the check method
@@ -365,13 +364,6 @@ private static Class<?> makeClassInItsOwnModule() throws IOException, ClassNotFo
         return layer.findLoader("org.example.plugin").loadClass("q.B");
     }
 
-    private static Class<?> makeClassInItsOwnUnnamedModule() throws IOException, ClassNotFoundException {
-        final Path home = createTempDir();
-        Path jar = createMockPluginJar(home);
-        var layer = createLayerForJar(jar, "org.example.plugin");
-        return layer.findLoader("org.example.plugin").loadClass("q.B");
-    }
-
     private static PolicyManager policyManager(String agentsPackageName, Module entitlementsModule) {
         return new PolicyManager(createEmptyTestServerPolicy(), List.of(), Map.of(), c -> "test", agentsPackageName, entitlementsModule);
     }

libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java

@@ -357,8 +357,6 @@ tests:
 - class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
   method: test {yaml=indices.get_alias/10_basic/Get aliases via /*/_alias/}
   issue: https://github.com/elastic/elasticsearch/issues/121290
-- class: org.elasticsearch.xpack.inference.common.InferenceServiceNodeLocalRateLimitCalculatorTests
-  issue: https://github.com/elastic/elasticsearch/issues/121294
 - class: org.elasticsearch.env.NodeEnvironmentTests
   method: testGetBestDowngradeVersion
   issue: https://github.com/elastic/elasticsearch/issues/121316
@@ -380,6 +378,11 @@ tests:
 - class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
   method: testActivateProfile
   issue: https://github.com/elastic/elasticsearch/issues/121151
+- class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
+  issue: https://github.com/elastic/elasticsearch/issues/121407
+- class: org.elasticsearch.xpack.ml.integration.ClassificationIT
+  method: testDependentVariableIsAliasToNested
+  issue: https://github.com/elastic/elasticsearch/issues/121415
 
 # Examples:
 #