Merge branch 'main' into entitlements/file_read_methods

Author: rjernst

.buildkite/hooks/pre-command

@@ -44,6 +44,9 @@ export GRADLE_BUILD_CACHE_USERNAME
 GRADLE_BUILD_CACHE_PASSWORD=$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)
 export GRADLE_BUILD_CACHE_PASSWORD
 
+DEVELOCITY_ACCESS_KEY="gradle-enterprise.elastic.co=$(vault read -field=accesskey secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)"
+export DEVELOCITY_ACCESS_KEY
+
 BUILDKITE_API_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/buildkite-api-token)
 export BUILDKITE_API_TOKEN
 

.ci/init.gradle

@@ -1,29 +1,24 @@
+final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url')
+final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false'))
+
 gradle.settingsEvaluated { settings ->
   settings.pluginManager.withPlugin("com.gradle.develocity") {
     settings.develocity {
-      server = 'https://gradle-enterprise.elastic.co'
+      server = "https://gradle-enterprise.elastic.co"
     }
-  }
-}
-
-final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url')
-final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false'))
-
-if (buildCacheUrl) {
-  gradle.settingsEvaluated { settings ->
-    settings.buildCache {
-      local {
-        // Disable the local build cache in CI since we use ephemeral workers and it incurs an IO penalty
-        enabled = false
-      }
-      remote(HttpBuildCache) {
-        url = buildCacheUrl
-        push = buildCachePush
-        credentials {
-          username = System.getenv("GRADLE_BUILD_CACHE_USERNAME")
-          password = System.getenv("GRADLE_BUILD_CACHE_PASSWORD")
+    if (buildCacheUrl) {
+      settings.buildCache {
+        local {
+          // Disable the local build cache in CI since we use ephemeral workers and it incurs an IO penalty
+          enabled = false
+        }
+        remote(settings.develocity.buildCache) {
+          enabled = true
+          push = buildCachePush
         }
       }
     }
   }
 }
+
+

build-conventions/settings.gradle

@@ -6,6 +6,11 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
+
+plugins {
+  id "com.gradle.develocity" version "3.18.1"
+}
+
 rootProject.name = 'build-conventions'
 
 dependencyResolutionManagement {

build-tools-internal/settings.gradle

@@ -8,6 +8,10 @@ pluginManagement {
     includeBuild "../build-tools"
 }
 
+plugins {
+  id "com.gradle.develocity" version "3.18.1"
+}
+
 dependencyResolutionManagement {
     versionCatalogs {
         buildLibs {

build-tools/settings.gradle

@@ -9,7 +9,9 @@
 pluginManagement {
     includeBuild "../build-conventions"
 }
-
+plugins {
+  id "com.gradle.develocity" version "3.18.1"
+}
 include 'reaper'
 
 dependencyResolutionManagement {

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/LoadNativeLibrariesCheckActions.java

@@ -12,15 +12,15 @@
 class LoadNativeLibrariesCheckActions {
     static void runtimeLoad() {
         try {
-            Runtime.getRuntime().load("libSomeLibFile.so");
+            Runtime.getRuntime().load(FileCheckActions.readDir().resolve("libSomeLibFile.so").toString());
         } catch (UnsatisfiedLinkError ignored) {
             // The library does not exist, so we expect to fail loading it
         }
     }
 
     static void systemLoad() {
         try {
-            System.load("libSomeLibFile.so");
+            System.load(FileCheckActions.readDir().resolve("libSomeLibFile.so").toString());
         } catch (UnsatisfiedLinkError ignored) {
             // The library does not exist, so we expect to fail loading it
         }

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NativeActions.java

@@ -113,7 +113,7 @@ static void memorySegmentReinterpretWithSizeAndCleanup() {
     @EntitlementTest(expectedAccess = PLUGINS)
     static void symbolLookupWithPath() {
         try {
-            SymbolLookup.libraryLookup(Path.of("/foo/bar/libFoo.so"), Arena.ofAuto());
+            SymbolLookup.libraryLookup(FileCheckActions.readDir().resolve("libFoo.so"), Arena.ofAuto());
         } catch (IllegalArgumentException e) {
             // IllegalArgumentException is thrown if path does not point to a valid library (and it does not)
         }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -18,6 +18,7 @@
 import org.elasticsearch.entitlement.instrumentation.MethodKey;
 import org.elasticsearch.entitlement.instrumentation.Transformer;
 import org.elasticsearch.entitlement.runtime.api.ElasticsearchEntitlementChecker;
+import org.elasticsearch.entitlement.runtime.policy.PathLookup;
 import org.elasticsearch.entitlement.runtime.policy.Policy;
 import org.elasticsearch.entitlement.runtime.policy.PolicyManager;
 import org.elasticsearch.entitlement.runtime.policy.Scope;
@@ -48,7 +49,6 @@
 import java.nio.file.attribute.FileAttribute;
 import java.nio.file.spi.FileSystemProvider;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -126,9 +126,9 @@ private static Class<?>[] findClassesToRetransform(Class<?>[] loadedClasses, Set
     }
 
     private static PolicyManager createPolicyManager() {
-        Map<String, Policy> pluginPolicies = EntitlementBootstrap.bootstrapArgs().pluginPolicies();
-        Path[] dataDirs = EntitlementBootstrap.bootstrapArgs().dataDirs();
-        Path tempDir = EntitlementBootstrap.bootstrapArgs().tempDir();
+        EntitlementBootstrap.BootstrapArgs bootstrapArgs = EntitlementBootstrap.bootstrapArgs();
+        Map<String, Policy> pluginPolicies = bootstrapArgs.pluginPolicies();
+        var pathLookup = new PathLookup(bootstrapArgs.configDir(), bootstrapArgs.dataDirs(), bootstrapArgs.tempDir());
 
         // TODO(ES-10031): Decide what goes in the elasticsearch default policy and extend it
         var serverPolicy = new Policy(
@@ -147,7 +147,7 @@ private static PolicyManager createPolicyManager() {
                         new LoadNativeLibrariesEntitlement(),
                         new ManageThreadsEntitlement(),
                         new FilesEntitlement(
-                            List.of(new FilesEntitlement.FileData(EntitlementBootstrap.bootstrapArgs().tempDir().toString(), READ_WRITE))
+                            List.of(FilesEntitlement.FileData.ofPath(EntitlementBootstrap.bootstrapArgs().tempDir(), READ_WRITE))
                         )
                     )
                 ),
@@ -159,7 +159,7 @@ private static PolicyManager createPolicyManager() {
                     "org.elasticsearch.nativeaccess",
                     List.of(
                         new LoadNativeLibrariesEntitlement(),
-                        new FilesEntitlement(Arrays.stream(dataDirs).map(d -> new FileData(d.toString(), READ_WRITE)).toList())
+                        new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE)))
                     )
                 )
             )
@@ -175,7 +175,7 @@ private static PolicyManager createPolicyManager() {
             resolver,
             AGENTS_PACKAGE_NAME,
             ENTITLEMENTS_MODULE,
-            tempDir
+            pathLookup
         );
     }
 

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java

@@ -838,7 +838,7 @@ public void checkSelectorProviderInheritedChannel(Class<?> callerClass, Selector
 
     @Override
     public void check$java_lang_Runtime$load(Class<?> callerClass, Runtime that, String filename) {
-        // TODO: check filesystem entitlement READ
+        policyManager.checkFileRead(callerClass, Path.of(filename));
         policyManager.checkLoadingNativeLibraries(callerClass);
     }
 
@@ -849,7 +849,7 @@ public void checkSelectorProviderInheritedChannel(Class<?> callerClass, Selector
 
     @Override
     public void check$java_lang_System$$load(Class<?> callerClass, String filename) {
-        // TODO: check filesystem entitlement READ
+        policyManager.checkFileRead(callerClass, Path.of(filename));
         policyManager.checkLoadingNativeLibraries(callerClass);
     }
 
@@ -933,7 +933,7 @@ public void checkSelectorProviderInheritedChannel(Class<?> callerClass, Selector
 
     @Override
     public void check$java_lang_foreign_SymbolLookup$$libraryLookup(Class<?> callerClass, Path path, Arena arena) {
-        // TODO: check filesystem entitlement READ
+        policyManager.checkFileRead(callerClass, path);
         policyManager.checkLoadingNativeLibraries(callerClass);
     }
 

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java

@@ -20,26 +20,30 @@
 import static org.elasticsearch.core.PathUtils.getDefaultFileSystem;
 
 public final class FileAccessTree {
+
     private static final String FILE_SEPARATOR = getDefaultFileSystem().getSeparator();
 
     private final String[] readPaths;
     private final String[] writePaths;
 
-    private FileAccessTree(FilesEntitlement filesEntitlement, Path tempDir) {
+    private FileAccessTree(FilesEntitlement filesEntitlement, PathLookup pathLookup) {
         List<String> readPaths = new ArrayList<>();
         List<String> writePaths = new ArrayList<>();
         for (FilesEntitlement.FileData fileData : filesEntitlement.filesData()) {
-            var path = normalizePath(Path.of(fileData.path()));
             var mode = fileData.mode();
-            if (mode == FilesEntitlement.Mode.READ_WRITE) {
-                writePaths.add(path);
-            }
-            readPaths.add(path);
+            var paths = fileData.resolvePaths(pathLookup);
+            paths.forEach(path -> {
+                var normalized = normalizePath(path);
+                if (mode == FilesEntitlement.Mode.READ_WRITE) {
+                    writePaths.add(normalized);
+                }
+                readPaths.add(normalized);
+            });
         }
 
         // everything has access to the temp dir
-        readPaths.add(tempDir.toString());
-        writePaths.add(tempDir.toString());
+        readPaths.add(pathLookup.tempDir().toString());
+        writePaths.add(pathLookup.tempDir().toString());
 
         readPaths.sort(String::compareTo);
         writePaths.sort(String::compareTo);
@@ -48,8 +52,8 @@ private FileAccessTree(FilesEntitlement filesEntitlement, Path tempDir) {
         this.writePaths = writePaths.toArray(new String[0]);
     }
 
-    public static FileAccessTree of(FilesEntitlement filesEntitlement, Path tempDir) {
-        return new FileAccessTree(filesEntitlement, tempDir);
+    public static FileAccessTree of(FilesEntitlement filesEntitlement, PathLookup pathLookup) {
+        return new FileAccessTree(filesEntitlement, pathLookup);
     }
 
     boolean canRead(Path path) {