Merge branch 'main' into entitlements/nio-path

Author: mosche

.github/CODEOWNERS

@@ -49,6 +49,7 @@ distribution/docker/src @elastic/es-delivery
 # Core/Infra
 distribution/tools @elastic/es-core-infra
 libs/core @elastic/es-core-infra
+libs/entitlement @elastic/es-core-infra
 libs/logging @elastic/es-core-infra
 libs/native @elastic/es-core-infra
 libs/plugin-analysis-api @elastic/es-core-infra

docs/changelog/121942.yaml

@@ -0,0 +1,5 @@
+pr: 121942
+summary: Allow partial results in ES|QL
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/122610.yaml

@@ -0,0 +1,5 @@
+pr: 122610
+summary: Canonicalize processor names and types in `IngestStats`
+area: Ingest Node
+type: bug
+issues: []

docs/changelog/122653.yaml

@@ -0,0 +1,6 @@
+pr: 122653
+summary: Knn vector rescoring to sort score docs
+area: Vector Search
+type: bug
+issues:
+ - 119711

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/LoadNativeLibrariesCheckActions.java

@@ -12,15 +12,15 @@
 class LoadNativeLibrariesCheckActions {
     static void runtimeLoad() {
         try {
-            Runtime.getRuntime().load("libSomeLibFile.so");
+            Runtime.getRuntime().load(FileCheckActions.readDir().resolve("libSomeLibFile.so").toString());
         } catch (UnsatisfiedLinkError ignored) {
             // The library does not exist, so we expect to fail loading it
         }
     }
 
     static void systemLoad() {
         try {
-            System.load("libSomeLibFile.so");
+            System.load(FileCheckActions.readDir().resolve("libSomeLibFile.so").toString());
         } catch (UnsatisfiedLinkError ignored) {
             // The library does not exist, so we expect to fail loading it
         }

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/NativeActions.java

@@ -113,7 +113,7 @@ static void memorySegmentReinterpretWithSizeAndCleanup() {
     @EntitlementTest(expectedAccess = PLUGINS)
     static void symbolLookupWithPath() {
         try {
-            SymbolLookup.libraryLookup(Path.of("/foo/bar/libFoo.so"), Arena.ofAuto());
+            SymbolLookup.libraryLookup(FileCheckActions.readDir().resolve("libFoo.so"), Arena.ofAuto());
         } catch (IllegalArgumentException e) {
             // IllegalArgumentException is thrown if path does not point to a valid library (and it does not)
         }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -18,6 +18,7 @@
 import org.elasticsearch.entitlement.instrumentation.MethodKey;
 import org.elasticsearch.entitlement.instrumentation.Transformer;
 import org.elasticsearch.entitlement.runtime.api.ElasticsearchEntitlementChecker;
+import org.elasticsearch.entitlement.runtime.policy.PathLookup;
 import org.elasticsearch.entitlement.runtime.policy.Policy;
 import org.elasticsearch.entitlement.runtime.policy.PolicyManager;
 import org.elasticsearch.entitlement.runtime.policy.Scope;
@@ -130,9 +131,9 @@ private static Class<?>[] findClassesToRetransform(Class<?>[] loadedClasses, Set
     }
 
     private static PolicyManager createPolicyManager() {
-        Map<String, Policy> pluginPolicies = EntitlementBootstrap.bootstrapArgs().pluginPolicies();
-        Path[] dataDirs = EntitlementBootstrap.bootstrapArgs().dataDirs();
-        Path tempDir = EntitlementBootstrap.bootstrapArgs().tempDir();
+        EntitlementBootstrap.BootstrapArgs bootstrapArgs = EntitlementBootstrap.bootstrapArgs();
+        Map<String, Policy> pluginPolicies = bootstrapArgs.pluginPolicies();
+        var pathLookup = new PathLookup(bootstrapArgs.configDir(), bootstrapArgs.dataDirs(), bootstrapArgs.tempDir());
 
         // TODO(ES-10031): Decide what goes in the elasticsearch default policy and extend it
         var serverPolicy = new Policy(
@@ -153,10 +154,10 @@ private static PolicyManager createPolicyManager() {
                         new FilesEntitlement(
                             Stream.concat(
                                 Stream.of(
-                                    new FileData(EntitlementBootstrap.bootstrapArgs().tempDir().toString(), READ_WRITE),
-                                    new FileData(EntitlementBootstrap.bootstrapArgs().configDir().toString(), READ)
+                                    FileData.ofPath(bootstrapArgs.tempDir(), READ_WRITE),
+                                    FileData.ofPath(bootstrapArgs.configDir(), READ)
                                 ),
-                                Arrays.stream(dataDirs).map(d -> new FileData(d.toString(), READ))
+                                Arrays.stream(bootstrapArgs.dataDirs()).map(d -> FileData.ofPath(d, READ))
                             ).toList()
                         )
                     )
@@ -170,8 +171,8 @@ private static PolicyManager createPolicyManager() {
                         new ManageThreadsEntitlement(),
                         new FilesEntitlement(
                             Stream.concat(
-                                Stream.of(new FileData(EntitlementBootstrap.bootstrapArgs().configDir().toString(), READ)),
-                                Arrays.stream(dataDirs).map(d -> new FileData(d.toString(), READ_WRITE))
+                                Stream.of(FileData.ofPath(bootstrapArgs.configDir(), READ)),
+                                Arrays.stream(bootstrapArgs.dataDirs()).map(d -> FileData.ofPath(d, READ_WRITE))
                             ).toList()
                         )
                     )
@@ -181,7 +182,7 @@ private static PolicyManager createPolicyManager() {
                     "org.elasticsearch.nativeaccess",
                     List.of(
                         new LoadNativeLibrariesEntitlement(),
-                        new FilesEntitlement(Arrays.stream(dataDirs).map(d -> new FileData(d.toString(), READ_WRITE)).toList())
+                        new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE)))
                     )
                 )
             )
@@ -197,7 +198,7 @@ private static PolicyManager createPolicyManager() {
             resolver,
             AGENTS_PACKAGE_NAME,
             ENTITLEMENTS_MODULE,
-            tempDir
+            pathLookup
         );
     }
 

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java

@@ -840,7 +840,7 @@ public void checkSelectorProviderInheritedChannel(Class<?> callerClass, Selector
 
     @Override
     public void check$java_lang_Runtime$load(Class<?> callerClass, Runtime that, String filename) {
-        // TODO: check filesystem entitlement READ
+        policyManager.checkFileRead(callerClass, Path.of(filename));
         policyManager.checkLoadingNativeLibraries(callerClass);
     }
 
@@ -851,7 +851,7 @@ public void checkSelectorProviderInheritedChannel(Class<?> callerClass, Selector
 
     @Override
     public void check$java_lang_System$$load(Class<?> callerClass, String filename) {
-        // TODO: check filesystem entitlement READ
+        policyManager.checkFileRead(callerClass, Path.of(filename));
         policyManager.checkLoadingNativeLibraries(callerClass);
     }
 
@@ -935,7 +935,7 @@ public void checkSelectorProviderInheritedChannel(Class<?> callerClass, Selector
 
     @Override
     public void check$java_lang_foreign_SymbolLookup$$libraryLookup(Class<?> callerClass, Path path, Arena arena) {
-        // TODO: check filesystem entitlement READ
+        policyManager.checkFileRead(callerClass, path);
         policyManager.checkLoadingNativeLibraries(callerClass);
     }
 

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java

@@ -20,26 +20,30 @@
 import static org.elasticsearch.core.PathUtils.getDefaultFileSystem;
 
 public final class FileAccessTree {
+
     private static final String FILE_SEPARATOR = getDefaultFileSystem().getSeparator();
 
     private final String[] readPaths;
     private final String[] writePaths;
 
-    private FileAccessTree(FilesEntitlement filesEntitlement, Path tempDir) {
+    private FileAccessTree(FilesEntitlement filesEntitlement, PathLookup pathLookup) {
         List<String> readPaths = new ArrayList<>();
         List<String> writePaths = new ArrayList<>();
         for (FilesEntitlement.FileData fileData : filesEntitlement.filesData()) {
-            var path = normalizePath(Path.of(fileData.path()));
             var mode = fileData.mode();
-            if (mode == FilesEntitlement.Mode.READ_WRITE) {
-                writePaths.add(path);
-            }
-            readPaths.add(path);
+            var paths = fileData.resolvePaths(pathLookup);
+            paths.forEach(path -> {
+                var normalized = normalizePath(path);
+                if (mode == FilesEntitlement.Mode.READ_WRITE) {
+                    writePaths.add(normalized);
+                }
+                readPaths.add(normalized);
+            });
         }
 
         // everything has access to the temp dir
-        readPaths.add(tempDir.toString());
-        writePaths.add(tempDir.toString());
+        readPaths.add(pathLookup.tempDir().toString());
+        writePaths.add(pathLookup.tempDir().toString());
 
         readPaths.sort(String::compareTo);
         writePaths.sort(String::compareTo);
@@ -48,8 +52,8 @@ private FileAccessTree(FilesEntitlement filesEntitlement, Path tempDir) {
         this.writePaths = writePaths.toArray(new String[0]);
     }
 
-    public static FileAccessTree of(FilesEntitlement filesEntitlement, Path tempDir) {
-        return new FileAccessTree(filesEntitlement, tempDir);
+    public static FileAccessTree of(FilesEntitlement filesEntitlement, PathLookup pathLookup) {
+        return new FileAccessTree(filesEntitlement, pathLookup);
     }
 
     boolean canRead(Path path) {

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PathLookup.java

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.runtime.policy;
+
+import java.nio.file.Path;
+
+public record PathLookup(Path configDir, Path[] dataDirs, Path tempDir) {}