Merge branch 'main' into fix-filter-ccs

Author: smalyshev

docs/changelog/126035.yaml

@@ -0,0 +1,5 @@
+pr: 126035
+summary: Fix top level knn search with scroll
+area: Vector Search
+type: bug
+issues: []

docs/changelog/126629.yaml

@@ -27,7 +27,7 @@ Many APIs which support date values also support [date math](/reference/elastics
 
 ## Custom date formats [custom-date-formats]
 
-Completely customizable date formats are supported. The syntax for these is explained in [DateTimeFormatter docs](https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/time/format/DateTimeFormatter.md).
+Completely customizable date formats are supported. The syntax for these is explained in [DateTimeFormatter docs](https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/time/format/DateTimeFormatter.html).
 
 Note that while the built-in formats for week dates use the ISO definition of weekyears, custom formatters using the `Y`, `W`, or `w` field specifiers use the JDK locale definition of weekyears. This can result in different values between the built-in formats and custom formats for week dates.
 

docs/reference/elasticsearch/mapping-reference/mapping-date-format.md

@@ -6,7 +6,7 @@
 * [`ST_X`](../../functions-operators/spatial-functions.md#esql-st_x)
 * [`ST_Y`](../../functions-operators/spatial-functions.md#esql-st_y)
 * [preview] [`ST_ENVELOPE`](../../functions-operators/spatial-functions.md#esql-st_envelope)
-* [preview] [`ST_XMAX`](../../functions-operators/spatial-functions.md#esql-st_xmax)
-* [preview] [`ST_XMIN`](../../functions-operators/spatial-functions.md#esql-st_xmin)
-* [preview] [`ST_YMAX`](../../functions-operators/spatial-functions.md#esql-st_ymax)
-* [preview] [`ST_YMIN`](../../functions-operators/spatial-functions.md#esql-st_ymin)
+  * [preview] [`ST_XMAX`](../../functions-operators/spatial-functions.md#esql-st_xmax)
+  * [preview] [`ST_XMIN`](../../functions-operators/spatial-functions.md#esql-st_xmin)
+  * [preview] [`ST_YMAX`](../../functions-operators/spatial-functions.md#esql-st_ymax)
+  * [preview] [`ST_YMIN`](../../functions-operators/spatial-functions.md#esql-st_ymin)

docs/reference/query-languages/esql/_snippets/functions/layout/st_xmax.md

@@ -15,9 +15,13 @@
 import com.sun.tools.attach.VirtualMachine;
 
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.entitlement.initialization.EntitlementInitialization;
+import org.elasticsearch.entitlement.runtime.policy.PathLookup;
+import org.elasticsearch.entitlement.runtime.policy.PathLookupImpl;
 import org.elasticsearch.entitlement.runtime.policy.Policy;
+import org.elasticsearch.entitlement.runtime.policy.PolicyManager;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 
@@ -36,36 +40,16 @@ public class EntitlementBootstrap {
     public record BootstrapArgs(
         @Nullable Policy serverPolicyPatch,
         Map<String, Policy> pluginPolicies,
-        Function<Class<?>, String> pluginResolver,
-        Function<String, Stream<String>> settingResolver,
-        Path[] dataDirs,
-        Path[] sharedRepoDirs,
-        Path configDir,
-        Path libDir,
-        Path modulesDir,
-        Path pluginsDir,
+        Function<Class<?>, PolicyManager.PolicyScope> scopeResolver,
+        PathLookup pathLookup,
         Map<String, Path> sourcePaths,
-        Path logsDir,
-        Path tempDir,
-        Path pidFile,
         Set<Class<?>> suppressFailureLogClasses
     ) {
         public BootstrapArgs {
             requireNonNull(pluginPolicies);
-            requireNonNull(pluginResolver);
-            requireNonNull(settingResolver);
-            requireNonNull(dataDirs);
-            if (dataDirs.length == 0) {
-                throw new IllegalArgumentException("must provide at least one data directory");
-            }
-            requireNonNull(sharedRepoDirs);
-            requireNonNull(configDir);
-            requireNonNull(libDir);
-            requireNonNull(modulesDir);
-            requireNonNull(pluginsDir);
+            requireNonNull(scopeResolver);
+            requireNonNull(pathLookup);
             requireNonNull(sourcePaths);
-            requireNonNull(logsDir);
-            requireNonNull(tempDir);
             requireNonNull(suppressFailureLogClasses);
         }
     }
@@ -82,7 +66,7 @@ public static BootstrapArgs bootstrapArgs() {
      *
      * @param serverPolicyPatch a policy with additional entitlements to patch the embedded server layer policy
      * @param pluginPolicies a map holding policies for plugins (and modules), by plugin (or module) name.
-     * @param pluginResolver a functor to map a Java Class to the plugin it belongs to (the plugin name).
+     * @param scopeResolver a functor to map a Java Class to the component and module it belongs to.
      * @param settingResolver a functor to resolve a setting name pattern for one or more Elasticsearch settings.
      * @param dataDirs       data directories for Elasticsearch
      * @param sharedRepoDirs shared repository directories for Elasticsearch
@@ -99,7 +83,7 @@ public static BootstrapArgs bootstrapArgs() {
     public static void bootstrap(
         Policy serverPolicyPatch,
         Map<String, Policy> pluginPolicies,
-        Function<Class<?>, String> pluginResolver,
+        Function<Class<?>, PolicyManager.PolicyScope> scopeResolver,
         Function<String, Stream<String>> settingResolver,
         Path[] dataDirs,
         Path[] sharedRepoDirs,
@@ -120,24 +104,35 @@ public static void bootstrap(
         EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(
             serverPolicyPatch,
             pluginPolicies,
-            pluginResolver,
-            settingResolver,
-            dataDirs,
-            sharedRepoDirs,
-            configDir,
-            libDir,
-            modulesDir,
-            pluginsDir,
+            scopeResolver,
+            new PathLookupImpl(
+                getUserHome(),
+                configDir,
+                dataDirs,
+                sharedRepoDirs,
+                libDir,
+                modulesDir,
+                pluginsDir,
+                logsDir,
+                tempDir,
+                pidFile,
+                settingResolver
+            ),
             sourcePaths,
-            logsDir,
-            tempDir,
-            pidFile,
             suppressFailureLogClasses
         );
         exportInitializationToAgent();
         loadAgent(findAgentJar());
     }
 
+    private static Path getUserHome() {
+        String userHome = System.getProperty("user.home");
+        if (userHome == null) {
+            throw new IllegalStateException("user.home system property is required");
+        }
+        return PathUtils.get(userHome);
+    }
+
     @SuppressForbidden(reason = "The VirtualMachine API is the only way to attach a java agent dynamically")
     private static void loadAgent(String agentPath) {
         try {

docs/reference/query-languages/esql/_snippets/functions/layout/st_xmin.md

@@ -58,9 +58,9 @@
 import java.nio.file.attribute.FileAttribute;
 import java.nio.file.spi.FileSystemProvider;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -70,10 +70,14 @@
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 
+import static org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir.CONFIG;
+import static org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir.DATA;
+import static org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir.LIB;
+import static org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir.LOGS;
+import static org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir.MODULES;
+import static org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir.PLUGINS;
+import static org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir.SHARED_REPO;
 import static org.elasticsearch.entitlement.runtime.policy.Platform.LINUX;
-import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.BaseDir.CONFIG;
-import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.BaseDir.DATA;
-import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.BaseDir.SHARED_REPO;
 import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ;
 import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ_WRITE;
 
@@ -86,7 +90,6 @@
  */
 public class EntitlementInitialization {
 
-    private static final String AGENTS_PACKAGE_NAME = "co.elastic.apm.agent";
     private static final Module ENTITLEMENTS_MODULE = PolicyManager.class.getModule();
 
     private static ElasticsearchEntitlementChecker manager;
@@ -165,27 +168,20 @@ private static Class<?>[] findClassesToRetransform(Class<?>[] loadedClasses, Set
     private static PolicyManager createPolicyManager() {
         EntitlementBootstrap.BootstrapArgs bootstrapArgs = EntitlementBootstrap.bootstrapArgs();
         Map<String, Policy> pluginPolicies = bootstrapArgs.pluginPolicies();
-        var pathLookup = new PathLookup(
-            getUserHome(),
-            bootstrapArgs.configDir(),
-            bootstrapArgs.dataDirs(),
-            bootstrapArgs.sharedRepoDirs(),
-            bootstrapArgs.tempDir(),
-            bootstrapArgs.settingResolver()
-        );
+        PathLookup pathLookup = bootstrapArgs.pathLookup();
 
         List<Scope> serverScopes = new ArrayList<>();
         List<FileData> serverModuleFileDatas = new ArrayList<>();
         Collections.addAll(
             serverModuleFileDatas,
             // Base ES directories
-            FileData.ofPath(bootstrapArgs.pluginsDir(), READ),
-            FileData.ofPath(bootstrapArgs.modulesDir(), READ),
-            FileData.ofPath(bootstrapArgs.configDir(), READ),
-            FileData.ofPath(bootstrapArgs.logsDir(), READ_WRITE),
-            FileData.ofPath(bootstrapArgs.libDir(), READ),
-            FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE),
-            FileData.ofRelativePath(Path.of(""), SHARED_REPO, READ_WRITE),
+            FileData.ofBaseDirPath(PLUGINS, READ),
+            FileData.ofBaseDirPath(MODULES, READ),
+            FileData.ofBaseDirPath(CONFIG, READ),
+            FileData.ofBaseDirPath(LOGS, READ_WRITE),
+            FileData.ofBaseDirPath(LIB, READ),
+            FileData.ofBaseDirPath(DATA, READ_WRITE),
+            FileData.ofBaseDirPath(SHARED_REPO, READ_WRITE),
             // exclusive settings file
             FileData.ofRelativePath(Path.of("operator/settings.json"), CONFIG, READ_WRITE).withExclusive(true),
             // OS release on Linux
@@ -206,8 +202,8 @@ private static PolicyManager createPolicyManager() {
             FileData.ofPath(Path.of("/proc/self/mountinfo"), READ).withPlatform(LINUX),
             FileData.ofPath(Path.of("/proc/diskstats"), READ).withPlatform(LINUX)
         );
-        if (bootstrapArgs.pidFile() != null) {
-            serverModuleFileDatas.add(FileData.ofPath(bootstrapArgs.pidFile(), READ_WRITE));
+        if (pathLookup.pidFile() != null) {
+            serverModuleFileDatas.add(FileData.ofPath(pathLookup.pidFile(), READ_WRITE));
         }
 
         Collections.addAll(
@@ -219,8 +215,8 @@ private static PolicyManager createPolicyManager() {
                     new FilesEntitlement(
                         List.of(
                             // TODO: what in es.base is accessing shared repo?
-                            FileData.ofRelativePath(Path.of(""), SHARED_REPO, READ_WRITE),
-                            FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE)
+                            FileData.ofBaseDirPath(SHARED_REPO, READ_WRITE),
+                            FileData.ofBaseDirPath(DATA, READ_WRITE)
                         )
                     )
                 )
@@ -245,25 +241,17 @@ private static PolicyManager createPolicyManager() {
                 List.of(
                     new LoadNativeLibrariesEntitlement(),
                     new ManageThreadsEntitlement(),
-                    new FilesEntitlement(
-                        List.of(FileData.ofPath(bootstrapArgs.configDir(), READ), FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE))
-                    )
+                    new FilesEntitlement(List.of(FileData.ofBaseDirPath(CONFIG, READ), FileData.ofBaseDirPath(DATA, READ_WRITE)))
                 )
             ),
-            new Scope(
-                "org.apache.lucene.misc",
-                List.of(new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE))))
-            ),
+            new Scope("org.apache.lucene.misc", List.of(new FilesEntitlement(List.of(FileData.ofBaseDirPath(DATA, READ_WRITE))))),
             new Scope(
                 "org.apache.logging.log4j.core",
-                List.of(new ManageThreadsEntitlement(), new FilesEntitlement(List.of(FileData.ofPath(bootstrapArgs.logsDir(), READ_WRITE))))
+                List.of(new ManageThreadsEntitlement(), new FilesEntitlement(List.of(FileData.ofBaseDirPath(LOGS, READ_WRITE))))
             ),
             new Scope(
                 "org.elasticsearch.nativeaccess",
-                List.of(
-                    new LoadNativeLibrariesEntitlement(),
-                    new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE)))
-                )
+                List.of(new LoadNativeLibrariesEntitlement(), new FilesEntitlement(List.of(FileData.ofBaseDirPath(DATA, READ_WRITE))))
             )
         );
 
@@ -288,7 +276,7 @@ private static PolicyManager createPolicyManager() {
                 new Scope(
                     "org.bouncycastle.fips.core",
                     // read to lib dir is required for checksum validation
-                    List.of(new FilesEntitlement(List.of(FileData.ofPath(bootstrapArgs.libDir(), READ))), new ManageThreadsEntitlement())
+                    List.of(new FilesEntitlement(List.of(FileData.ofBaseDirPath(LIB, READ))), new ManageThreadsEntitlement())
                 )
             );
         }
@@ -312,50 +300,35 @@ private static PolicyManager createPolicyManager() {
             new LoadNativeLibrariesEntitlement(),
             new FilesEntitlement(
                 List.of(
-                    FileData.ofPath(bootstrapArgs.logsDir(), READ_WRITE),
+                    FileData.ofBaseDirPath(LOGS, READ_WRITE),
                     FileData.ofPath(Path.of("/proc/meminfo"), READ),
                     FileData.ofPath(Path.of("/sys/fs/cgroup/"), READ)
                 )
             )
         );
 
-        validateFilesEntitlements(
-            pluginPolicies,
-            pathLookup,
-            bootstrapArgs.configDir(),
-            bootstrapArgs.pluginsDir(),
-            bootstrapArgs.modulesDir(),
-            bootstrapArgs.libDir()
-        );
+        validateFilesEntitlements(pluginPolicies, pathLookup);
 
         return new PolicyManager(
             serverPolicy,
             agentEntitlements,
             pluginPolicies,
-            EntitlementBootstrap.bootstrapArgs().pluginResolver(),
+            EntitlementBootstrap.bootstrapArgs().scopeResolver(),
             EntitlementBootstrap.bootstrapArgs().sourcePaths(),
-            AGENTS_PACKAGE_NAME,
             ENTITLEMENTS_MODULE,
             pathLookup,
             bootstrapArgs.suppressFailureLogClasses()
         );
     }
 
-    private static Set<Path> pathSet(Path... paths) {
-        return Arrays.stream(paths).map(x -> x.toAbsolutePath().normalize()).collect(Collectors.toUnmodifiableSet());
-    }
-
     // package visible for tests
-    static void validateFilesEntitlements(
-        Map<String, Policy> pluginPolicies,
-        PathLookup pathLookup,
-        Path configDir,
-        Path pluginsDir,
-        Path modulesDir,
-        Path libDir
-    ) {
-        var readAccessForbidden = pathSet(pluginsDir, modulesDir, libDir);
-        var writeAccessForbidden = pathSet(configDir);
+    static void validateFilesEntitlements(Map<String, Policy> pluginPolicies, PathLookup pathLookup) {
+        Set<Path> readAccessForbidden = new HashSet<>();
+        pathLookup.getBaseDirPaths(PLUGINS).forEach(p -> readAccessForbidden.add(p.toAbsolutePath().normalize()));
+        pathLookup.getBaseDirPaths(MODULES).forEach(p -> readAccessForbidden.add(p.toAbsolutePath().normalize()));
+        pathLookup.getBaseDirPaths(LIB).forEach(p -> readAccessForbidden.add(p.toAbsolutePath().normalize()));
+        Set<Path> writeAccessForbidden = new HashSet<>();
+        pathLookup.getBaseDirPaths(CONFIG).forEach(p -> writeAccessForbidden.add(p.toAbsolutePath().normalize()));
         for (var pluginPolicy : pluginPolicies.entrySet()) {
             for (var scope : pluginPolicy.getValue().scopes()) {
                 var filesEntitlement = scope.entitlements()