Merge main into multi-project

Author: tvernum

docs/changelog/122860.yaml

@@ -0,0 +1,5 @@
+pr: 122860
+summary: Improved error message when index field type is invalid
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/123079.yaml

@@ -0,0 +1,5 @@
+pr: 123079
+summary: Register `IngestGeoIpMetadata` as a NamedXContent
+area: Ingest Node
+type: bug
+issues: []

docs/changelog/123197.yaml

@@ -0,0 +1,5 @@
+pr: 123197
+summary: Fix early termination in `LuceneSourceOperator`
+area: ES|QL
+type: bug
+issues: []

docs/changelog/123272.yaml

@@ -0,0 +1,5 @@
+pr: 123272
+summary: Set Connect Timeout to 5s
+area: Machine Learning
+type: bug
+issues: []

libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java

@@ -39,8 +39,8 @@ public record BootstrapArgs(
         Function<Class<?>, String> pluginResolver,
         Function<String, String> settingResolver,
         Function<String, Stream<String>> settingGlobResolver,
-        Function<String, Path> repoDirResolver,
         Path[] dataDirs,
+        Path[] sharedRepoDirs,
         Path configDir,
         Path libDir,
         Path logsDir,
@@ -51,11 +51,11 @@ public record BootstrapArgs(
             requireNonNull(pluginResolver);
             requireNonNull(settingResolver);
             requireNonNull(settingGlobResolver);
-            requireNonNull(repoDirResolver);
             requireNonNull(dataDirs);
             if (dataDirs.length == 0) {
                 throw new IllegalArgumentException("must provide at least one data directory");
             }
+            requireNonNull(sharedRepoDirs);
             requireNonNull(configDir);
             requireNonNull(libDir);
             requireNonNull(logsDir);
@@ -77,8 +77,8 @@ public static BootstrapArgs bootstrapArgs() {
      * @param pluginResolver a functor to map a Java Class to the plugin it belongs to (the plugin name).
      * @param settingResolver a functor to resolve the value of an Elasticsearch setting.
      * @param settingGlobResolver a functor to resolve a glob expression for one or more Elasticsearch settings.
-     * @param repoDirResolver a functor to map a repository location to its Elasticsearch path.
      * @param dataDirs       data directories for Elasticsearch
+     * @param sharedRepoDirs       shared repository directories for Elasticsearch
      * @param configDir      the config directory for Elasticsearch
      * @param libDir         the lib directory for Elasticsearch
      * @param tempDir        the temp directory for Elasticsearch
@@ -89,8 +89,8 @@ public static void bootstrap(
         Function<Class<?>, String> pluginResolver,
         Function<String, String> settingResolver,
         Function<String, Stream<String>> settingGlobResolver,
-        Function<String, Path> repoDirResolver,
         Path[] dataDirs,
+        Path[] sharedRepoDirs,
         Path configDir,
         Path libDir,
         Path logsDir,
@@ -105,8 +105,8 @@ public static void bootstrap(
             pluginResolver,
             settingResolver,
             settingGlobResolver,
-            repoDirResolver,
             dataDirs,
+            sharedRepoDirs,
             configDir,
             libDir,
             logsDir,

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -63,8 +63,11 @@
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 
+import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.BaseDir.DATA;
+import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.BaseDir.SHARED_REPO;
 import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ;
 import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ_WRITE;
+import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Platform.LINUX;
 
 /**
  * Called by the agent during {@code agentmain} to configure the entitlement system,
@@ -138,6 +141,7 @@ private static PolicyManager createPolicyManager() {
             getUserHome(),
             bootstrapArgs.configDir(),
             bootstrapArgs.dataDirs(),
+            bootstrapArgs.sharedRepoDirs(),
             bootstrapArgs.tempDir(),
             bootstrapArgs.settingResolver(),
             bootstrapArgs.settingGlobResolver()
@@ -152,8 +156,8 @@ private static PolicyManager createPolicyManager() {
                     new CreateClassLoaderEntitlement(),
                     new FilesEntitlement(
                         List.of(
-                            FileData.ofPath(bootstrapArgs.repoDirResolver().apply(""), READ_WRITE),
-                            FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE)
+                            FileData.ofRelativePath(Path.of(""), SHARED_REPO, READ_WRITE),
+                            FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE)
                         )
                     )
                 )
@@ -175,26 +179,26 @@ private static PolicyManager createPolicyManager() {
                             FileData.ofPath(bootstrapArgs.tempDir(), READ_WRITE),
                             FileData.ofPath(bootstrapArgs.configDir(), READ),
                             FileData.ofPath(bootstrapArgs.logsDir(), READ_WRITE),
-                            FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE),
-                            FileData.ofPath(bootstrapArgs.repoDirResolver().apply(""), READ_WRITE),
+                            FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE),
+                            FileData.ofRelativePath(Path.of(""), SHARED_REPO, READ_WRITE),
 
                             // OS release on Linux
-                            FileData.ofPath(Path.of("/etc/os-release"), READ),
-                            FileData.ofPath(Path.of("/etc/system-release"), READ),
-                            FileData.ofPath(Path.of("/usr/lib/os-release"), READ),
+                            FileData.ofPath(Path.of("/etc/os-release"), READ).withPlatform(LINUX),
+                            FileData.ofPath(Path.of("/etc/system-release"), READ).withPlatform(LINUX),
+                            FileData.ofPath(Path.of("/usr/lib/os-release"), READ).withPlatform(LINUX),
                             // read max virtual memory areas
-                            FileData.ofPath(Path.of("/proc/sys/vm/max_map_count"), READ),
-                            FileData.ofPath(Path.of("/proc/meminfo"), READ),
+                            FileData.ofPath(Path.of("/proc/sys/vm/max_map_count"), READ).withPlatform(LINUX),
+                            FileData.ofPath(Path.of("/proc/meminfo"), READ).withPlatform(LINUX),
                             // load averages on Linux
-                            FileData.ofPath(Path.of("/proc/loadavg"), READ),
+                            FileData.ofPath(Path.of("/proc/loadavg"), READ).withPlatform(LINUX),
                             // control group stats on Linux. cgroup v2 stats are in an unpredicable
                             // location under `/sys/fs/cgroup`, so unfortunately we have to allow
                             // read access to the entire directory hierarchy.
-                            FileData.ofPath(Path.of("/proc/self/cgroup"), READ),
-                            FileData.ofPath(Path.of("/sys/fs/cgroup/"), READ),
+                            FileData.ofPath(Path.of("/proc/self/cgroup"), READ).withPlatform(LINUX),
+                            FileData.ofPath(Path.of("/sys/fs/cgroup/"), READ).withPlatform(LINUX),
                             // // io stats on Linux
-                            FileData.ofPath(Path.of("/proc/self/mountinfo"), READ),
-                            FileData.ofPath(Path.of("/proc/diskstats"), READ)
+                            FileData.ofPath(Path.of("/proc/self/mountinfo"), READ).withPlatform(LINUX),
+                            FileData.ofPath(Path.of("/proc/diskstats"), READ).withPlatform(LINUX)
                         )
                     )
                 )
@@ -210,21 +214,21 @@ private static PolicyManager createPolicyManager() {
                         List.of(
                             FileData.ofPath(bootstrapArgs.configDir(), READ),
                             FileData.ofPath(bootstrapArgs.tempDir(), READ),
-                            FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE)
+                            FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE)
                         )
                     )
                 )
             ),
             new Scope(
                 "org.apache.lucene.misc",
-                List.of(new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE))))
+                List.of(new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE))))
             ),
             new Scope("org.apache.logging.log4j.core", List.of(new ManageThreadsEntitlement())),
             new Scope(
                 "org.elasticsearch.nativeaccess",
                 List.of(
                     new LoadNativeLibrariesEntitlement(),
-                    new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE)))
+                    new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE)))
                 )
             )
         );

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java

@@ -14,6 +14,7 @@
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Objects;
 
@@ -30,6 +31,10 @@ private FileAccessTree(FilesEntitlement filesEntitlement, PathLookup pathLookup)
         List<String> readPaths = new ArrayList<>();
         List<String> writePaths = new ArrayList<>();
         for (FilesEntitlement.FileData fileData : filesEntitlement.filesData()) {
+            var platform = fileData.platform();
+            if (platform != null && platform.isCurrent() == false) {
+                continue;
+            }
             var mode = fileData.mode();
             var paths = fileData.resolvePaths(pathLookup);
             paths.forEach(path -> {
@@ -46,8 +51,8 @@ private FileAccessTree(FilesEntitlement filesEntitlement, PathLookup pathLookup)
         readPaths.add(tempDir);
         writePaths.add(tempDir);
 
-        readPaths.sort(String::compareTo);
-        writePaths.sort(String::compareTo);
+        readPaths.sort(PATH_ORDER);
+        writePaths.sort(PATH_ORDER);
 
         this.readPaths = pruneSortedPaths(readPaths).toArray(new String[0]);
         this.writePaths = pruneSortedPaths(writePaths).toArray(new String[0]);
@@ -60,7 +65,7 @@ private static List<String> pruneSortedPaths(List<String> paths) {
             prunedReadPaths.add(currentPath);
             for (int i = 1; i < paths.size(); ++i) {
                 String nextPath = paths.get(i);
-                if (nextPath.startsWith(currentPath) == false) {
+                if (isParent(currentPath, nextPath) == false) {
                     prunedReadPaths.add(nextPath);
                     currentPath = nextPath;
                 }
@@ -88,21 +93,28 @@ static String normalizePath(Path path) {
         // Note that toAbsolutePath produces paths separated by the default file separator,
         // so on Windows, if the given path uses forward slashes, this consistently
         // converts it to backslashes.
-        return path.toAbsolutePath().normalize().toString();
+        String result = path.toAbsolutePath().normalize().toString();
+        while (result.endsWith(FILE_SEPARATOR)) {
+            result = result.substring(0, result.length() - FILE_SEPARATOR.length());
+        }
+        return result;
     }
 
     private static boolean checkPath(String path, String[] paths) {
         if (paths.length == 0) {
             return false;
         }
-        int ndx = Arrays.binarySearch(paths, path);
+        int ndx = Arrays.binarySearch(paths, path, PATH_ORDER);
         if (ndx < -1) {
-            String maybeParent = paths[-ndx - 2];
-            return path.startsWith(maybeParent) && path.startsWith(FILE_SEPARATOR, maybeParent.length());
+            return isParent(paths[-ndx - 2], path);
         }
         return ndx >= 0;
     }
 
+    private static boolean isParent(String maybeParent, String path) {
+        return path.startsWith(maybeParent) && path.startsWith(FILE_SEPARATOR, maybeParent.length());
+    }
+
     @Override
     public boolean equals(Object o) {
         if (o == null || getClass() != o.getClass()) return false;
@@ -114,4 +126,30 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(Arrays.hashCode(readPaths), Arrays.hashCode(writePaths));
     }
+
+    /**
+     * For our lexicographic sort trick to work correctly, we must have path separators sort before
+     * any other character so that files in a directory appear immediately after that directory.
+     * For example, we require [/a, /a/b, /a.xml] rather than the natural order [/a, /a.xml, /a/b].
+     */
+    private static final Comparator<String> PATH_ORDER = (s1, s2) -> {
+        Path p1 = Path.of(s1);
+        Path p2 = Path.of(s2);
+        var i1 = p1.iterator();
+        var i2 = p2.iterator();
+        while (i1.hasNext() && i2.hasNext()) {
+            int cmp = i1.next().compareTo(i2.next());
+            if (cmp != 0) {
+                return cmp;
+            }
+        }
+        if (i1.hasNext()) {
+            return 1;
+        } else if (i2.hasNext()) {
+            return -1;
+        } else {
+            assert p1.equals(p2);
+            return 0;
+        }
+    };
 }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PathLookup.java

@@ -17,6 +17,7 @@ public record PathLookup(
     Path homeDir,
     Path configDir,
     Path[] dataDirs,
+    Path[] sharedRepoDirs,
     Path tempDir,
     Function<String, String> settingResolver,
     Function<String, Stream<String>> settingGlobResolver