Merge remote-tracking branch 'upstream/main' into tsdb-timestamp-sparse-index

Author: jordan-powers

docs/changelog/122860.yaml

@@ -0,0 +1,5 @@
+pr: 122860
+summary: Improved error message when index field type is invalid
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/123272.yaml

@@ -0,0 +1,5 @@
+pr: 123272
+summary: Set Connect Timeout to 5s
+area: Machine Learning
+type: bug
+issues: []

libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java

@@ -39,8 +39,8 @@ public record BootstrapArgs(
         Function<Class<?>, String> pluginResolver,
         Function<String, String> settingResolver,
         Function<String, Stream<String>> settingGlobResolver,
-        Function<String, Path> repoDirResolver,
         Path[] dataDirs,
+        Path[] sharedRepoDirs,
         Path configDir,
         Path libDir,
         Path logsDir,
@@ -51,11 +51,11 @@ public record BootstrapArgs(
             requireNonNull(pluginResolver);
             requireNonNull(settingResolver);
             requireNonNull(settingGlobResolver);
-            requireNonNull(repoDirResolver);
             requireNonNull(dataDirs);
             if (dataDirs.length == 0) {
                 throw new IllegalArgumentException("must provide at least one data directory");
             }
+            requireNonNull(sharedRepoDirs);
             requireNonNull(configDir);
             requireNonNull(libDir);
             requireNonNull(logsDir);
@@ -77,8 +77,8 @@ public static BootstrapArgs bootstrapArgs() {
      * @param pluginResolver a functor to map a Java Class to the plugin it belongs to (the plugin name).
      * @param settingResolver a functor to resolve the value of an Elasticsearch setting.
      * @param settingGlobResolver a functor to resolve a glob expression for one or more Elasticsearch settings.
-     * @param repoDirResolver a functor to map a repository location to its Elasticsearch path.
      * @param dataDirs       data directories for Elasticsearch
+     * @param sharedRepoDirs       shared repository directories for Elasticsearch
      * @param configDir      the config directory for Elasticsearch
      * @param libDir         the lib directory for Elasticsearch
      * @param tempDir        the temp directory for Elasticsearch
@@ -89,8 +89,8 @@ public static void bootstrap(
         Function<Class<?>, String> pluginResolver,
         Function<String, String> settingResolver,
         Function<String, Stream<String>> settingGlobResolver,
-        Function<String, Path> repoDirResolver,
         Path[] dataDirs,
+        Path[] sharedRepoDirs,
         Path configDir,
         Path libDir,
         Path logsDir,
@@ -105,8 +105,8 @@ public static void bootstrap(
             pluginResolver,
             settingResolver,
             settingGlobResolver,
-            repoDirResolver,
             dataDirs,
+            sharedRepoDirs,
             configDir,
             libDir,
             logsDir,

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -63,6 +63,8 @@
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 
+import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.BaseDir.DATA;
+import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.BaseDir.SHARED_REPO;
 import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ;
 import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ_WRITE;
 
@@ -138,6 +140,7 @@ private static PolicyManager createPolicyManager() {
             getUserHome(),
             bootstrapArgs.configDir(),
             bootstrapArgs.dataDirs(),
+            bootstrapArgs.sharedRepoDirs(),
             bootstrapArgs.tempDir(),
             bootstrapArgs.settingResolver(),
             bootstrapArgs.settingGlobResolver()
@@ -152,8 +155,8 @@ private static PolicyManager createPolicyManager() {
                     new CreateClassLoaderEntitlement(),
                     new FilesEntitlement(
                         List.of(
-                            FileData.ofPath(bootstrapArgs.repoDirResolver().apply(""), READ_WRITE),
-                            FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE)
+                            FileData.ofRelativePath(Path.of(""), SHARED_REPO, READ_WRITE),
+                            FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE)
                         )
                     )
                 )
@@ -175,8 +178,8 @@ private static PolicyManager createPolicyManager() {
                             FileData.ofPath(bootstrapArgs.tempDir(), READ_WRITE),
                             FileData.ofPath(bootstrapArgs.configDir(), READ),
                             FileData.ofPath(bootstrapArgs.logsDir(), READ_WRITE),
-                            FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE),
-                            FileData.ofPath(bootstrapArgs.repoDirResolver().apply(""), READ_WRITE),
+                            FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE),
+                            FileData.ofRelativePath(Path.of(""), SHARED_REPO, READ_WRITE),
 
                             // OS release on Linux
                             FileData.ofPath(Path.of("/etc/os-release"), READ),
@@ -210,21 +213,21 @@ private static PolicyManager createPolicyManager() {
                         List.of(
                             FileData.ofPath(bootstrapArgs.configDir(), READ),
                             FileData.ofPath(bootstrapArgs.tempDir(), READ),
-                            FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE)
+                            FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE)
                         )
                     )
                 )
             ),
             new Scope(
                 "org.apache.lucene.misc",
-                List.of(new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE))))
+                List.of(new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE))))
             ),
             new Scope("org.apache.logging.log4j.core", List.of(new ManageThreadsEntitlement())),
             new Scope(
                 "org.elasticsearch.nativeaccess",
                 List.of(
                     new LoadNativeLibrariesEntitlement(),
-                    new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE)))
+                    new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE)))
                 )
             )
         );

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java

@@ -14,6 +14,7 @@
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Objects;
 
@@ -46,8 +47,8 @@ private FileAccessTree(FilesEntitlement filesEntitlement, PathLookup pathLookup)
         readPaths.add(tempDir);
         writePaths.add(tempDir);
 
-        readPaths.sort(String::compareTo);
-        writePaths.sort(String::compareTo);
+        readPaths.sort(PATH_ORDER);
+        writePaths.sort(PATH_ORDER);
 
         this.readPaths = pruneSortedPaths(readPaths).toArray(new String[0]);
         this.writePaths = pruneSortedPaths(writePaths).toArray(new String[0]);
@@ -60,7 +61,7 @@ private static List<String> pruneSortedPaths(List<String> paths) {
             prunedReadPaths.add(currentPath);
             for (int i = 1; i < paths.size(); ++i) {
                 String nextPath = paths.get(i);
-                if (nextPath.startsWith(currentPath) == false) {
+                if (isParent(currentPath, nextPath) == false) {
                     prunedReadPaths.add(nextPath);
                     currentPath = nextPath;
                 }
@@ -88,21 +89,28 @@ static String normalizePath(Path path) {
         // Note that toAbsolutePath produces paths separated by the default file separator,
         // so on Windows, if the given path uses forward slashes, this consistently
         // converts it to backslashes.
-        return path.toAbsolutePath().normalize().toString();
+        String result = path.toAbsolutePath().normalize().toString();
+        while (result.endsWith(FILE_SEPARATOR)) {
+            result = result.substring(0, result.length() - FILE_SEPARATOR.length());
+        }
+        return result;
     }
 
     private static boolean checkPath(String path, String[] paths) {
         if (paths.length == 0) {
             return false;
         }
-        int ndx = Arrays.binarySearch(paths, path);
+        int ndx = Arrays.binarySearch(paths, path, PATH_ORDER);
         if (ndx < -1) {
-            String maybeParent = paths[-ndx - 2];
-            return path.startsWith(maybeParent) && path.startsWith(FILE_SEPARATOR, maybeParent.length());
+            return isParent(paths[-ndx - 2], path);
         }
         return ndx >= 0;
     }
 
+    private static boolean isParent(String maybeParent, String path) {
+        return path.startsWith(maybeParent) && path.startsWith(FILE_SEPARATOR, maybeParent.length());
+    }
+
     @Override
     public boolean equals(Object o) {
         if (o == null || getClass() != o.getClass()) return false;
@@ -114,4 +122,30 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(Arrays.hashCode(readPaths), Arrays.hashCode(writePaths));
     }
+
+    /**
+     * For our lexicographic sort trick to work correctly, we must have path separators sort before
+     * any other character so that files in a directory appear immediately after that directory.
+     * For example, we require [/a, /a/b, /a.xml] rather than the natural order [/a, /a.xml, /a/b].
+     */
+    private static final Comparator<String> PATH_ORDER = (s1, s2) -> {
+        Path p1 = Path.of(s1);
+        Path p2 = Path.of(s2);
+        var i1 = p1.iterator();
+        var i2 = p2.iterator();
+        while (i1.hasNext() && i2.hasNext()) {
+            int cmp = i1.next().compareTo(i2.next());
+            if (cmp != 0) {
+                return cmp;
+            }
+        }
+        if (i1.hasNext()) {
+            return 1;
+        } else if (i2.hasNext()) {
+            return -1;
+        } else {
+            assert p1.equals(p2);
+            return 0;
+        }
+    };
 }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PathLookup.java

@@ -17,6 +17,7 @@ public record PathLookup(
     Path homeDir,
     Path configDir,
     Path[] dataDirs,
+    Path[] sharedRepoDirs,
     Path tempDir,
     Function<String, String> settingResolver,
     Function<String, Stream<String>> settingGlobResolver

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlement.java

@@ -21,6 +21,8 @@
 import java.util.Objects;
 import java.util.stream.Stream;
 
+import static java.lang.Character.isLetter;
+
 /**
  * Describes a file entitlement with a path and mode.
  */
@@ -36,6 +38,7 @@ public enum Mode {
     public enum BaseDir {
         CONFIG,
         DATA,
+        SHARED_REPO,
         HOME
     }
 
@@ -60,6 +63,51 @@ static FileData ofPathSetting(String setting, Mode mode) {
         static FileData ofRelativePathSetting(String setting, BaseDir baseDir, Mode mode) {
             return new RelativePathSettingFileData(setting, baseDir, mode);
         }
+
+        /**
+         * Tests if a path is absolute or relative, taking into consideration both Unix and Windows conventions.
+         * Note that this leads to a conflict, resolved in favor of Unix rules: `/foo` can be either a Unix absolute path, or a Windows
+         * relative path with "wrong" directory separator (using non-canonical slash in Windows).
+         */
+        static boolean isAbsolutePath(String path) {
+            if (path.isEmpty()) {
+                return false;
+            }
+            if (path.charAt(0) == '/') {
+                // Unix/BSD absolute
+                return true;
+            }
+
+            return isWindowsAbsolutePath(path);
+        }
+
+        private static boolean isSlash(char c) {
+            return (c == '\\') || (c == '/');
+        }
+
+        private static boolean isWindowsAbsolutePath(String input) {
+            // if a prefix is present, we expected (long) UNC or (long) absolute
+            if (input.startsWith("\\\\?\\")) {
+                return true;
+            }
+
+            if (input.length() > 1) {
+                char c0 = input.charAt(0);
+                char c1 = input.charAt(1);
+                char c = 0;
+                int next = 2;
+                if (isSlash(c0) && isSlash(c1)) {
+                    // Two slashes or more: UNC
+                    return true;
+                }
+                if (isLetter(c0) && c1 == ':') {
+                    // A drive: absolute
+                    return true;
+                }
+            }
+            // Otherwise relative
+            return false;
+        }
     }
 
     private sealed interface RelativeFileData extends FileData {
@@ -75,14 +123,9 @@ default Stream<Path> resolvePaths(PathLookup pathLookup) {
                 case CONFIG:
                     return relativePaths.map(relativePath -> pathLookup.configDir().resolve(relativePath));
                 case DATA:
-                    // multiple data dirs are a pain...we need the combination of relative paths and data dirs
-                    List<Path> paths = new ArrayList<>();
-                    for (var relativePath : relativePaths.toList()) {
-                        for (var dataDir : pathLookup.dataDirs()) {
-                            paths.add(dataDir.resolve(relativePath));
-                        }
-                    }
-                    return paths.stream();
+                    return relativePathsCombination(pathLookup.dataDirs(), relativePaths);
+                case SHARED_REPO:
+                    return relativePathsCombination(pathLookup.sharedRepoDirs(), relativePaths);
                 case HOME:
                     return relativePaths.map(relativePath -> pathLookup.homeDir().resolve(relativePath));
                 default:
@@ -91,6 +134,17 @@ default Stream<Path> resolvePaths(PathLookup pathLookup) {
         }
     }
 
+    private static Stream<Path> relativePathsCombination(Path[] baseDirs, Stream<Path> relativePaths) {
+        // multiple base dirs are a pain...we need the combination of the base dirs and relative paths
+        List<Path> paths = new ArrayList<>();
+        for (var relativePath : relativePaths.toList()) {
+            for (var dataDir : baseDirs) {
+                paths.add(dataDir.resolve(relativePath));
+            }
+        }
+        return paths.stream();
+    }
+
     private record AbsolutePathFileData(Path path, Mode mode) implements FileData {
         @Override
         public Stream<Path> resolvePaths(PathLookup pathLookup) {
@@ -142,6 +196,7 @@ private static BaseDir parseBaseDir(String baseDir) {
             case "config" -> BaseDir.CONFIG;
             case "data" -> BaseDir.DATA;
             case "home" -> BaseDir.HOME;
+            // NOTE: shared_repo is _not_ accessible to policy files, only internally
             default -> throw new PolicyValidationException(
                 "invalid relative directory: " + baseDir + ", valid values: [config, data, home]"
             );
@@ -190,17 +245,15 @@ public static FilesEntitlement build(List<Object> paths) {
                     throw new PolicyValidationException("files entitlement with a 'relative_path' must specify 'relative_to'");
                 }
 
-                Path relativePath = Path.of(relativePathAsString);
-                if (relativePath.isAbsolute()) {
+                if (FileData.isAbsolutePath(relativePathAsString)) {
                     throw new PolicyValidationException("'relative_path' [" + relativePathAsString + "] must be relative");
                 }
-                filesData.add(FileData.ofRelativePath(relativePath, baseDir, mode));
+                filesData.add(FileData.ofRelativePath(Path.of(relativePathAsString), baseDir, mode));
             } else if (pathAsString != null) {
-                Path path = Path.of(pathAsString);
-                if (path.isAbsolute() == false) {
+                if (FileData.isAbsolutePath(pathAsString) == false) {
                     throw new PolicyValidationException("'path' [" + pathAsString + "] must be absolute");
                 }
-                filesData.add(FileData.ofPath(path, mode));
+                filesData.add(FileData.ofPath(Path.of(pathAsString), mode));
             } else if (pathSetting != null) {
                 filesData.add(FileData.ofPathSetting(pathSetting, mode));
             } else if (relativePathSetting != null) {