Merge branch 'main' into esql_pragma_load_source

Author: nik9000

.buildkite/scripts/dra-update-staging.sh

@@ -37,7 +37,7 @@ for BRANCH in "${BRANCHES[@]}"; do
 
   if [[ "$SHOULD_TRIGGER" == "true" ]]; then
     if [[ "$BRANCH" == "9.0" ]]; then
-      export VERSION_QUALIFIER="beta1"
+      export VERSION_QUALIFIER="rc1"
     fi
     echo "Triggering DRA staging workflow for $BRANCH"
     cat << EOF | buildkite-agent pipeline upload

.buildkite/scripts/dra-workflow.trigger.sh

@@ -8,7 +8,7 @@ source .buildkite/scripts/branches.sh
 
 for BRANCH in "${BRANCHES[@]}"; do
   if [[ "$BRANCH" == "9.0" ]]; then
-    export VERSION_QUALIFIER="beta1"
+    export VERSION_QUALIFIER="rc1"
   fi
 
   INTAKE_PIPELINE_SLUG="elasticsearch-intake"

docs/changelog/122731.yaml

@@ -0,0 +1,5 @@
+pr: 122731
+summary: Fork post-snapshot-delete cleanup off master thread
+area: Snapshot/Restore
+type: bug
+issues: []

docs/changelog/122857.yaml

@@ -0,0 +1,5 @@
+pr: 122857
+summary: Run `TransportGetWatcherSettingsAction` on local node
+area: Watcher
+type: enhancement
+issues: []

libs/core/src/main/java/org/elasticsearch/core/UpdateForV9.java

@@ -24,14 +24,11 @@
     enum Owner {
         CORE_INFRA,
         DATA_MANAGEMENT,
-        DISTRIBUTED_COORDINATION,
         DISTRIBUTED_INDEXING,
         ENTERPRISE_SEARCH,
         MACHINE_LEARNING,
         PROFILING,
         SEARCH_ANALYTICS,
-        SEARCH_FOUNDATIONS,
-        SEARCH_RELEVANCE,
         SECURITY,
     }
 

libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java

@@ -28,6 +28,7 @@
 import java.nio.file.Path;
 import java.util.Map;
 import java.util.function.Function;
+import java.util.stream.Stream;
 
 import static java.util.Objects.requireNonNull;
 
@@ -36,19 +37,24 @@ public class EntitlementBootstrap {
     public record BootstrapArgs(
         Map<String, Policy> pluginPolicies,
         Function<Class<?>, String> pluginResolver,
+        Function<String, String> settingResolver,
+        Function<String, Stream<String>> settingGlobResolver,
         Path[] dataDirs,
         Path configDir,
-        Path tempDir,
-        Path logsDir
+        Path logsDir,
+        Path tempDir
     ) {
         public BootstrapArgs {
             requireNonNull(pluginPolicies);
             requireNonNull(pluginResolver);
+            requireNonNull(settingResolver);
+            requireNonNull(settingGlobResolver);
             requireNonNull(dataDirs);
             if (dataDirs.length == 0) {
                 throw new IllegalArgumentException("must provide at least one data directory");
             }
             requireNonNull(configDir);
+            requireNonNull(logsDir);
             requireNonNull(tempDir);
         }
     }
@@ -73,16 +79,27 @@ public static BootstrapArgs bootstrapArgs() {
     public static void bootstrap(
         Map<String, Policy> pluginPolicies,
         Function<Class<?>, String> pluginResolver,
+        Function<String, String> settingResolver,
+        Function<String, Stream<String>> settingGlobResolver,
         Path[] dataDirs,
         Path configDir,
-        Path tempDir,
-        Path logsDir
+        Path logsDir,
+        Path tempDir
     ) {
         logger.debug("Loading entitlement agent");
         if (EntitlementBootstrap.bootstrapArgs != null) {
             throw new IllegalStateException("plugin data is already set");
         }
-        EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(pluginPolicies, pluginResolver, dataDirs, configDir, tempDir, logsDir);
+        EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(
+            pluginPolicies,
+            pluginResolver,
+            settingResolver,
+            settingGlobResolver,
+            dataDirs,
+            configDir,
+            logsDir,
+            tempDir
+        );
         exportInitializationToAgent();
         loadAgent(findAgentJar());
         selfTest();

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -135,8 +135,14 @@ private static Class<?>[] findClassesToRetransform(Class<?>[] loadedClasses, Set
     private static PolicyManager createPolicyManager() {
         EntitlementBootstrap.BootstrapArgs bootstrapArgs = EntitlementBootstrap.bootstrapArgs();
         Map<String, Policy> pluginPolicies = bootstrapArgs.pluginPolicies();
-        var pathLookup = new PathLookup(getUserHome(), bootstrapArgs.configDir(), bootstrapArgs.dataDirs(), bootstrapArgs.tempDir());
-        Path logsDir = EntitlementBootstrap.bootstrapArgs().logsDir();
+        var pathLookup = new PathLookup(
+            getUserHome(),
+            bootstrapArgs.configDir(),
+            bootstrapArgs.dataDirs(),
+            bootstrapArgs.tempDir(),
+            bootstrapArgs.settingResolver(),
+            bootstrapArgs.settingGlobResolver()
+        );
 
         List<Scope> serverScopes = new ArrayList<>();
         Collections.addAll(

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java

@@ -42,8 +42,9 @@ private FileAccessTree(FilesEntitlement filesEntitlement, PathLookup pathLookup)
         }
 
         // everything has access to the temp dir
-        readPaths.add(pathLookup.tempDir().toString());
-        writePaths.add(pathLookup.tempDir().toString());
+        String tempDir = normalizePath(pathLookup.tempDir());
+        readPaths.add(tempDir);
+        writePaths.add(tempDir);
 
         readPaths.sort(String::compareTo);
         writePaths.sort(String::compareTo);

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PathLookup.java

@@ -10,5 +10,14 @@
 package org.elasticsearch.entitlement.runtime.policy;
 
 import java.nio.file.Path;
+import java.util.function.Function;
+import java.util.stream.Stream;
 
-public record PathLookup(Path homeDir, Path configDir, Path[] dataDirs, Path tempDir) {}
+public record PathLookup(
+    Path homeDir,
+    Path configDir,
+    Path[] dataDirs,
+    Path tempDir,
+    Function<String, String> settingResolver,
+    Function<String, Stream<String>> settingGlobResolver
+) {}

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlement.java

@@ -15,7 +15,6 @@
 
 import java.nio.file.Path;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -53,33 +52,81 @@ static FileData ofPath(Path path, Mode mode) {
         static FileData ofRelativePath(Path relativePath, BaseDir baseDir, Mode mode) {
             return new RelativePathFileData(relativePath, baseDir, mode);
         }
-    }
 
-    private record AbsolutePathFileData(Path path, Mode mode) implements FileData {
-        @Override
-        public Stream<Path> resolvePaths(PathLookup pathLookup) {
-            return Stream.of(path);
+        static FileData ofPathSetting(String setting, Mode mode) {
+            return new PathSettingFileData(setting, mode);
+        }
+
+        static FileData ofRelativePathSetting(String setting, BaseDir baseDir, Mode mode) {
+            return new RelativePathSettingFileData(setting, baseDir, mode);
         }
     }
 
-    private record RelativePathFileData(Path relativePath, BaseDir baseDir, Mode mode) implements FileData {
+    private sealed interface RelativeFileData extends FileData {
+        BaseDir baseDir();
+
+        Stream<Path> resolveRelativePaths(PathLookup pathLookup);
 
         @Override
-        public Stream<Path> resolvePaths(PathLookup pathLookup) {
+        default Stream<Path> resolvePaths(PathLookup pathLookup) {
             Objects.requireNonNull(pathLookup);
-            switch (baseDir) {
+            var relativePaths = resolveRelativePaths(pathLookup);
+            switch (baseDir()) {
                 case CONFIG:
-                    return Stream.of(pathLookup.configDir().resolve(relativePath));
+                    return relativePaths.map(relativePath -> pathLookup.configDir().resolve(relativePath));
                 case DATA:
-                    return Arrays.stream(pathLookup.dataDirs()).map(d -> d.resolve(relativePath));
+                    // multiple data dirs are a pain...we need the combination of relative paths and data dirs
+                    List<Path> paths = new ArrayList<>();
+                    for (var relativePath : relativePaths.toList()) {
+                        for (var dataDir : pathLookup.dataDirs()) {
+                            paths.add(dataDir.resolve(relativePath));
+                        }
+                    }
+                    return paths.stream();
                 case HOME:
-                    return Stream.of(pathLookup.homeDir().resolve(relativePath));
+                    return relativePaths.map(relativePath -> pathLookup.homeDir().resolve(relativePath));
                 default:
                     throw new IllegalArgumentException();
             }
         }
     }
 
+    private record AbsolutePathFileData(Path path, Mode mode) implements FileData {
+        @Override
+        public Stream<Path> resolvePaths(PathLookup pathLookup) {
+            return Stream.of(path);
+        }
+    }
+
+    private record RelativePathFileData(Path relativePath, BaseDir baseDir, Mode mode) implements FileData, RelativeFileData {
+        @Override
+        public Stream<Path> resolveRelativePaths(PathLookup pathLookup) {
+            return Stream.of(relativePath);
+        }
+    }
+
+    private record PathSettingFileData(String setting, Mode mode) implements FileData {
+        @Override
+        public Stream<Path> resolvePaths(PathLookup pathLookup) {
+            return resolvePathSettings(pathLookup, setting);
+        }
+    }
+
+    private record RelativePathSettingFileData(String setting, BaseDir baseDir, Mode mode) implements FileData, RelativeFileData {
+        @Override
+        public Stream<Path> resolveRelativePaths(PathLookup pathLookup) {
+            return resolvePathSettings(pathLookup, setting);
+        }
+    }
+
+    private static Stream<Path> resolvePathSettings(PathLookup pathLookup, String setting) {
+        if (setting.contains("*")) {
+            return pathLookup.settingGlobResolver().apply(setting).map(Path::of);
+        }
+        String path = pathLookup.settingResolver().apply(setting);
+        return path == null ? Stream.of() : Stream.of(Path.of(path));
+    }
+
     private static Mode parseMode(String mode) {
         if (mode.equals("read")) {
             return Mode.READ;
@@ -113,37 +160,56 @@ public static FilesEntitlement build(List<Object> paths) {
             String pathAsString = file.remove("path");
             String relativePathAsString = file.remove("relative_path");
             String relativeTo = file.remove("relative_to");
-            String mode = file.remove("mode");
+            String pathSetting = file.remove("path_setting");
+            String relativePathSetting = file.remove("relative_path_setting");
+            String modeAsString = file.remove("mode");
 
             if (file.isEmpty() == false) {
                 throw new PolicyValidationException("unknown key(s) [" + file + "] in a listed file for files entitlement");
             }
-            if (mode == null) {
+            int foundKeys = (pathAsString != null ? 1 : 0) + (relativePathAsString != null ? 1 : 0) + (pathSetting != null ? 1 : 0)
+                + (relativePathSetting != null ? 1 : 0);
+            if (foundKeys != 1) {
+                throw new PolicyValidationException(
+                    "a files entitlement entry must contain one of " + "[path, relative_path, path_setting, relative_path_setting]"
+                );
+            }
+
+            if (modeAsString == null) {
                 throw new PolicyValidationException("files entitlement must contain 'mode' for every listed file");
             }
-            if (pathAsString != null && relativePathAsString != null) {
-                throw new PolicyValidationException("a files entitlement entry cannot contain both 'path' and 'relative_path'");
+            Mode mode = parseMode(modeAsString);
+
+            BaseDir baseDir = null;
+            if (relativeTo != null) {
+                baseDir = parseBaseDir(relativeTo);
             }
 
             if (relativePathAsString != null) {
-                if (relativeTo == null) {
+                if (baseDir == null) {
                     throw new PolicyValidationException("files entitlement with a 'relative_path' must specify 'relative_to'");
                 }
-                final BaseDir baseDir = parseBaseDir(relativeTo);
 
                 Path relativePath = Path.of(relativePathAsString);
                 if (relativePath.isAbsolute()) {
                     throw new PolicyValidationException("'relative_path' [" + relativePathAsString + "] must be relative");
                 }
-                filesData.add(FileData.ofRelativePath(relativePath, baseDir, parseMode(mode)));
+                filesData.add(FileData.ofRelativePath(relativePath, baseDir, mode));
             } else if (pathAsString != null) {
                 Path path = Path.of(pathAsString);
                 if (path.isAbsolute() == false) {
                     throw new PolicyValidationException("'path' [" + pathAsString + "] must be absolute");
                 }
-                filesData.add(FileData.ofPath(path, parseMode(mode)));
+                filesData.add(FileData.ofPath(path, mode));
+            } else if (pathSetting != null) {
+                filesData.add(FileData.ofPathSetting(pathSetting, mode));
+            } else if (relativePathSetting != null) {
+                if (baseDir == null) {
+                    throw new PolicyValidationException("files entitlement with a 'relative_path_setting' must specify 'relative_to'");
+                }
+                filesData.add(FileData.ofRelativePathSetting(relativePathSetting, baseDir, mode));
             } else {
-                throw new PolicyValidationException("files entitlement must contain either 'path' or 'relative_path' for every entry");
+                throw new AssertionError("File entry validation error");
             }
         }
         return new FilesEntitlement(filesData);