Backport: Add exclusive file entitlement for settings (#125272) (#126006)

Author: jdconrad

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -68,6 +68,7 @@
 import java.util.stream.StreamSupport;
 
 import static org.elasticsearch.entitlement.runtime.policy.Platform.LINUX;
+import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.BaseDir.CONFIG;
 import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.BaseDir.DATA;
 import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.BaseDir.SHARED_REPO;
 import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ;
@@ -182,6 +183,8 @@ private static PolicyManager createPolicyManager() {
             FileData.ofPath(bootstrapArgs.libDir(), READ),
             FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE),
             FileData.ofRelativePath(Path.of(""), SHARED_REPO, READ_WRITE),
+            // exclusive settings file
+            FileData.ofRelativePath(Path.of("operator/settings.json"), CONFIG, READ_WRITE).withExclusive(true),
 
             // OS release on Linux
             FileData.ofPath(Path.of("/etc/os-release"), READ).withPlatform(LINUX),

server/src/main/java/org/elasticsearch/common/file/AbstractFileWatchingService.java

@@ -16,16 +16,18 @@
 import org.elasticsearch.reservedstate.service.FileChangedListener;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.file.ClosedWatchServiceException;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardWatchEventKinds;
 import java.nio.file.WatchKey;
 import java.nio.file.WatchService;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.FileTime;
 import java.util.List;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.ExecutionException;
+import java.util.stream.Stream;
 
 /**
  * A skeleton service for watching and reacting to a single file changing on disk
@@ -119,20 +121,20 @@ public final boolean watching() {
     // platform independent way to tell if a file changed
     // we compare the file modified timestamp, the absolute path (symlinks), and file id on the system
     final boolean watchedFileChanged(Path path) throws IOException {
-        if (Files.exists(path) == false) {
+        if (filesExists(path) == false) {
             return false;
         }
 
         FileUpdateState previousUpdateState = fileUpdateState;
 
-        BasicFileAttributes attr = Files.readAttributes(path, BasicFileAttributes.class);
+        BasicFileAttributes attr = filesReadAttributes(path, BasicFileAttributes.class);
         fileUpdateState = new FileUpdateState(attr.lastModifiedTime().toMillis(), path.toRealPath().toString(), attr.fileKey());
 
         return (previousUpdateState == null || previousUpdateState.equals(fileUpdateState) == false);
     }
 
     protected final synchronized void startWatcher() {
-        if (Files.exists(watchedFileDir.getParent()) == false) {
+        if (filesExists(watchedFileDir.getParent()) == false) {
             logger.warn("File watcher for [{}] cannot start because grandparent directory does not exist", watchedFile);
             return;
         }
@@ -147,7 +149,7 @@ protected final synchronized void startWatcher() {
         try {
             Path settingsDirPath = watchedFileDir();
             this.watchService = settingsDirPath.getParent().getFileSystem().newWatchService();
-            if (Files.exists(settingsDirPath)) {
+            if (filesExists(settingsDirPath)) {
                 settingsDirWatchKey = enableDirectoryWatcher(settingsDirWatchKey, settingsDirPath);
             } else {
                 logger.debug("watched directory [{}] not found, will watch for its creation...", settingsDirPath);
@@ -181,7 +183,7 @@ protected final void watcherThread() {
 
             Path path = watchedFile();
 
-            if (Files.exists(path)) {
+            if (filesExists(path)) {
                 logger.debug("found initial operator settings file [{}], applying...", path);
                 processSettingsOnServiceStartAndNotifyListeners();
             } else {
@@ -209,7 +211,7 @@ protected final void watcherThread() {
                  * real path of our desired file. We don't actually care what changed, we just re-check ourselves.
                  */
                 Path settingsPath = watchedFileDir();
-                if (Files.exists(settingsPath)) {
+                if (filesExists(settingsPath)) {
                     try {
                         if (logger.isDebugEnabled()) {
                             key.pollEvents().forEach(e -> logger.debug("{}:{}", e.kind().toString(), e.context().toString()));
@@ -341,4 +343,19 @@ long retryDelayMillis(int failedCount) {
      * class to determine if a file has been changed.
      */
     private record FileUpdateState(long timestamp, String path, Object fileKey) {}
+
+    // the following methods are a workaround to ensure exclusive access for files
+    // required by child watchers; this is required because we only check the caller's module
+    // not the entire stack
+    protected abstract boolean filesExists(Path path);
+
+    protected abstract boolean filesIsDirectory(Path path);
+
+    protected abstract <A extends BasicFileAttributes> A filesReadAttributes(Path path, Class<A> clazz) throws IOException;
+
+    protected abstract Stream<Path> filesList(Path dir) throws IOException;
+
+    protected abstract Path filesSetLastModifiedTime(Path path, FileTime time) throws IOException;
+
+    protected abstract InputStream filesNewInputStream(Path path) throws IOException;
 }

server/src/main/java/org/elasticsearch/common/file/MasterNodeFileWatchingService.java

@@ -19,10 +19,13 @@
 import org.elasticsearch.gateway.GatewayService;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.nio.file.attribute.FileTime;
 import java.time.Instant;
+import java.util.stream.Stream;
 
 public abstract class MasterNodeFileWatchingService extends AbstractFileWatchingService implements ClusterStateListener {
 
@@ -41,7 +44,7 @@ protected void doStart() {
         // We start the file watcher when we know we are master from a cluster state change notification.
         // We need the additional active flag, since cluster state can change after we've shutdown the service
         // causing the watcher to start again.
-        this.active = Files.exists(watchedFileDir().getParent());
+        this.active = filesExists(watchedFileDir().getParent());
         if (active == false) {
             // we don't have a config directory, we can't possibly launch the file settings service
             return;
@@ -86,9 +89,9 @@ public final void clusterChanged(ClusterChangedEvent event) {
      */
     private void refreshExistingFileStateIfNeeded(ClusterState clusterState) {
         if (watching()) {
-            if (shouldRefreshFileState(clusterState) && Files.exists(watchedFile())) {
+            if (shouldRefreshFileState(clusterState) && filesExists(watchedFile())) {
                 try {
-                    Files.setLastModifiedTime(watchedFile(), FileTime.from(Instant.now()));
+                    filesSetLastModifiedTime(watchedFile(), FileTime.from(Instant.now()));
                 } catch (IOException e) {
                     logger.warn("encountered I/O error trying to update file settings timestamp", e);
                 }
@@ -107,4 +110,37 @@ private void refreshExistingFileStateIfNeeded(ClusterState clusterState) {
     protected boolean shouldRefreshFileState(ClusterState clusterState) {
         return false;
     }
+
+    // the following methods are a workaround to ensure exclusive access for files
+    // required by child watchers; this is required because we only check the caller's module
+    // not the entire stack
+    @Override
+    protected boolean filesExists(Path path) {
+        return Files.exists(path);
+    }
+
+    @Override
+    protected boolean filesIsDirectory(Path path) {
+        return Files.isDirectory(path);
+    }
+
+    @Override
+    protected <A extends BasicFileAttributes> A filesReadAttributes(Path path, Class<A> clazz) throws IOException {
+        return Files.readAttributes(path, clazz);
+    }
+
+    @Override
+    protected Stream<Path> filesList(Path dir) throws IOException {
+        return Files.list(dir);
+    }
+
+    @Override
+    protected Path filesSetLastModifiedTime(Path path, FileTime time) throws IOException {
+        return Files.setLastModifiedTime(path, time);
+    }
+
+    @Override
+    protected InputStream filesNewInputStream(Path path) throws IOException {
+        return Files.newInputStream(path);
+    }
 }

server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java

@@ -33,10 +33,15 @@
 
 import java.io.BufferedInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.FileTime;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.health.HealthStatus.GREEN;
 import static org.elasticsearch.health.HealthStatus.YELLOW;
@@ -110,7 +115,7 @@ public void handleSnapshotRestore(ClusterState clusterState, Metadata.Builder md
         // since we don't know the current operator configuration, e.g. file settings could be disabled
         // on the target cluster. If file settings exist and the cluster state has lost it's reserved
         // state for the "file_settings" namespace, we touch our file settings file to cause it to re-process the file.
-        if (watching() && Files.exists(watchedFile())) {
+        if (watching() && filesExists(watchedFile())) {
             if (fileSettingsMetadata != null) {
                 ReservedStateMetadata withResetVersion = new ReservedStateMetadata.Builder(fileSettingsMetadata).version(0L).build();
                 mdBuilder.put(withResetVersion);
@@ -174,7 +179,7 @@ protected void processFileOnServiceStart() throws IOException, ExecutionExceptio
     private void processFileChanges(ReservedStateVersionCheck versionCheck) throws IOException, InterruptedException, ExecutionException {
         PlainActionFuture<Void> completion = new PlainActionFuture<>();
         try (
-            var fis = Files.newInputStream(watchedFile());
+            var fis = filesNewInputStream(watchedFile());
             var bis = new BufferedInputStream(fis);
             var parser = JSON.xContent().createParser(XContentParserConfiguration.EMPTY, bis)
         ) {
@@ -291,4 +296,37 @@ public synchronized HealthIndicatorResult calculate(boolean verbose, int maxAffe
             }
         }
     }
+
+    // the following methods are a workaround to ensure exclusive access for files
+    // required by child watchers; this is required because we only check the caller's module
+    // not the entire stack
+    @Override
+    protected boolean filesExists(Path path) {
+        return Files.exists(path);
+    }
+
+    @Override
+    protected boolean filesIsDirectory(Path path) {
+        return Files.isDirectory(path);
+    }
+
+    @Override
+    protected <A extends BasicFileAttributes> A filesReadAttributes(Path path, Class<A> clazz) throws IOException {
+        return Files.readAttributes(path, clazz);
+    }
+
+    @Override
+    protected Stream<Path> filesList(Path dir) throws IOException {
+        return Files.list(dir);
+    }
+
+    @Override
+    protected Path filesSetLastModifiedTime(Path path, FileTime time) throws IOException {
+        return Files.setLastModifiedTime(path, time);
+    }
+
+    @Override
+    protected InputStream filesNewInputStream(Path path) throws IOException {
+        return Files.newInputStream(path);
+    }
 }

server/src/test/java/org/elasticsearch/common/file/AbstractFileWatchingServiceTests.java

@@ -24,12 +24,14 @@
 import org.junit.Before;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.nio.file.StandardWatchEventKinds;
 import java.nio.file.WatchKey;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.nio.file.attribute.FileTime;
 import java.time.Instant;
 import java.time.LocalDateTime;
@@ -38,6 +40,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.node.Node.NODE_NAME_SETTING;
 import static org.hamcrest.Matchers.sameInstance;
@@ -81,6 +84,39 @@ protected void processInitialFileMissing() {
                 countDownLatch.countDown();
             }
         }
+
+        // the following methods are a workaround to ensure exclusive access for files
+        // required by child watchers; this is required because we only check the caller's module
+        // not the entire stack
+        @Override
+        protected boolean filesExists(Path path) {
+            return Files.exists(path);
+        }
+
+        @Override
+        protected boolean filesIsDirectory(Path path) {
+            return Files.isDirectory(path);
+        }
+
+        @Override
+        protected <A extends BasicFileAttributes> A filesReadAttributes(Path path, Class<A> clazz) throws IOException {
+            return Files.readAttributes(path, clazz);
+        }
+
+        @Override
+        protected Stream<Path> filesList(Path dir) throws IOException {
+            return Files.list(dir);
+        }
+
+        @Override
+        protected Path filesSetLastModifiedTime(Path path, FileTime time) throws IOException {
+            return Files.setLastModifiedTime(path, time);
+        }
+
+        @Override
+        protected InputStream filesNewInputStream(Path path) throws IOException {
+            return Files.newInputStream(path);
+        }
     }
 
     private AbstractFileWatchingService fileWatchingService;

x-pack/plugin/core/src/main/config/log4j2.properties

@@ -115,3 +115,6 @@ logger.samlxml_decrypt.name = org.opensaml.xmlsec.encryption.support.Decrypter
 logger.samlxml_decrypt.level = fatal
 logger.saml2_decrypt.name = org.opensaml.saml.saml2.encryption.Decrypter
 logger.saml2_decrypt.level = fatal
+
+logger.entitlements_xpack_security.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.x-pack-security.org.elasticsearch.security
+logger.entitlements_xpack_security.level = error