From 72249f3ef7ad43b78ef965a05e409ec4d334145e Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Tue, 3 Jun 2025 14:46:21 +0100 Subject: [PATCH 1/2] Catch FileSystemException when opening direct IO --- .../index/store/FsDirectoryFactory.java | 44 +++++++++++++------ 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java index 3b74a4a49d66d..0eb3d3c6034ab 100644 --- a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java +++ b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java @@ -22,6 +22,7 @@ import org.apache.lucene.store.NativeFSLockFactory; import org.apache.lucene.store.ReadAdvice; import org.apache.lucene.store.SimpleFSLockFactory; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.core.IOUtils; @@ -35,6 +36,7 @@ import org.elasticsearch.plugins.IndexStorePlugin; import java.io.IOException; +import java.nio.file.FileSystemException; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashSet; @@ -154,22 +156,38 @@ protected boolean useDirectIO(String name, IOContext context, OptionalLong fileL @Override public IndexInput openInput(String name, IOContext context) throws IOException { + Throwable directIOException = null; if (directIODelegate != null && context.hints().contains(DirectIOHint.INSTANCE)) { ensureOpen(); ensureCanRead(name); - Log.debug("Opening {} with direct IO", name); - return directIODelegate.openInput(name, context); - } else if (useDelegate(name, context)) { - // we need to do these checks on the outer directory since the inner doesn't know about pending deletes - ensureOpen(); - ensureCanRead(name); - // we only use the mmap to open inputs. Everything else is managed by the NIOFSDirectory otherwise - // we might run into trouble with files that are pendingDelete in one directory but still - // listed in listAll() from the other. We on the other hand don't want to list files from both dirs - // and intersect for perf reasons. - return delegate.openInput(name, context); - } else { - return super.openInput(name, context); + try { + Log.debug("Opening {} with direct IO", name); + return directIODelegate.openInput(name, context); + } catch (FileSystemException e) { + Log.debug(() -> Strings.format("Could not open %s with direct IO", name), e); + directIOException = e; + // and fallthrough to normal opening below + } + } + + try { + if (useDelegate(name, context)) { + // we need to do these checks on the outer directory since the inner doesn't know about pending deletes + ensureOpen(); + ensureCanRead(name); + // we only use the mmap to open inputs. Everything else is managed by the NIOFSDirectory otherwise + // we might run into trouble with files that are pendingDelete in one directory but still + // listed in listAll() from the other. We on the other hand don't want to list files from both dirs + // and intersect for perf reasons. + return delegate.openInput(name, context); + } else { + return super.openInput(name, context); + } + } catch (Throwable t) { + if (directIOException != null) { + t.addSuppressed(directIOException); + } + throw t; } } From 1fa220de4fbc4a2fe892b54046c0c903f93b4c56 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Thu, 12 Jun 2025 16:39:56 +0100 Subject: [PATCH 2/2] DirectIOIT now checks that direct IO not supported is also logged --- .../elasticsearch/index/store/DirectIOIT.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/store/DirectIOIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/store/DirectIOIT.java index c80eac73f4f6a..13a37a101db90 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/store/DirectIOIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/store/DirectIOIT.java @@ -42,13 +42,16 @@ @LuceneTestCase.SuppressCodecs("*") // only use our own codecs public class DirectIOIT extends ESIntegTestCase { + private static boolean SUPPORTED; + @BeforeClass public static void checkSupported() throws IOException { Path path = createTempDir("directIOProbe"); try (Directory dir = open(path); IndexOutput out = dir.createOutput("out", IOContext.DEFAULT)) { out.writeString("test"); + SUPPORTED = true; } catch (IOException e) { - assumeNoException("test requires filesystem that supports Direct IO", e); + SUPPORTED = false; } } @@ -106,15 +109,21 @@ static void assertBBQIndexType(String type) { @TestLogging(value = "org.elasticsearch.index.store.FsDirectoryFactory:DEBUG", reason = "to capture trace logging for direct IO") public void testDirectIOUsed() { try (MockLog mockLog = MockLog.capture(FsDirectoryFactory.class)) { - // we're just looking for some evidence direct IO is used - mockLog.addExpectation( - new MockLog.PatternSeenEventExpectation( + // we're just looking for some evidence direct IO is used (or not) + MockLog.LoggingExpectation expectation = SUPPORTED + ? new MockLog.PatternSeenEventExpectation( "Direct IO used", FsDirectoryFactory.class.getCanonicalName(), Level.DEBUG, "Opening .*\\.vec with direct IO" ) - ); + : new MockLog.PatternSeenEventExpectation( + "Direct IO not used", + FsDirectoryFactory.class.getCanonicalName(), + Level.DEBUG, + "Could not open .*\\.vec with direct IO" + ); + mockLog.addExpectation(expectation); indexVectors();