From cbaf6d22948927670345197892fc2e279061d7e5 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Fri, 19 Sep 2025 18:51:48 -0400 Subject: [PATCH 1/3] Bypass MMap arena grouping as this has caused issues with too many regions being mapped (#135012) There is a JDK issue where closing sharedArenas from many threads can significantly harm performance. This ref-counting of shared arenas was designed as a way to get around this performance issue. However, we have noticed a significant increase in leaks and issues with mmap regions since this change. https://bugs.openjdk.org/browse/JDK-8335480 should have helped the performance impact of closing shared arenas (though possibly not fully mitigated it). I am proposing we turn off the grouping as it appears (at least to me), not worth it. I am willing to backdown if we thing other fixes should be done. I also suggest this gets backported to 9.1, 8.19, and is merged into 9.2 (cherry picked from commit 2672cd0088a3b4f13be84b0c150912de8029f9cb) --- distribution/src/config/jvm.options | 3 ++ docs/changelog/135012.yaml | 6 ++++ .../store/smb/SmbMmapFsDirectoryFactory.java | 7 ++--- .../index/store/FsDirectoryFactory.java | 31 ++++++++++++++++++- 4 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 docs/changelog/135012.yaml diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options index 340baedded785..94fc6f2cb9025 100644 --- a/distribution/src/config/jvm.options +++ b/distribution/src/config/jvm.options @@ -62,6 +62,9 @@ 23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.setAsTypeCache 23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.asTypeUncached +# Lucene 10: apply MADV_NORMAL advice to enable more aggressive readahead +-Dorg.apache.lucene.store.defaultReadAdvice=normal + ## heap dumps # generate a heap dump when an allocation from the Java heap fails; heap dumps diff --git a/docs/changelog/135012.yaml b/docs/changelog/135012.yaml new file mode 100644 index 0000000000000..abbf003ccbd39 --- /dev/null +++ b/docs/changelog/135012.yaml @@ -0,0 +1,6 @@ +pr: 135012 +summary: Bypass MMap arena grouping as this has caused issues with too many regions + being mapped +area: "Engine" +type: bug +issues: [] diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java index 4594e8d71c6fb..df7969481bfcf 100644 --- a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java +++ b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java @@ -24,12 +24,9 @@ public final class SmbMmapFsDirectoryFactory extends FsDirectoryFactory { @Override protected Directory newFSDirectory(Path location, LockFactory lockFactory, IndexSettings indexSettings) throws IOException { + MMapDirectory mMapDirectory = adjustSharedArenaGrouping(new MMapDirectory(location, lockFactory)); return new SmbDirectoryWrapper( - setPreload( - new MMapDirectory(location, lockFactory), - lockFactory, - new HashSet<>(indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING)) - ) + setPreload(mMapDirectory, lockFactory, new HashSet<>(indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING))) ); } } diff --git a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java index e76060b6c0dc8..3f67e51ccc24f 100644 --- a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java +++ b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java @@ -35,8 +35,28 @@ import java.util.HashSet; import java.util.Set; +import static org.apache.lucene.store.MMapDirectory.SHARED_ARENA_MAX_PERMITS_SYSPROP; + public class FsDirectoryFactory implements IndexStorePlugin.DirectoryFactory { + private static final Logger Log = LogManager.getLogger(FsDirectoryFactory.class); + private static final int sharedArenaMaxPermits; + static { + String prop = System.getProperty(SHARED_ARENA_MAX_PERMITS_SYSPROP); + int value = 1; + if (prop != null) { + try { + value = Integer.parseInt(prop); // ensure it's a valid integer + } catch (NumberFormatException e) { + Log.warn( + () -> "unable to parse system property [" + SHARED_ARENA_MAX_PERMITS_SYSPROP + "] with value [" + prop + "]", + e + ); + } + } + sharedArenaMaxPermits = value; // default to 1 + } + private static final FeatureFlag MADV_RANDOM_FEATURE_FLAG = new FeatureFlag("madv_random"); public static final Setting INDEX_LOCK_FACTOR_SETTING = new Setting<>("index.store.fs.fs_lock", "native", (s) -> { @@ -70,6 +90,7 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index // Use Lucene defaults final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory); if (primaryDirectory instanceof MMapDirectory mMapDirectory) { + mMapDirectory = adjustSharedArenaGrouping(mMapDirectory); Directory dir = new HybridDirectory(lockFactory, setPreload(mMapDirectory, lockFactory, preLoadExtensions)); if (MADV_RANDOM_FEATURE_FLAG.isEnabled() == false) { dir = disableRandomAdvice(dir); @@ -79,7 +100,8 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index return primaryDirectory; } case MMAPFS: - Directory dir = setPreload(new MMapDirectory(location, lockFactory), lockFactory, preLoadExtensions); + MMapDirectory mMapDirectory = adjustSharedArenaGrouping(new MMapDirectory(location, lockFactory)); + Directory dir = setPreload(mMapDirectory, lockFactory, preLoadExtensions); if (MADV_RANDOM_FEATURE_FLAG.isEnabled() == false) { dir = disableRandomAdvice(dir); } @@ -105,6 +127,13 @@ public static MMapDirectory setPreload(MMapDirectory mMapDirectory, LockFactory return mMapDirectory; } + public MMapDirectory adjustSharedArenaGrouping(MMapDirectory mMapDirectory) { + if (sharedArenaMaxPermits <= 1) { + mMapDirectory.setGroupingFunction(MMapDirectory.NO_GROUPING); + } + return mMapDirectory; + } + /** * Return a {@link FilterDirectory} around the provided {@link Directory} that forcefully disables {@link IOContext#RANDOM random * access}. From 45dd3979778498ddc315ea1442db9930ac4f5dcb Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Sat, 20 Sep 2025 13:38:25 +0000 Subject: [PATCH 2/3] [CI] Auto commit changes from spotless --- .../org/elasticsearch/index/store/FsDirectoryFactory.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java index 3f67e51ccc24f..d037db830b7d4 100644 --- a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java +++ b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java @@ -48,10 +48,7 @@ public class FsDirectoryFactory implements IndexStorePlugin.DirectoryFactory { try { value = Integer.parseInt(prop); // ensure it's a valid integer } catch (NumberFormatException e) { - Log.warn( - () -> "unable to parse system property [" + SHARED_ARENA_MAX_PERMITS_SYSPROP + "] with value [" + prop + "]", - e - ); + Log.warn(() -> "unable to parse system property [" + SHARED_ARENA_MAX_PERMITS_SYSPROP + "] with value [" + prop + "]", e); } } sharedArenaMaxPermits = value; // default to 1 From 90707469d2a690b288294ffd700cd74ff672fa87 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Sat, 20 Sep 2025 09:52:42 -0400 Subject: [PATCH 3/3] fixing compilation --- .../java/org/elasticsearch/index/store/FsDirectoryFactory.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java index 3f67e51ccc24f..703827528a31a 100644 --- a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java +++ b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java @@ -27,6 +27,8 @@ import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardPath; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.plugins.IndexStorePlugin; import java.io.IOException;