From 50ad283c1ef87293616687d2770a48692db809af Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Fri, 19 Sep 2025 18:51:48 -0400 Subject: [PATCH] Bypass MMap arena grouping as this has caused issues with too many regions being mapped (#135012) There is a JDK issue where closing sharedArenas from many threads can significantly harm performance. This ref-counting of shared arenas was designed as a way to get around this performance issue. However, we have noticed a significant increase in leaks and issues with mmap regions since this change. https://bugs.openjdk.org/browse/JDK-8335480 should have helped the performance impact of closing shared arenas (though possibly not fully mitigated it). I am proposing we turn off the grouping as it appears (at least to me), not worth it. I am willing to backdown if we thing other fixes should be done. I also suggest this gets backported to 9.1, 8.19, and is merged into 9.2 --- distribution/src/config/jvm.options | 5 +++ docs/changelog/135012.yaml | 6 ++++ .../store/smb/SmbMmapFsDirectoryFactory.java | 6 ++-- .../index/store/FsDirectoryFactory.java | 31 ++++++++++++++++++- 4 files changed, 43 insertions(+), 5 deletions(-) create mode 100644 docs/changelog/135012.yaml diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options index 081ee38303d8c..f4cc3b1bf6191 100644 --- a/distribution/src/config/jvm.options +++ b/distribution/src/config/jvm.options @@ -65,6 +65,11 @@ # Lucene 10: apply MADV_NORMAL advice to enable more aggressive readahead -Dorg.apache.lucene.store.defaultReadAdvice=normal +# Lucene provides a mechanism for shared mmapped arenas to be referenced between multiple threads +# this is to get around potential performance issues when closing shared arenas on many threads +# default to 1 to disable this feature +-Dorg.apache.lucene.store.MMapDirectory.sharedArenaMaxPermits=1 + ## heap dumps # generate a heap dump when an allocation from the Java heap fails; heap dumps diff --git a/docs/changelog/135012.yaml b/docs/changelog/135012.yaml new file mode 100644 index 0000000000000..abbf003ccbd39 --- /dev/null +++ b/docs/changelog/135012.yaml @@ -0,0 +1,6 @@ +pr: 135012 +summary: Bypass MMap arena grouping as this has caused issues with too many regions + being mapped +area: "Engine" +type: bug +issues: [] diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java index b9f4943b1dab6..c4b0519309311 100644 --- a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java +++ b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java @@ -24,11 +24,9 @@ public final class SmbMmapFsDirectoryFactory extends FsDirectoryFactory { @Override protected Directory newFSDirectory(Path location, LockFactory lockFactory, IndexSettings indexSettings) throws IOException { + MMapDirectory mMapDirectory = adjustSharedArenaGrouping(new MMapDirectory(location, lockFactory)); return new SmbDirectoryWrapper( - setPreload( - new MMapDirectory(location, lockFactory), - new HashSet<>(indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING)) - ) + setPreload(mMapDirectory, new HashSet<>(indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING))) ); } } diff --git a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java index 39a5c03e27942..faae096075fdd 100644 --- a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java +++ b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java @@ -42,8 +42,28 @@ import java.util.Set; import java.util.function.BiPredicate; +import static org.apache.lucene.store.MMapDirectory.SHARED_ARENA_MAX_PERMITS_SYSPROP; + public class FsDirectoryFactory implements IndexStorePlugin.DirectoryFactory { + private static final int sharedArenaMaxPermits; + static { + String prop = System.getProperty(SHARED_ARENA_MAX_PERMITS_SYSPROP); + int value = 1; + if (prop != null) { + try { + value = Integer.parseInt(prop); // ensure it's a valid integer + } catch (NumberFormatException e) { + Logger logger = LogManager.getLogger(FsDirectoryFactory.class); + logger.warn( + () -> "unable to parse system property [" + SHARED_ARENA_MAX_PERMITS_SYSPROP + "] with value [" + prop + "]", + e + ); + } + } + sharedArenaMaxPermits = value; // default to 1 + } + private static final Logger Log = LogManager.getLogger(FsDirectoryFactory.class); private static final FeatureFlag MADV_RANDOM_FEATURE_FLAG = new FeatureFlag("madv_random"); @@ -78,6 +98,7 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index // Use Lucene defaults final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory); if (primaryDirectory instanceof MMapDirectory mMapDirectory) { + mMapDirectory = adjustSharedArenaGrouping(mMapDirectory); Directory dir = new HybridDirectory(lockFactory, setPreload(mMapDirectory, preLoadExtensions)); if (MADV_RANDOM_FEATURE_FLAG.isEnabled() == false) { dir = disableRandomAdvice(dir); @@ -87,7 +108,8 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index return primaryDirectory; } case MMAPFS: - Directory dir = setPreload(new MMapDirectory(location, lockFactory), preLoadExtensions); + MMapDirectory mMapDirectory = adjustSharedArenaGrouping(new MMapDirectory(location, lockFactory)); + Directory dir = setPreload(mMapDirectory, preLoadExtensions); if (MADV_RANDOM_FEATURE_FLAG.isEnabled() == false) { dir = disableRandomAdvice(dir); } @@ -107,6 +129,13 @@ public MMapDirectory setPreload(MMapDirectory mMapDirectory, Set preLoad return mMapDirectory; } + public MMapDirectory adjustSharedArenaGrouping(MMapDirectory mMapDirectory) { + if (sharedArenaMaxPermits <= 1) { + mMapDirectory.setGroupingFunction(MMapDirectory.NO_GROUPING); + } + return mMapDirectory; + } + /** Gets a preload function based on the given preLoadExtensions. */ static BiPredicate getPreloadFunc(Set preLoadExtensions) { if (preLoadExtensions.isEmpty() == false) {