diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options index 94fc6f2cb9025..724701a9affe4 100644 --- a/distribution/src/config/jvm.options +++ b/distribution/src/config/jvm.options @@ -65,6 +65,11 @@ # Lucene 10: apply MADV_NORMAL advice to enable more aggressive readahead -Dorg.apache.lucene.store.defaultReadAdvice=normal +# Lucene provides a mechanism for shared mmapped arenas to be referenced between multiple threads +# this is to get around potential performance issues when closing shared arenas on many threads +# default to 1 to disable this feature +-Dorg.apache.lucene.store.MMapDirectory.sharedArenaMaxPermits=1 + ## heap dumps # generate a heap dump when an allocation from the Java heap fails; heap dumps diff --git a/docs/changelog/135012.yaml b/docs/changelog/135012.yaml new file mode 100644 index 0000000000000..abbf003ccbd39 --- /dev/null +++ b/docs/changelog/135012.yaml @@ -0,0 +1,6 @@ +pr: 135012 +summary: Bypass MMap arena grouping as this has caused issues with too many regions + being mapped +area: "Engine" +type: bug +issues: [] diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java index b9f4943b1dab6..c4b0519309311 100644 --- a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java +++ b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java @@ -24,11 +24,9 @@ public final class SmbMmapFsDirectoryFactory extends FsDirectoryFactory { @Override protected Directory newFSDirectory(Path location, LockFactory lockFactory, IndexSettings indexSettings) throws IOException { + MMapDirectory mMapDirectory = adjustSharedArenaGrouping(new MMapDirectory(location, lockFactory)); return new SmbDirectoryWrapper( - setPreload( - new MMapDirectory(location, lockFactory), - new HashSet<>(indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING)) - ) + setPreload(mMapDirectory, new HashSet<>(indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING))) ); } } diff --git a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java index c8c5ffa19dc80..6c47dd1214e08 100644 --- a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java +++ b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java @@ -28,6 +28,8 @@ import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardPath; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.plugins.IndexStorePlugin; import java.io.IOException; @@ -37,8 +39,25 @@ import java.util.Set; import java.util.function.BiPredicate; +import static org.apache.lucene.store.MMapDirectory.SHARED_ARENA_MAX_PERMITS_SYSPROP; + public class FsDirectoryFactory implements IndexStorePlugin.DirectoryFactory { + private static final Logger Log = LogManager.getLogger(FsDirectoryFactory.class); + private static final int sharedArenaMaxPermits; + static { + String prop = System.getProperty(SHARED_ARENA_MAX_PERMITS_SYSPROP); + int value = 1; + if (prop != null) { + try { + value = Integer.parseInt(prop); // ensure it's a valid integer + } catch (NumberFormatException e) { + Log.warn(() -> "unable to parse system property [" + SHARED_ARENA_MAX_PERMITS_SYSPROP + "] with value [" + prop + "]", e); + } + } + sharedArenaMaxPermits = value; // default to 1 + } + private static final FeatureFlag MADV_RANDOM_FEATURE_FLAG = new FeatureFlag("madv_random"); public static final Setting INDEX_LOCK_FACTOR_SETTING = new Setting<>("index.store.fs.fs_lock", "native", (s) -> { @@ -72,6 +91,7 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index // Use Lucene defaults final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory); if (primaryDirectory instanceof MMapDirectory mMapDirectory) { + mMapDirectory = adjustSharedArenaGrouping(mMapDirectory); Directory dir = new HybridDirectory(lockFactory, setPreload(mMapDirectory, preLoadExtensions)); if (MADV_RANDOM_FEATURE_FLAG.isEnabled() == false) { dir = disableRandomAdvice(dir); @@ -81,7 +101,8 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index return primaryDirectory; } case MMAPFS: - Directory dir = setPreload(new MMapDirectory(location, lockFactory), preLoadExtensions); + MMapDirectory mMapDirectory = adjustSharedArenaGrouping(new MMapDirectory(location, lockFactory)); + Directory dir = setPreload(mMapDirectory, preLoadExtensions); if (MADV_RANDOM_FEATURE_FLAG.isEnabled() == false) { dir = disableRandomAdvice(dir); } @@ -101,6 +122,13 @@ public MMapDirectory setPreload(MMapDirectory mMapDirectory, Set preLoad return mMapDirectory; } + public MMapDirectory adjustSharedArenaGrouping(MMapDirectory mMapDirectory) { + if (sharedArenaMaxPermits <= 1) { + mMapDirectory.setGroupingFunction(MMapDirectory.NO_GROUPING); + } + return mMapDirectory; + } + /** Gets a preload function based on the given preLoadExtensions. */ static BiPredicate getPreloadFunc(Set preLoadExtensions) { if (preLoadExtensions.isEmpty() == false) {