Bypass MMap arena grouping as this has caused issues with too many regions being mapped (elastic#135012)

There is a JDK issue where closing sharedArenas from many threads can
significantly harm performance.

This ref-counting of shared arenas was designed as a way to get around
this performance issue. However, we have noticed a significant increase
in leaks and issues with mmap regions since this change.

https://bugs.openjdk.org/browse/JDK-8335480 should have helped the
performance impact of closing shared arenas (though possibly not fully
mitigated it).

I am proposing we turn off the grouping as it appears (at least to me),
not worth it.

I am willing to backdown if we thing other fixes should be done.

I also suggest this gets backported to 9.1, 8.19, and is merged into 9.2

(cherry picked from commit 2672cd0)

Author: benwtrent

distribution/src/config/jvm.options

@@ -65,6 +65,11 @@
 # Lucene 10: apply MADV_NORMAL advice to enable more aggressive readahead
 -Dorg.apache.lucene.store.defaultReadAdvice=normal
 
+# Lucene provides a mechanism for shared mmapped arenas to be referenced between multiple threads
+# this is to get around potential performance issues when closing shared arenas on many threads
+# default to 1 to disable this feature
+-Dorg.apache.lucene.store.MMapDirectory.sharedArenaMaxPermits=1
+
 ## heap dumps
 
 # generate a heap dump when an allocation from the Java heap fails; heap dumps

docs/changelog/135012.yaml

@@ -0,0 +1,6 @@
+pr: 135012
+summary: Bypass MMap arena grouping as this has caused issues with too many regions
+  being mapped
+area: "Engine"
+type: bug
+issues: []

plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java

@@ -24,11 +24,9 @@ public final class SmbMmapFsDirectoryFactory extends FsDirectoryFactory {
 
     @Override
     protected Directory newFSDirectory(Path location, LockFactory lockFactory, IndexSettings indexSettings) throws IOException {
+        MMapDirectory mMapDirectory = adjustSharedArenaGrouping(new MMapDirectory(location, lockFactory));
         return new SmbDirectoryWrapper(
-            setPreload(
-                new MMapDirectory(location, lockFactory),
-                new HashSet<>(indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING))
-            )
+            setPreload(mMapDirectory, new HashSet<>(indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING)))
         );
     }
 }

server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java

@@ -37,8 +37,28 @@
 import java.util.Set;
 import java.util.function.BiPredicate;
 
+import static org.apache.lucene.store.MMapDirectory.SHARED_ARENA_MAX_PERMITS_SYSPROP;
+
 public class FsDirectoryFactory implements IndexStorePlugin.DirectoryFactory {
 
+    private static final Logger Log = LogManager.getLogger(FsDirectoryFactory.class);
+    private static final int sharedArenaMaxPermits;
+    static {
+        String prop = System.getProperty(SHARED_ARENA_MAX_PERMITS_SYSPROP);
+        int value = 1;
+        if (prop != null) {
+            try {
+                value = Integer.parseInt(prop); // ensure it's a valid integer
+            } catch (NumberFormatException e) {
+                Log.warn(
+                    () -> "unable to parse system property [" + SHARED_ARENA_MAX_PERMITS_SYSPROP + "] with value [" + prop + "]",
+                    e
+                );
+            }
+        }
+        sharedArenaMaxPermits = value; // default to 1
+    }
+
     private static final FeatureFlag MADV_RANDOM_FEATURE_FLAG = new FeatureFlag("madv_random");
 
     public static final Setting<LockFactory> INDEX_LOCK_FACTOR_SETTING = new Setting<>("index.store.fs.fs_lock", "native", (s) -> {
@@ -72,6 +92,7 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index
                 // Use Lucene defaults
                 final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory);
                 if (primaryDirectory instanceof MMapDirectory mMapDirectory) {
+                    mMapDirectory = adjustSharedArenaGrouping(mMapDirectory);
                     Directory dir = new HybridDirectory(lockFactory, setPreload(mMapDirectory, preLoadExtensions));
                     if (MADV_RANDOM_FEATURE_FLAG.isEnabled() == false) {
                         dir = disableRandomAdvice(dir);
@@ -81,7 +102,8 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index
                     return primaryDirectory;
                 }
             case MMAPFS:
-                Directory dir = setPreload(new MMapDirectory(location, lockFactory), preLoadExtensions);
+                MMapDirectory mMapDirectory = adjustSharedArenaGrouping(new MMapDirectory(location, lockFactory));
+                Directory dir = setPreload(mMapDirectory, preLoadExtensions);
                 if (MADV_RANDOM_FEATURE_FLAG.isEnabled() == false) {
                     dir = disableRandomAdvice(dir);
                 }
@@ -101,6 +123,13 @@ public MMapDirectory setPreload(MMapDirectory mMapDirectory, Set<String> preLoad
         return mMapDirectory;
     }
 
+    public MMapDirectory adjustSharedArenaGrouping(MMapDirectory mMapDirectory) {
+        if (sharedArenaMaxPermits <= 1) {
+            mMapDirectory.setGroupingFunction(MMapDirectory.NO_GROUPING);
+        }
+        return mMapDirectory;
+    }
+
     /** Gets a preload function based on the given preLoadExtensions. */
     static BiPredicate<String, IOContext> getPreloadFunc(Set<String> preLoadExtensions) {
         if (preLoadExtensions.isEmpty() == false) {