Bypass MMap arena grouping as this has caused issues with too many regions being mapped (#135012) (#135115)

There is a JDK issue where closing sharedArenas from many threads can
significantly harm performance. 

This ref-counting of shared arenas was designed as a way to get around
this performance issue. However, we have noticed a significant increase
in leaks and issues with mmap regions since this change. 

https://bugs.openjdk.org/browse/JDK-8335480 should have helped the
performance impact of closing shared arenas (though possibly not fully
mitigated it).

I am proposing we turn off the grouping as it appears (at least to me),
not worth it. 

I am willing to backdown if we thing other fixes should be done.

I also suggest this gets backported to 9.1, 8.19, and is merged into 9.2

Author: benwtrent

distribution/src/config/jvm.options

@@ -65,6 +65,11 @@
 # Lucene 10: apply MADV_NORMAL advice to enable more aggressive readahead
 -Dorg.apache.lucene.store.defaultReadAdvice=normal
 
+# Lucene provides a mechanism for shared mmapped arenas to be referenced between multiple threads
+# this is to get around potential performance issues when closing shared arenas on many threads
+# default to 1 to disable this feature
+-Dorg.apache.lucene.store.MMapDirectory.sharedArenaMaxPermits=1
+
 ## heap dumps
 
 # generate a heap dump when an allocation from the Java heap fails; heap dumps

docs/changelog/135012.yaml

@@ -0,0 +1,6 @@
+pr: 135012
+summary: Bypass MMap arena grouping as this has caused issues with too many regions
+  being mapped
+area: "Engine"
+type: bug
+issues: []

plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java

@@ -24,11 +24,9 @@ public final class SmbMmapFsDirectoryFactory extends FsDirectoryFactory {
 
     @Override
     protected Directory newFSDirectory(Path location, LockFactory lockFactory, IndexSettings indexSettings) throws IOException {
+        MMapDirectory mMapDirectory = adjustSharedArenaGrouping(new MMapDirectory(location, lockFactory));
         return new SmbDirectoryWrapper(
-            setPreload(
-                new MMapDirectory(location, lockFactory),
-                new HashSet<>(indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING))
-            )
+            setPreload(mMapDirectory, new HashSet<>(indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING)))
         );
     }
 }

server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java

@@ -42,8 +42,28 @@
 import java.util.Set;
 import java.util.function.BiPredicate;
 
+import static org.apache.lucene.store.MMapDirectory.SHARED_ARENA_MAX_PERMITS_SYSPROP;
+
 public class FsDirectoryFactory implements IndexStorePlugin.DirectoryFactory {
 
+    private static final int sharedArenaMaxPermits;
+    static {
+        String prop = System.getProperty(SHARED_ARENA_MAX_PERMITS_SYSPROP);
+        int value = 1;
+        if (prop != null) {
+            try {
+                value = Integer.parseInt(prop); // ensure it's a valid integer
+            } catch (NumberFormatException e) {
+                Logger logger = LogManager.getLogger(FsDirectoryFactory.class);
+                logger.warn(
+                    () -> "unable to parse system property [" + SHARED_ARENA_MAX_PERMITS_SYSPROP + "] with value [" + prop + "]",
+                    e
+                );
+            }
+        }
+        sharedArenaMaxPermits = value; // default to 1
+    }
+
     private static final Logger Log = LogManager.getLogger(FsDirectoryFactory.class);
     private static final FeatureFlag MADV_RANDOM_FEATURE_FLAG = new FeatureFlag("madv_random");
 
@@ -78,6 +98,7 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index
                 // Use Lucene defaults
                 final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory);
                 if (primaryDirectory instanceof MMapDirectory mMapDirectory) {
+                    mMapDirectory = adjustSharedArenaGrouping(mMapDirectory);
                     Directory dir = new HybridDirectory(lockFactory, setPreload(mMapDirectory, preLoadExtensions));
                     if (MADV_RANDOM_FEATURE_FLAG.isEnabled() == false) {
                         dir = disableRandomAdvice(dir);
@@ -87,7 +108,8 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index
                     return primaryDirectory;
                 }
             case MMAPFS:
-                Directory dir = setPreload(new MMapDirectory(location, lockFactory), preLoadExtensions);
+                MMapDirectory mMapDirectory = adjustSharedArenaGrouping(new MMapDirectory(location, lockFactory));
+                Directory dir = setPreload(mMapDirectory, preLoadExtensions);
                 if (MADV_RANDOM_FEATURE_FLAG.isEnabled() == false) {
                     dir = disableRandomAdvice(dir);
                 }
@@ -107,6 +129,13 @@ public MMapDirectory setPreload(MMapDirectory mMapDirectory, Set<String> preLoad
         return mMapDirectory;
     }
 
+    public MMapDirectory adjustSharedArenaGrouping(MMapDirectory mMapDirectory) {
+        if (sharedArenaMaxPermits <= 1) {
+            mMapDirectory.setGroupingFunction(MMapDirectory.NO_GROUPING);
+        }
+        return mMapDirectory;
+    }
+
     /** Gets a preload function based on the given preLoadExtensions. */
     static BiPredicate<String, IOContext> getPreloadFunc(Set<String> preLoadExtensions) {
         if (preLoadExtensions.isEmpty() == false) {