Skip to content

Commit 2b13cff

Browse files
authored
Bypass MMap arena grouping as this has caused issues with too many regions being mapped (elastic#135012) (elastic#135115)
There is a JDK issue where closing sharedArenas from many threads can significantly harm performance. This ref-counting of shared arenas was designed as a way to get around this performance issue. However, we have noticed a significant increase in leaks and issues with mmap regions since this change. https://bugs.openjdk.org/browse/JDK-8335480 should have helped the performance impact of closing shared arenas (though possibly not fully mitigated it). I am proposing we turn off the grouping as it appears (at least to me), not worth it. I am willing to backdown if we thing other fixes should be done. I also suggest this gets backported to 9.1, 8.19, and is merged into 9.2
1 parent 8a8e816 commit 2b13cff

File tree

4 files changed

+43
-5
lines changed

4 files changed

+43
-5
lines changed

distribution/src/config/jvm.options

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,11 @@
6565
# Lucene 10: apply MADV_NORMAL advice to enable more aggressive readahead
6666
-Dorg.apache.lucene.store.defaultReadAdvice=normal
6767

68+
# Lucene provides a mechanism for shared mmapped arenas to be referenced between multiple threads
69+
# this is to get around potential performance issues when closing shared arenas on many threads
70+
# default to 1 to disable this feature
71+
-Dorg.apache.lucene.store.MMapDirectory.sharedArenaMaxPermits=1
72+
6873
## heap dumps
6974

7075
# generate a heap dump when an allocation from the Java heap fails; heap dumps

docs/changelog/135012.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 135012
2+
summary: Bypass MMap arena grouping as this has caused issues with too many regions
3+
being mapped
4+
area: "Engine"
5+
type: bug
6+
issues: []

plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,9 @@ public final class SmbMmapFsDirectoryFactory extends FsDirectoryFactory {
2424

2525
@Override
2626
protected Directory newFSDirectory(Path location, LockFactory lockFactory, IndexSettings indexSettings) throws IOException {
27+
MMapDirectory mMapDirectory = adjustSharedArenaGrouping(new MMapDirectory(location, lockFactory));
2728
return new SmbDirectoryWrapper(
28-
setPreload(
29-
new MMapDirectory(location, lockFactory),
30-
new HashSet<>(indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING))
31-
)
29+
setPreload(mMapDirectory, new HashSet<>(indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING)))
3230
);
3331
}
3432
}

server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,28 @@
4242
import java.util.Set;
4343
import java.util.function.BiPredicate;
4444

45+
import static org.apache.lucene.store.MMapDirectory.SHARED_ARENA_MAX_PERMITS_SYSPROP;
46+
4547
public class FsDirectoryFactory implements IndexStorePlugin.DirectoryFactory {
4648

49+
private static final int sharedArenaMaxPermits;
50+
static {
51+
String prop = System.getProperty(SHARED_ARENA_MAX_PERMITS_SYSPROP);
52+
int value = 1;
53+
if (prop != null) {
54+
try {
55+
value = Integer.parseInt(prop); // ensure it's a valid integer
56+
} catch (NumberFormatException e) {
57+
Logger logger = LogManager.getLogger(FsDirectoryFactory.class);
58+
logger.warn(
59+
() -> "unable to parse system property [" + SHARED_ARENA_MAX_PERMITS_SYSPROP + "] with value [" + prop + "]",
60+
e
61+
);
62+
}
63+
}
64+
sharedArenaMaxPermits = value; // default to 1
65+
}
66+
4767
private static final Logger Log = LogManager.getLogger(FsDirectoryFactory.class);
4868
private static final FeatureFlag MADV_RANDOM_FEATURE_FLAG = new FeatureFlag("madv_random");
4969

@@ -78,6 +98,7 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index
7898
// Use Lucene defaults
7999
final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory);
80100
if (primaryDirectory instanceof MMapDirectory mMapDirectory) {
101+
mMapDirectory = adjustSharedArenaGrouping(mMapDirectory);
81102
Directory dir = new HybridDirectory(lockFactory, setPreload(mMapDirectory, preLoadExtensions));
82103
if (MADV_RANDOM_FEATURE_FLAG.isEnabled() == false) {
83104
dir = disableRandomAdvice(dir);
@@ -87,7 +108,8 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index
87108
return primaryDirectory;
88109
}
89110
case MMAPFS:
90-
Directory dir = setPreload(new MMapDirectory(location, lockFactory), preLoadExtensions);
111+
MMapDirectory mMapDirectory = adjustSharedArenaGrouping(new MMapDirectory(location, lockFactory));
112+
Directory dir = setPreload(mMapDirectory, preLoadExtensions);
91113
if (MADV_RANDOM_FEATURE_FLAG.isEnabled() == false) {
92114
dir = disableRandomAdvice(dir);
93115
}
@@ -107,6 +129,13 @@ public MMapDirectory setPreload(MMapDirectory mMapDirectory, Set<String> preLoad
107129
return mMapDirectory;
108130
}
109131

132+
public MMapDirectory adjustSharedArenaGrouping(MMapDirectory mMapDirectory) {
133+
if (sharedArenaMaxPermits <= 1) {
134+
mMapDirectory.setGroupingFunction(MMapDirectory.NO_GROUPING);
135+
}
136+
return mMapDirectory;
137+
}
138+
110139
/** Gets a preload function based on the given preLoadExtensions. */
111140
static BiPredicate<String, IOContext> getPreloadFunc(Set<String> preLoadExtensions) {
112141
if (preLoadExtensions.isEmpty() == false) {

0 commit comments

Comments
 (0)