diff --git a/changelog/unreleased/puneet/SOLR-17942-raising-ramPerThreadHardLimit.yml b/changelog/unreleased/puneet/SOLR-17942-raising-ramPerThreadHardLimit.yml new file mode 100644 index 00000000000..ec735e3599e --- /dev/null +++ b/changelog/unreleased/puneet/SOLR-17942-raising-ramPerThreadHardLimit.yml @@ -0,0 +1,8 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: Raising the per ramPerThreadHardLimit to be configurable >2GB +type: other # added, changed, fixed, deprecated, removed, dependency_update, security, other +authors: + - name: punAhuja +links: + - name: SOLR-17942 + url: https://issues.apache.org/jira/browse/SOLR-17942 diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 7b0393fd50e..d0024f6bc2a 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -87,6 +87,8 @@ New Features * SOLR-17815: Add parameter to regulate for ACORN-based filtering in vector search. (Anna Ruggero, Alessandro Benedetti) +* SOLR-17942: Raising the per ramPerThreadHardLimit to be configurable >2GB (Puneet Ahuja) + Improvements --------------------- diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java index 2842c5acb2e..0eedd1873d4 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; import java.util.Collections; import java.util.Map; import org.apache.lucene.analysis.Analyzer; @@ -34,6 +35,7 @@ import org.apache.solr.common.ConfigNode; import org.apache.solr.common.MapSerializable; import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SuppressForbidden; import org.apache.solr.core.DirectoryFactory; import org.apache.solr.core.PluginInfo; import org.apache.solr.core.SolrConfig; @@ -251,7 +253,11 @@ public IndexWriterConfig toIndexWriterConfig(SolrCore core) throws IOException { if (ramBufferSizeMB != -1) iwc.setRAMBufferSizeMB(ramBufferSizeMB); if (ramPerThreadHardLimitMB != -1) { - iwc.setRAMPerThreadHardLimitMB(ramPerThreadHardLimitMB); + if (ramPerThreadHardLimitMB > 2048) { + setPerThreadRAMLimitViaVarHandle(iwc, ramPerThreadHardLimitMB); + } else { + iwc.setRAMPerThreadHardLimitMB(ramPerThreadHardLimitMB); + } } if (maxCommitMergeWaitMillis > 0) { @@ -351,4 +357,33 @@ private MergeScheduler buildMergeScheduler(SolrResourceLoader resourceLoader) { return scheduler; } + + @SuppressForbidden(reason = "Need to override Lucene's 2GB per-thread limit for large datasets") + private static void setPerThreadRAMLimitViaVarHandle(IndexWriterConfig config, int limitMB) { + MethodHandles.Lookup lookup = MethodHandles.lookup(); + VarHandle fieldHandle = null; + Class currentClass = config.getClass(); + + // Traverse the hierarchy to find the field + while (currentClass != null && fieldHandle == null) { + try { + MethodHandles.Lookup privateLookup = MethodHandles.privateLookupIn(currentClass, lookup); + fieldHandle = privateLookup.findVarHandle(currentClass, "perThreadHardLimitMB", int.class); + } catch (IllegalAccessException | NoSuchFieldException e) { + currentClass = currentClass.getSuperclass(); + } + } + + if (fieldHandle == null) { + log.error("Could not find VarHandle for perThreadHardLimitMB field"); + return; + } + + try { + fieldHandle.set(config, limitMB); + log.info("Set perThreadHardLimitMB to {} MB via VarHandle", limitMB); + } catch (RuntimeException | Error e) { + log.error("Failed to set per-thread RAM limit via VarHandle", e); + } + } } diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc index a06851de955..9ee5b9bccf5 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc @@ -77,7 +77,7 @@ The default is `false`. === ramPerThreadHardLimitMB Sets the maximum memory (defined in megabytes) consumption per thread triggering a forced flush if exceeded. -The given value must be greater than `0` and less than `2048` MB (2GB). +The given value must be greater than `0`. [source,xml] ---- @@ -86,6 +86,8 @@ The given value must be greater than `0` and less than `2048` MB (2GB). NOTE: This is an expert level parameter as it triggers forced flush even if `ramBufferSizeMB` has not been exceeded. +WARNING: Values greater than 2048 MB (2GB) will bypass Lucene's built-in limit using reflection. This is an advanced feature that should only be used by experienced administrators who understand the memory implications. + == Merging Index Segments The following settings define when segments are merged.