SOLR-17942: Raising configurable RAM per thread hard limit using reflection

changelog/unreleased/puneet/SOLR-17942-raising-ramPerThreadHardLimit.yml

@@ -0,0 +1,8 @@
+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc
+title: Raising the per ramPerThreadHardLimit to be configurable >2GB
+type: other # added, changed, fixed, deprecated, removed, dependency_update, security, other
+authors:
+  - name: punAhuja
+links:
+  - name: SOLR-17942
+    url: https://issues.apache.org/jira/browse/SOLR-17942

solr/CHANGES.txt

@@ -87,6 +87,8 @@ New Features
 
 * SOLR-17815: Add parameter to regulate for ACORN-based filtering in vector search. (Anna Ruggero, Alessandro Benedetti)
 
+* SOLR-17942: Raising the per ramPerThreadHardLimit to be configurable >2GB (Puneet Ahuja)
+
 Improvements
 ---------------------
 

solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java

@@ -20,6 +20,7 @@
 
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
+import java.lang.invoke.VarHandle;
 import java.util.Collections;
 import java.util.Map;
 import org.apache.lucene.analysis.Analyzer;
@@ -34,6 +35,7 @@
 import org.apache.solr.common.ConfigNode;
 import org.apache.solr.common.MapSerializable;
 import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SuppressForbidden;
 import org.apache.solr.core.DirectoryFactory;
 import org.apache.solr.core.PluginInfo;
 import org.apache.solr.core.SolrConfig;
@@ -251,7 +253,11 @@ public IndexWriterConfig toIndexWriterConfig(SolrCore core) throws IOException {
     if (ramBufferSizeMB != -1) iwc.setRAMBufferSizeMB(ramBufferSizeMB);
 
     if (ramPerThreadHardLimitMB != -1) {
-      iwc.setRAMPerThreadHardLimitMB(ramPerThreadHardLimitMB);
+      if (ramPerThreadHardLimitMB > 2048) {
+        setPerThreadRAMLimitViaVarHandle(iwc, ramPerThreadHardLimitMB);
+      } else {
+        iwc.setRAMPerThreadHardLimitMB(ramPerThreadHardLimitMB);
+      }
     }
 
     if (maxCommitMergeWaitMillis > 0) {
@@ -351,4 +357,33 @@ private MergeScheduler buildMergeScheduler(SolrResourceLoader resourceLoader) {
 
     return scheduler;
   }
+
+  @SuppressForbidden(reason = "Need to override Lucene's 2GB per-thread limit for large datasets")
+  private static void setPerThreadRAMLimitViaVarHandle(IndexWriterConfig config, int limitMB) {
+    MethodHandles.Lookup lookup = MethodHandles.lookup();
+    VarHandle fieldHandle = null;
+    Class<?> currentClass = config.getClass();
+
+    // Traverse the hierarchy to find the field
+    while (currentClass != null && fieldHandle == null) {
+      try {
+        MethodHandles.Lookup privateLookup = MethodHandles.privateLookupIn(currentClass, lookup);
+        fieldHandle = privateLookup.findVarHandle(currentClass, "perThreadHardLimitMB", int.class);
+      } catch (IllegalAccessException | NoSuchFieldException e) {
+        currentClass = currentClass.getSuperclass();
+      }
+    }
+
+    if (fieldHandle == null) {
+      log.error("Could not find VarHandle for perThreadHardLimitMB field");
+      return;
+    }
+
+    try {
+      fieldHandle.set(config, limitMB);
+      log.info("Set perThreadHardLimitMB to {} MB via VarHandle", limitMB);
+    } catch (RuntimeException | Error e) {
+      log.error("Failed to set per-thread RAM limit via VarHandle", e);
+    }
+  }
 }

solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc

@@ -77,7 +77,7 @@ The default is `false`.
 === ramPerThreadHardLimitMB
 
 Sets the maximum memory (defined in megabytes) consumption per thread triggering a forced flush if exceeded.
-The given value must be greater than `0` and less than `2048` MB (2GB).
+The given value must be greater than `0`.
 
 [source,xml]
 ----
@@ -86,6 +86,8 @@ The given value must be greater than `0` and less than `2048` MB (2GB).
 
 NOTE: This is an expert level parameter as it triggers forced flush even if `ramBufferSizeMB` has not been exceeded.
 
+WARNING: Values greater than 2048 MB (2GB) will bypass Lucene's built-in limit using reflection. This is an advanced feature that should only be used by experienced administrators who understand the memory implications.
+
 == Merging Index Segments
 
 The following settings define when segments are merged.