Fst lucene90

lucene/CHANGES.txt

@@ -147,6 +147,8 @@ Optimizations
 
 * GITHUB#12552: Make FSTPostingsFormat load FSTs off-heap. (Tony X)
 
+* GITHUB#12985: Make Lucene90BlockTreePostingsFormat to build FST off-heap. (Anh Dung Bui)
+
 Bug Fixes
 ---------------------
 

lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java

@@ -48,19 +48,23 @@ public final class Lucene90RWPostingsFormat extends PostingsFormat {
 
   private final int minTermBlockSize;
   private final int maxTermBlockSize;
+  private long blockHeapSizeLimitBytes;
 
   /** Creates {@code Lucene90RWPostingsFormat} with default settings. */
   public Lucene90RWPostingsFormat() {
     this(
         Lucene90BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,
-        Lucene90BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
+        Lucene90BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE,
+        Lucene90BlockTreeTermsWriter.DEFAULT_BLOCK_HEAP_LIMIT_BYTES);
   }
 
-  public Lucene90RWPostingsFormat(int minTermBlockSize, int maxTermBlockSize) {
+  public Lucene90RWPostingsFormat(
+      int minTermBlockSize, int maxTermBlockSize, long blockHeapSizeLimitBytes) {
     super("Lucene90");
     Lucene90BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);
     this.minTermBlockSize = minTermBlockSize;
     this.maxTermBlockSize = maxTermBlockSize;
+    this.blockHeapSizeLimitBytes = blockHeapSizeLimitBytes;
   }
 
   @Override
@@ -79,7 +83,8 @@ public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException
               postingsWriter,
               minTermBlockSize,
               maxTermBlockSize,
-              Lucene90BlockTreeTermsReader.VERSION_START);
+              Lucene90BlockTreeTermsReader.VERSION_START,
+              blockHeapSizeLimitBytes);
       success = true;
       return ret;
     } finally {

lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90PostingsFormat.java

@@ -26,6 +26,7 @@
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
 import org.apache.lucene.codecs.lucene90.blocktree.FieldReader;
+import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
 import org.apache.lucene.codecs.lucene90.blocktree.Stats;
 import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;
 import org.apache.lucene.codecs.lucene99.Lucene99SkipWriter;
@@ -45,7 +46,27 @@
 import org.apache.lucene.tests.util.TestUtil;
 
 public class TestLucene90PostingsFormat extends BasePostingsFormatTestCase {
-  private final Codec codec = TestUtil.alwaysPostingsFormat(new Lucene90RWPostingsFormat());
+  private final Codec codec =
+      TestUtil.alwaysPostingsFormat(
+          new Lucene90RWPostingsFormat(
+              Lucene90BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,
+              Lucene90BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE,
+              getBlockHeapSizeLimitBytes()));
+
+  private static long getBlockHeapSizeLimitBytes() {
+    // randomize the block heap max size between 3 states:
+    // - 0, effectively disable on-heap FST and always use off-heap
+    // - DEFAULT_BLOCK_HEAP_LIMIT_BYTES
+    // - a random number between 0 and DEFAULT_BLOCK_HEAP_LIMIT_BYTES
+    int r = random().nextInt(2);
+    if (r == 0) {
+      return 0;
+    }
+    if (r == 1) {
+      return Lucene90BlockTreeTermsWriter.DEFAULT_BLOCK_HEAP_LIMIT_BYTES;
+    }
+    return random().nextLong(Lucene90BlockTreeTermsWriter.DEFAULT_BLOCK_HEAP_LIMIT_BYTES);
+  }
 
   @Override
   protected Codec getCodec() {