Merge branch 'remove-fst-ctor' into pr-12543-1

dungba88 · dungba88 · commit f8a6fe7fd623 · 2023-11-22T13:01:29.000+09:00
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java
@@ -16,6 +16,8 @@
  */
 package org.apache.lucene.analysis.ja.dict;
 
+import static org.apache.lucene.util.fst.FST.readMetadata;
+
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -103,7 +105,7 @@ private TokenInfoDictionary(
     FST<Long> fst;
     try (InputStream is = new BufferedInputStream(fstResource.get())) {
       DataInput in = new InputStreamDataInput(is);
-      fst = new FST<>(in, in, PositiveIntOutputs.getSingleton());
+      fst = new FST<>(readMetadata(in, PositiveIntOutputs.getSingleton()), in);
     }
     // TODO: some way to configure?
     this.fst = new TokenInfoFST(fst, true);
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary.java
@@ -16,6 +16,8 @@
  */
 package org.apache.lucene.analysis.ko.dict;
 
+import static org.apache.lucene.util.fst.FST.readMetadata;
+
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -102,7 +104,7 @@ private TokenInfoDictionary(
     FST<Long> fst;
     try (InputStream is = new BufferedInputStream(fstResource.get())) {
       DataInput in = new InputStreamDataInput(is);
-      fst = new FST<>(in, in, PositiveIntOutputs.getSingleton());
+      fst = new FST<>(readMetadata(in, PositiveIntOutputs.getSingleton()), in);
     }
     this.fst = new TokenInfoFST(fst);
   }
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/FieldReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/FieldReader.java
@@ -16,6 +16,8 @@
  */
 package org.apache.lucene.backward_codecs.lucene40.blocktree;
 
+import static org.apache.lucene.util.fst.FST.readMetadata;
+
 import java.io.IOException;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexOptions;
@@ -89,9 +91,17 @@ public final class FieldReader extends Terms {
     final IndexInput clone = indexIn.clone();
     clone.seek(indexStartFP);
     if (metaIn == indexIn) { // Only true before Lucene 8.6
-      index = new FST<>(clone, clone, ByteSequenceOutputs.getSingleton(), new OffHeapFSTStore());
+      index =
+          new FST<>(
+              readMetadata(clone, ByteSequenceOutputs.getSingleton()),
+              clone,
+              new OffHeapFSTStore());
     } else {
-      index = new FST<>(metaIn, clone, ByteSequenceOutputs.getSingleton(), new OffHeapFSTStore());
+      index =
+          new FST<>(
+              readMetadata(metaIn, ByteSequenceOutputs.getSingleton()),
+              clone,
+              new OffHeapFSTStore());
     }
     /*
      if (false) {
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java
@@ -16,6 +16,8 @@
  */
 package org.apache.lucene.codecs.blockterms;
 
+import static org.apache.lucene.util.fst.FST.readMetadata;
+
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
@@ -154,7 +156,7 @@ private final class FieldIndexData implements Accountable {
     public FieldIndexData(IndexInput in, FieldInfo fieldInfo, long indexStart) throws IOException {
       IndexInput clone = in.clone();
       clone.seek(indexStart);
-      fst = new FST<>(clone, clone, fstOutputs);
+      fst = new FST<>(readMetadata(clone, fstOutputs), clone);
       clone.close();
 
       /*
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsFieldReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsFieldReader.java
@@ -16,6 +16,8 @@
  */
 package org.apache.lucene.codecs.blocktreeords;
 
+import static org.apache.lucene.util.fst.FST.readMetadata;
+
 import java.io.IOException;
 import org.apache.lucene.codecs.blocktreeords.FSTOrdsOutputs.Output;
 import org.apache.lucene.index.FieldInfo;
@@ -85,7 +87,7 @@ final class OrdsFieldReader extends Terms {
       final IndexInput clone = indexIn.clone();
       // System.out.println("start=" + indexStartFP + " field=" + fieldInfo.name);
       clone.seek(indexStartFP);
-      index = new FST<>(clone, clone, OrdsBlockTreeTermsWriter.FST_OUTPUTS);
+      index = new FST<>(readMetadata(clone, OrdsBlockTreeTermsWriter.FST_OUTPUTS), clone);
 
       /*
       if (true) {
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
@@ -194,7 +194,8 @@ final class TermsReader extends Terms {
       this.sumDocFreq = sumDocFreq;
       this.docCount = docCount;
       OffHeapFSTStore offHeapFSTStore = new OffHeapFSTStore();
-      this.dict = new FST<>(in, in, new FSTTermOutputs(fieldInfo), offHeapFSTStore);
+      FSTTermOutputs outputs = new FSTTermOutputs(fieldInfo);
+      this.dict = new FST<>(FST.readMetadata(in, outputs), in, offHeapFSTStore);
       in.skipBytes(offHeapFSTStore.size());
     }
 
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/FSTDictionary.java b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/FSTDictionary.java
@@ -89,10 +89,11 @@ protected static FSTDictionary read(
       isFSTOnHeap = true;
     }
     PositiveIntOutputs fstOutputs = PositiveIntOutputs.getSingleton();
+    FST.FSTMetadata<Long> metadata = FST.readMetadata(fstDataInput, fstOutputs);
     FST<Long> fst =
         isFSTOnHeap
-            ? new FST<>(fstDataInput, fstDataInput, fstOutputs)
-            : new FST<>(fstDataInput, fstDataInput, fstOutputs, new OffHeapFSTStore());
+            ? new FST<>(metadata, fstDataInput)
+            : new FST<>(metadata, fstDataInput, new OffHeapFSTStore());
     return new FSTDictionary(fst);
   }
 
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/FieldReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/FieldReader.java
@@ -91,7 +91,11 @@ public final class FieldReader extends Terms {
     // Initialize FST always off-heap.
     final IndexInput clone = indexIn.clone();
     clone.seek(indexStartFP);
-    index = new FST<>(metaIn, clone, ByteSequenceOutputs.getSingleton(), new OffHeapFSTStore());
+    index =
+        new FST<>(
+            FST.readMetadata(metaIn, ByteSequenceOutputs.getSingleton()),
+            clone,
+            new OffHeapFSTStore());
     /*
      if (false) {
      final String dotFileName = segment + "_" + fieldInfo.name + ".dot";
diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
@@ -402,34 +402,23 @@ private static boolean flag(int flags, int bit) {
    * Load a previously saved FST with a DataInput for metdata using an {@link OnHeapFSTStore} with
    * maxBlockBits set to {@link #DEFAULT_MAX_BLOCK_BITS}
    */
-  public FST(DataInput metaIn, DataInput in, Outputs<T> outputs) throws IOException {
-    this(metaIn, in, outputs, new OnHeapFSTStore(DEFAULT_MAX_BLOCK_BITS));
-  }
-
-  /**
-   * Load a previously saved FST with a DataInput for metdata and a FSTStore. If using {@link
-   * OnHeapFSTStore}, setting maxBlockBits allows you to control the size of the byte[] pages used
-   * to hold the FST bytes.
-   */
-  public FST(DataInput metaIn, DataInput in, Outputs<T> outputs, FSTStore fstStore)
-      throws IOException {
-    this(readMetadata(metaIn, outputs), in, outputs, fstStore);
+  public FST(FSTMetadata<T> metadata, DataInput in) throws IOException {
+    this(metadata, in, new OnHeapFSTStore(DEFAULT_MAX_BLOCK_BITS));
   }
 
   /**
    * Load a previously saved FST with a metdata object and a FSTStore. If using {@link
    * OnHeapFSTStore}, setting maxBlockBits allows you to control the size of the byte[] pages used
    * to hold the FST bytes.
    */
-  public FST(FSTMetadata<T> metadata, DataInput in, Outputs<T> outputs, FSTStore fstStore)
-      throws IOException {
-    this(metadata, outputs, fstStore.init(in, metadata.numBytes));
+  public FST(FSTMetadata<T> metadata, DataInput in, FSTStore fstStore) throws IOException {
+    this(metadata, fstStore.init(in, metadata.numBytes));
   }
 
   /** Create the FST with a metadata object and a FSTReader. */
-  FST(FSTMetadata<T> metadata, Outputs<T> outputs, FSTReader fstReader) {
+  FST(FSTMetadata<T> metadata, FSTReader fstReader) {
     this.metadata = metadata;
-    this.outputs = outputs;
+    this.outputs = metadata.outputs;
     this.fstReader = fstReader;
   }
 
@@ -491,7 +480,7 @@ public static <T> FSTMetadata<T> readMetadata(DataInput metaIn, Outputs<T> outpu
     }
     long startNode = metaIn.readVLong();
     long numBytes = metaIn.readVLong();
-    return new FSTMetadata<>(inputType, emptyOutput, startNode, version, numBytes);
+    return new FSTMetadata<>(inputType, outputs, emptyOutput, startNode, version, numBytes);
   }
 
   @Override
@@ -594,7 +583,7 @@ public void save(final Path path) throws IOException {
   public static <T> FST<T> read(Path path, Outputs<T> outputs) throws IOException {
     try (InputStream is = Files.newInputStream(path)) {
       DataInput in = new InputStreamDataInput(new BufferedInputStream(is));
-      return new FST<>(in, in, outputs);
+      return new FST<>(readMetadata(in, outputs), in);
     }
   }
 
@@ -1230,6 +1219,7 @@ public abstract static class BytesReader extends DataInput {
    */
   public static final class FSTMetadata<T> {
     final INPUT_TYPE inputType;
+    final Outputs<T> outputs;
     final int version;
     // if non-null, this FST accepts the empty string and
     // produces this output
@@ -1238,8 +1228,14 @@ public static final class FSTMetadata<T> {
     long numBytes;
 
     public FSTMetadata(
-        INPUT_TYPE inputType, T emptyOutput, long startNode, int version, long numBytes) {
+        INPUT_TYPE inputType,
+        Outputs<T> outputs,
+        T emptyOutput,
+        long startNode,
+        int version,
+        long numBytes) {
       this.inputType = inputType;
+      this.outputs = outputs;
       this.emptyOutput = emptyOutput;
       this.startNode = startNode;
       this.version = version;
diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java
@@ -156,8 +156,7 @@ private FSTCompiler(
     this.dataOutput = dataOutput;
     fst =
         new FST<>(
-            new FST.FSTMetadata<>(inputType, null, -1, VERSION_CURRENT, 0),
-            outputs,
+            new FST.FSTMetadata<>(inputType, outputs, null, -1, VERSION_CURRENT, 0),
             toFSTReader(dataOutput));
     if (suffixRAMLimitMB < 0) {
       throw new IllegalArgumentException("ramLimitMB must be >= 0; got: " + suffixRAMLimitMB);
@@ -774,21 +773,6 @@ private void freezeTail(int prefixLenPlus1) throws IOException {
    * IntSequenceOutputs}) then you cannot reuse across calls.
    */
   public void add(IntsRef input, T output) throws IOException {
-    /*
-    if (DEBUG) {
-      BytesRef b = new BytesRef(input.length);
-      for(int x=0;x<input.length;x++) {
-        b.bytes[x] = (byte) input.ints[x];
-      }
-      b.length = input.length;
-      if (output == NO_OUTPUT) {
-        System.out.println("\nFST ADD: input=" + toString(b) + " " + b);
-      } else {
-        System.out.println("\nFST ADD: input=" + toString(b) + " " + b + " output=" + fst.outputs.outputToString(output));
-      }
-    }
-    */
-
     // De-dup NO_OUTPUT since it must be a singleton:
     if (output.equals(NO_OUTPUT)) {
       output = NO_OUTPUT;
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java b/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
@@ -141,7 +141,7 @@ public void test() throws Exception {
             fst.save(out, out);
             out.close();
             IndexInput in = dir.openInput("fst", IOContext.DEFAULT);
-            fst = new FST<>(in, in, outputs);
+            fst = new FST<>(FST.readMetadata(in, outputs), in);
             in.close();
           } else {
             dir.deleteFile("fst");
@@ -228,7 +228,7 @@ public void test() throws Exception {
             fst.save(out, out);
             out.close();
             IndexInput in = dir.openInput("fst", IOContext.DEFAULT);
-            fst = new FST<>(in, in, outputs);
+            fst = new FST<>(FST.readMetadata(in, outputs), in);
             in.close();
           } else {
             dir.deleteFile("fst");
@@ -320,7 +320,7 @@ public void test() throws Exception {
             fst.save(out, out);
             out.close();
             IndexInput in = dir.openInput("fst", IOContext.DEFAULT);
-            fst = new FST<>(in, in, outputs);
+            fst = new FST<>(FST.readMetadata(in, outputs), in);
             in.close();
           } else {
             dir.deleteFile("fst");
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTDirectAddressing.java b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTDirectAddressing.java
@@ -16,6 +16,8 @@
  */
 package org.apache.lucene.util.fst;
 
+import static org.apache.lucene.util.fst.FST.readMetadata;
+
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -218,7 +220,7 @@ public static void main(String... args) throws Exception {
   private static void countFSTArcs(String fstFilePath) throws IOException {
     byte[] buf = Files.readAllBytes(Paths.get(fstFilePath));
     DataInput in = new ByteArrayDataInput(buf);
-    FST<BytesRef> fst = new FST<>(in, in, ByteSequenceOutputs.getSingleton());
+    FST<BytesRef> fst = new FST<>(readMetadata(in, ByteSequenceOutputs.getSingleton()), in);
     BytesRefFSTEnum<BytesRef> fstEnum = new BytesRefFSTEnum<>(fst);
     int binarySearchArcCount = 0,
         directAddressingArcCount = 0,
@@ -285,7 +287,8 @@ private static void recompileAndWalk(String fstFilePath) throws IOException {
 
       System.out.println("Reading FST");
       long startTimeMs = System.nanoTime();
-      FST<CharsRef> originalFst = new FST<>(in, in, CharSequenceOutputs.getSingleton());
+      FST<CharsRef> originalFst =
+          new FST<>(readMetadata(in, CharSequenceOutputs.getSingleton()), in);
       long endTimeMs = System.nanoTime();
       System.out.println(
           "time = " + TimeUnit.NANOSECONDS.toMillis(endTimeMs - startTimeMs) + " ms");
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
@@ -1225,7 +1225,7 @@ public void testSaveDifferentMetaOut() throws Exception {
 
     // load the FST, which will force it to use FSTStore instead of BytesStore
     ByteArrayDataInput in = new ByteArrayDataInput(outOS.toByteArray());
-    FST<Long> loadedFST = new FST<>(in, in, outputs);
+    FST<Long> loadedFST = new FST<>(FST.readMetadata(in, outputs), in);
 
     // now save the FST again, this time to different DataOutput for meta
     ByteArrayOutputStream metdataOS = new ByteArrayOutputStream();
@@ -1237,7 +1237,7 @@ public void testSaveDifferentMetaOut() throws Exception {
     // finally load it again
     ByteArrayDataInput metaIn = new ByteArrayDataInput(metdataOS.toByteArray());
     ByteArrayDataInput dataIn = new ByteArrayDataInput(dataOS.toByteArray());
-    loadedFST = new FST<>(metaIn, dataIn, outputs);
+    loadedFST = new FST<>(FST.readMetadata(metaIn, outputs), dataIn);
 
     assertEquals(22L, Util.get(loadedFST, Util.toIntsRef(newBytesRef("aab"), scratch)).longValue());
     assertEquals(7L, Util.get(loadedFST, Util.toIntsRef(newBytesRef("aac"), scratch)).longValue());
@@ -1298,7 +1298,7 @@ public void testNonFinalStopNode() throws Exception {
     out.close();
 
     IndexInput in = dir.openInput("fst", IOContext.DEFAULT);
-    final FST<Long> fst2 = new FST<>(in, in, outputs);
+    final FST<Long> fst2 = new FST<>(FST.readMetadata(in, outputs), in);
     checkStopNodes(fst2, outputs);
     in.close();
     dir.close();
diff --git a/lucene/demo/src/java/org/apache/lucene/demo/knn/KnnVectorDict.java b/lucene/demo/src/java/org/apache/lucene/demo/knn/KnnVectorDict.java
@@ -16,6 +16,8 @@
  */
 package org.apache.lucene.demo.knn;
 
+import static org.apache.lucene.util.fst.FST.readMetadata;
+
 import java.io.BufferedReader;
 import java.io.Closeable;
 import java.io.IOException;
@@ -58,7 +60,7 @@ public class KnnVectorDict implements Closeable {
    */
   public KnnVectorDict(Directory directory, String dictName) throws IOException {
     try (IndexInput fstIn = directory.openInput(dictName + ".fst", IOContext.READ)) {
-      fst = new FST<>(fstIn, fstIn, PositiveIntOutputs.getSingleton());
+      fst = new FST<>(readMetadata(fstIn, PositiveIntOutputs.getSingleton()), fstIn);
     }
 
     vectors = directory.openInput(dictName + ".bin", IOContext.READ);
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/idversion/VersionFieldReader.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/idversion/VersionFieldReader.java
@@ -16,6 +16,8 @@
  */
 package org.apache.lucene.sandbox.codecs.idversion;
 
+import static org.apache.lucene.util.fst.FST.readMetadata;
+
 import java.io.IOException;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexOptions;
@@ -86,7 +88,7 @@ final class VersionFieldReader extends Terms {
       final IndexInput clone = indexIn.clone();
       // System.out.println("start=" + indexStartFP + " field=" + fieldInfo.name);
       clone.seek(indexStartFP);
-      index = new FST<>(clone, clone, VersionBlockTreeTermsWriter.FST_OUTPUTS);
+      index = new FST<>(readMetadata(clone, VersionBlockTreeTermsWriter.FST_OUTPUTS), clone);
 
       /*
         if (false) {
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
@@ -612,12 +612,9 @@ public boolean store(DataOutput output) throws IOException {
   @Override
   public boolean load(DataInput input) throws IOException {
     count = input.readVLong();
-    this.fst =
-        new FST<>(
-            input,
-            input,
-            new PairOutputs<>(
-                PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
+    PairOutputs<Long, BytesRef> outputs =
+        new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton());
+    this.fst = new FST<>(FST.readMetadata(input, outputs), input);
     maxAnalyzedPathsForOneInput = input.readVInt();
     hasPayloads = input.readByte() == 1;
     return true;
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggester.java
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/fst/FSTTester.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/fst/FSTTester.java

Original file line number	Diff line number	Diff line change
`@@ -194,7 +194,8 @@ final class TermsReader extends Terms {`
`194`	`194`	`this.sumDocFreq = sumDocFreq;`
`195`	`195`	`this.docCount = docCount;`
`196`	`196`	`OffHeapFSTStore offHeapFSTStore = new OffHeapFSTStore();`
`197`		`- this.dict = new FST<>(in, in, new FSTTermOutputs(fieldInfo), offHeapFSTStore);`
	`197`	`+ FSTTermOutputs outputs = new FSTTermOutputs(fieldInfo);`
	`198`	`+ this.dict = new FST<>(FST.readMetadata(in, outputs), in, offHeapFSTStore);`
`198`	`199`	`in.skipBytes(offHeapFSTStore.size());`
`199`	`200`	`}`
`200`	`201`
Original file line number	Diff line number	Diff line change
`@@ -89,10 +89,11 @@ protected static FSTDictionary read(`
`89`	`89`	`isFSTOnHeap = true;`
`90`	`90`	`}`
`91`	`91`	`PositiveIntOutputs fstOutputs = PositiveIntOutputs.getSingleton();`
	`92`	`+ FST.FSTMetadata<Long> metadata = FST.readMetadata(fstDataInput, fstOutputs);`
`92`	`93`	`FST<Long> fst =`
`93`	`94`	`isFSTOnHeap`
`94`		`- ? new FST<>(fstDataInput, fstDataInput, fstOutputs)`
`95`		`- : new FST<>(fstDataInput, fstDataInput, fstOutputs, new OffHeapFSTStore());`
	`95`	`+ ? new FST<>(metadata, fstDataInput)`
	`96`	`+ : new FST<>(metadata, fstDataInput, new OffHeapFSTStore());`
`96`	`97`	`return new FSTDictionary(fst);`
`97`	`98`	`}`
`98`	`99`