Move Outputs to FSTMetadata

dungba88 · dungba88 · commit 4f85e19eda26 · 2023-11-14T14:28:27.000+09:00
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java
@@ -16,6 +16,8 @@
  */
 package org.apache.lucene.analysis.ja.dict;
 
+import static org.apache.lucene.util.fst.FST.readMetadata;
+
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -103,8 +105,7 @@ private TokenInfoDictionary(
     FST<Long> fst;
     try (InputStream is = new BufferedInputStream(fstResource.get())) {
       DataInput in = new InputStreamDataInput(is);
-      FST.FSTMetadata<Long> metadata = FST.readMetadata(in, PositiveIntOutputs.getSingleton());
-      fst = new FST<>(metadata, in, PositiveIntOutputs.getSingleton());
+      fst = new FST<>(readMetadata(in, PositiveIntOutputs.getSingleton()), in);
     }
     // TODO: some way to configure?
     this.fst = new TokenInfoFST(fst, true);
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary.java
@@ -16,6 +16,8 @@
  */
 package org.apache.lucene.analysis.ko.dict;
 
+import static org.apache.lucene.util.fst.FST.readMetadata;
+
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -102,8 +104,7 @@ private TokenInfoDictionary(
     FST<Long> fst;
     try (InputStream is = new BufferedInputStream(fstResource.get())) {
       DataInput in = new InputStreamDataInput(is);
-      FST.FSTMetadata<Long> metadata = FST.readMetadata(in, PositiveIntOutputs.getSingleton());
-      fst = new FST<>(metadata, in, PositiveIntOutputs.getSingleton());
+      fst = new FST<>(readMetadata(in, PositiveIntOutputs.getSingleton()), in);
     }
     this.fst = new TokenInfoFST(fst);
   }
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/FieldReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/FieldReader.java
@@ -16,6 +16,8 @@
  */
 package org.apache.lucene.backward_codecs.lucene40.blocktree;
 
+import static org.apache.lucene.util.fst.FST.readMetadata;
+
 import java.io.IOException;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexOptions;
@@ -89,13 +91,17 @@ public final class FieldReader extends Terms {
     final IndexInput clone = indexIn.clone();
     clone.seek(indexStartFP);
     if (metaIn == indexIn) { // Only true before Lucene 8.6
-      FST.FSTMetadata<BytesRef> metadata =
-          FST.readMetadata(clone, ByteSequenceOutputs.getSingleton());
-      index = new FST<>(metadata, clone, ByteSequenceOutputs.getSingleton(), new OffHeapFSTStore());
+      index =
+          new FST<>(
+              readMetadata(clone, ByteSequenceOutputs.getSingleton()),
+              clone,
+              new OffHeapFSTStore());
     } else {
-      FST.FSTMetadata<BytesRef> metadata =
-          FST.readMetadata(metaIn, ByteSequenceOutputs.getSingleton());
-      index = new FST<>(metadata, clone, ByteSequenceOutputs.getSingleton(), new OffHeapFSTStore());
+      index =
+          new FST<>(
+              readMetadata(metaIn, ByteSequenceOutputs.getSingleton()),
+              clone,
+              new OffHeapFSTStore());
     }
     /*
      if (false) {
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java
@@ -16,6 +16,8 @@
  */
 package org.apache.lucene.codecs.blockterms;
 
+import static org.apache.lucene.util.fst.FST.readMetadata;
+
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
@@ -154,7 +156,7 @@ private final class FieldIndexData implements Accountable {
     public FieldIndexData(IndexInput in, FieldInfo fieldInfo, long indexStart) throws IOException {
       IndexInput clone = in.clone();
       clone.seek(indexStart);
-      fst = new FST<>(FST.readMetadata(clone, fstOutputs), clone, fstOutputs);
+      fst = new FST<>(readMetadata(clone, fstOutputs), clone);
       clone.close();
 
       /*
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsFieldReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsFieldReader.java
@@ -16,6 +16,8 @@
  */
 package org.apache.lucene.codecs.blocktreeords;
 
+import static org.apache.lucene.util.fst.FST.readMetadata;
+
 import java.io.IOException;
 import org.apache.lucene.codecs.blocktreeords.FSTOrdsOutputs.Output;
 import org.apache.lucene.index.FieldInfo;
@@ -85,9 +87,7 @@ final class OrdsFieldReader extends Terms {
       final IndexInput clone = indexIn.clone();
       // System.out.println("start=" + indexStartFP + " field=" + fieldInfo.name);
       clone.seek(indexStartFP);
-      FST.FSTMetadata<Output> metadata =
-          FST.readMetadata(clone, OrdsBlockTreeTermsWriter.FST_OUTPUTS);
-      index = new FST<>(metadata, clone, OrdsBlockTreeTermsWriter.FST_OUTPUTS);
+      index = new FST<>(readMetadata(clone, OrdsBlockTreeTermsWriter.FST_OUTPUTS), clone);
 
       /*
       if (true) {
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
@@ -195,7 +195,7 @@ final class TermsReader extends Terms {
       this.docCount = docCount;
       OffHeapFSTStore offHeapFSTStore = new OffHeapFSTStore();
       FSTTermOutputs outputs = new FSTTermOutputs(fieldInfo);
-      this.dict = new FST<>(FST.readMetadata(in, outputs), in, outputs, offHeapFSTStore);
+      this.dict = new FST<>(FST.readMetadata(in, outputs), in, offHeapFSTStore);
       in.skipBytes(offHeapFSTStore.size());
     }
 
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/FSTDictionary.java b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/FSTDictionary.java
@@ -92,8 +92,8 @@ protected static FSTDictionary read(
     FST.FSTMetadata<Long> metadata = FST.readMetadata(fstDataInput, fstOutputs);
     FST<Long> fst =
         isFSTOnHeap
-            ? new FST<>(metadata, fstDataInput, fstOutputs)
-            : new FST<>(metadata, fstDataInput, fstOutputs, new OffHeapFSTStore());
+            ? new FST<>(metadata, fstDataInput)
+            : new FST<>(metadata, fstDataInput, new OffHeapFSTStore());
     return new FSTDictionary(fst);
   }
 
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/FieldReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/FieldReader.java
@@ -91,9 +91,11 @@ public final class FieldReader extends Terms {
     // Initialize FST always off-heap.
     final IndexInput clone = indexIn.clone();
     clone.seek(indexStartFP);
-    FST.FSTMetadata<BytesRef> metadata =
-        FST.readMetadata(metaIn, ByteSequenceOutputs.getSingleton());
-    index = new FST<>(metadata, clone, ByteSequenceOutputs.getSingleton(), new OffHeapFSTStore());
+    index =
+        new FST<>(
+            FST.readMetadata(metaIn, ByteSequenceOutputs.getSingleton()),
+            clone,
+            new OffHeapFSTStore());
     /*
      if (false) {
      final String dotFileName = segment + "_" + fieldInfo.name + ".dot";
diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
@@ -404,24 +404,23 @@ private static boolean flag(int flags, int bit) {
    * Load a previously saved FST with a DataInput for metdata using an {@link OnHeapFSTStore} with
    * maxBlockBits set to {@link #DEFAULT_MAX_BLOCK_BITS}
    */
-  public FST(FSTMetadata<T> metadata, DataInput in, Outputs<T> outputs) throws IOException {
-    this(metadata, in, outputs, new OnHeapFSTStore(DEFAULT_MAX_BLOCK_BITS));
+  public FST(FSTMetadata<T> metadata, DataInput in) throws IOException {
+    this(metadata, in, new OnHeapFSTStore(DEFAULT_MAX_BLOCK_BITS));
   }
 
   /**
    * Load a previously saved FST with a metdata object and a FSTStore. If using {@link
    * OnHeapFSTStore}, setting maxBlockBits allows you to control the size of the byte[] pages used
    * to hold the FST bytes.
    */
-  public FST(FSTMetadata<T> metadata, DataInput in, Outputs<T> outputs, FSTStore fstStore)
-      throws IOException {
-    this(metadata, outputs, fstStore.init(in, metadata.numBytes));
+  public FST(FSTMetadata<T> metadata, DataInput in, FSTStore fstStore) throws IOException {
+    this(metadata, fstStore.init(in, metadata.numBytes));
   }
 
   /** Create the FST with a metadata object and a FSTReader. */
-  FST(FSTMetadata<T> metadata, Outputs<T> outputs, FSTReader fstReader) {
+  FST(FSTMetadata<T> metadata, FSTReader fstReader) {
     this.metadata = metadata;
-    this.outputs = outputs;
+    this.outputs = metadata.outputs;
     this.fstReader = fstReader;
   }
 
@@ -476,7 +475,7 @@ public static <T> FSTMetadata<T> readMetadata(DataInput metaIn, Outputs<T> outpu
     }
     long startNode = metaIn.readVLong();
     long numBytes = metaIn.readVLong();
-    return new FSTMetadata<>(inputType, emptyOutput, startNode, version, numBytes);
+    return new FSTMetadata<>(inputType, outputs, emptyOutput, startNode, version, numBytes);
   }
 
   @Override
@@ -564,7 +563,7 @@ public void save(final Path path) throws IOException {
   public static <T> FST<T> read(Path path, Outputs<T> outputs) throws IOException {
     try (InputStream is = Files.newInputStream(path)) {
       DataInput in = new InputStreamDataInput(new BufferedInputStream(is));
-      return new FST<>(readMetadata(in, outputs), in, outputs);
+      return new FST<>(readMetadata(in, outputs), in);
     }
   }
 
@@ -1192,6 +1191,7 @@ public abstract static class BytesReader extends DataInput {
    */
   public static final class FSTMetadata<T> {
     final INPUT_TYPE inputType;
+    final Outputs<T> outputs;
     final int version;
     // if non-null, this FST accepts the empty string and
     // produces this output
@@ -1200,8 +1200,14 @@ public static final class FSTMetadata<T> {
     long numBytes;
 
     public FSTMetadata(
-        INPUT_TYPE inputType, T emptyOutput, long startNode, int version, long numBytes) {
+        INPUT_TYPE inputType,
+        Outputs<T> outputs,
+        T emptyOutput,
+        long startNode,
+        int version,
+        long numBytes) {
       this.inputType = inputType;
+      this.outputs = outputs;
       this.emptyOutput = emptyOutput;
       this.startNode = startNode;
       this.version = version;
diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java
@@ -144,7 +144,7 @@ private FSTCompiler(
     // pad: ensure no node gets address 0 which is reserved to mean
     // the stop state w/ no arcs
     bytes.writeByte((byte) 0);
-    fst = new FST<>(new FST.FSTMetadata<>(inputType, null, -1, VERSION_CURRENT, 0), outputs, bytes);
+    fst = new FST<>(new FST.FSTMetadata<>(inputType, outputs, null, -1, VERSION_CURRENT, 0), bytes);
     if (suffixRAMLimitMB < 0) {
       throw new IllegalArgumentException("ramLimitMB must be >= 0; got: " + suffixRAMLimitMB);
     } else if (suffixRAMLimitMB > 0) {
@@ -726,21 +726,6 @@ private void freezeTail(int prefixLenPlus1) throws IOException {
    * IntSequenceOutputs}) then you cannot reuse across calls.
    */
   public void add(IntsRef input, T output) throws IOException {
-    /*
-    if (DEBUG) {
-      BytesRef b = new BytesRef(input.length);
-      for(int x=0;x<input.length;x++) {
-        b.bytes[x] = (byte) input.ints[x];
-      }
-      b.length = input.length;
-      if (output == NO_OUTPUT) {
-        System.out.println("\nFST ADD: input=" + toString(b) + " " + b);
-      } else {
-        System.out.println("\nFST ADD: input=" + toString(b) + " " + b + " output=" + fst.outputs.outputToString(output));
-      }
-    }
-    */
-
     // De-dup NO_OUTPUT since it must be a singleton:
     if (output.equals(NO_OUTPUT)) {
       output = NO_OUTPUT;
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java b/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
@@ -140,7 +140,7 @@ public void test() throws Exception {
             fst.save(out, out);
             out.close();
             IndexInput in = dir.openInput("fst", IOContext.DEFAULT);
-            fst = new FST<>(FST.readMetadata(in, outputs), in, outputs);
+            fst = new FST<>(FST.readMetadata(in, outputs), in);
             in.close();
           } else {
             dir.deleteFile("fst");
@@ -226,7 +226,7 @@ public void test() throws Exception {
             fst.save(out, out);
             out.close();
             IndexInput in = dir.openInput("fst", IOContext.DEFAULT);
-            fst = new FST<>(FST.readMetadata(in, outputs), in, outputs);
+            fst = new FST<>(FST.readMetadata(in, outputs), in);
             in.close();
           } else {
             dir.deleteFile("fst");
@@ -317,7 +317,7 @@ public void test() throws Exception {
             fst.save(out, out);
             out.close();
             IndexInput in = dir.openInput("fst", IOContext.DEFAULT);
-            fst = new FST<>(FST.readMetadata(in, outputs), in, outputs);
+            fst = new FST<>(FST.readMetadata(in, outputs), in);
             in.close();
           } else {
             dir.deleteFile("fst");
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTDirectAddressing.java b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTDirectAddressing.java
@@ -16,6 +16,8 @@
  */
 package org.apache.lucene.util.fst;
 
+import static org.apache.lucene.util.fst.FST.readMetadata;
+
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -218,8 +220,7 @@ public static void main(String... args) throws Exception {
   private static void countFSTArcs(String fstFilePath) throws IOException {
     byte[] buf = Files.readAllBytes(Paths.get(fstFilePath));
     DataInput in = new ByteArrayDataInput(buf);
-    FST.FSTMetadata<BytesRef> metadata = FST.readMetadata(in, ByteSequenceOutputs.getSingleton());
-    FST<BytesRef> fst = new FST<>(metadata, in, ByteSequenceOutputs.getSingleton());
+    FST<BytesRef> fst = new FST<>(readMetadata(in, ByteSequenceOutputs.getSingleton()), in);
     BytesRefFSTEnum<BytesRef> fstEnum = new BytesRefFSTEnum<>(fst);
     int binarySearchArcCount = 0,
         directAddressingArcCount = 0,
@@ -286,8 +287,8 @@ private static void recompileAndWalk(String fstFilePath) throws IOException {
 
       System.out.println("Reading FST");
       long startTimeMs = System.nanoTime();
-      FST.FSTMetadata<CharsRef> metadata = FST.readMetadata(in, CharSequenceOutputs.getSingleton());
-      FST<CharsRef> originalFst = new FST<>(metadata, in, CharSequenceOutputs.getSingleton());
+      FST<CharsRef> originalFst =
+          new FST<>(readMetadata(in, CharSequenceOutputs.getSingleton()), in);
       long endTimeMs = System.nanoTime();
       System.out.println(
           "time = " + TimeUnit.NANOSECONDS.toMillis(endTimeMs - startTimeMs) + " ms");
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
@@ -1219,7 +1219,7 @@ public void testSaveDifferentMetaOut() throws Exception {
 
     // load the FST, which will force it to use FSTStore instead of BytesStore
     ByteArrayDataInput in = new ByteArrayDataInput(outOS.toByteArray());
-    FST<Long> loadedFST = new FST<>(FST.readMetadata(in, outputs), in, outputs);
+    FST<Long> loadedFST = new FST<>(FST.readMetadata(in, outputs), in);
 
     // now save the FST again, this time to different DataOutput for meta
     ByteArrayOutputStream metdataOS = new ByteArrayOutputStream();
@@ -1231,7 +1231,7 @@ public void testSaveDifferentMetaOut() throws Exception {
     // finally load it again
     ByteArrayDataInput metaIn = new ByteArrayDataInput(metdataOS.toByteArray());
     ByteArrayDataInput dataIn = new ByteArrayDataInput(dataOS.toByteArray());
-    loadedFST = new FST<>(FST.readMetadata(metaIn, outputs), dataIn, outputs);
+    loadedFST = new FST<>(FST.readMetadata(metaIn, outputs), dataIn);
 
     assertEquals(22L, Util.get(loadedFST, Util.toIntsRef(newBytesRef("aab"), scratch)).longValue());
     assertEquals(7L, Util.get(loadedFST, Util.toIntsRef(newBytesRef("aac"), scratch)).longValue());
@@ -1291,7 +1291,7 @@ public void testNonFinalStopNode() throws Exception {
     out.close();
 
     IndexInput in = dir.openInput("fst", IOContext.DEFAULT);
-    final FST<Long> fst2 = new FST<>(FST.readMetadata(in, outputs), in, outputs);
+    final FST<Long> fst2 = new FST<>(FST.readMetadata(in, outputs), in);
     checkStopNodes(fst2, outputs);
     in.close();
     dir.close();
diff --git a/lucene/demo/src/java/org/apache/lucene/demo/knn/KnnVectorDict.java b/lucene/demo/src/java/org/apache/lucene/demo/knn/KnnVectorDict.java
@@ -16,6 +16,8 @@
  */
 package org.apache.lucene.demo.knn;
 
+import static org.apache.lucene.util.fst.FST.readMetadata;
+
 import java.io.BufferedReader;
 import java.io.Closeable;
 import java.io.IOException;
@@ -58,8 +60,7 @@ public class KnnVectorDict implements Closeable {
    */
   public KnnVectorDict(Directory directory, String dictName) throws IOException {
     try (IndexInput fstIn = directory.openInput(dictName + ".fst", IOContext.READ)) {
-      FST.FSTMetadata<Long> metadata = FST.readMetadata(fstIn, PositiveIntOutputs.getSingleton());
-      fst = new FST<>(metadata, fstIn, PositiveIntOutputs.getSingleton());
+      fst = new FST<>(readMetadata(fstIn, PositiveIntOutputs.getSingleton()), fstIn);
     }
 
     vectors = directory.openInput(dictName + ".bin", IOContext.READ);
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/idversion/VersionFieldReader.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/idversion/VersionFieldReader.java
@@ -16,6 +16,8 @@
  */
 package org.apache.lucene.sandbox.codecs.idversion;
 
+import static org.apache.lucene.util.fst.FST.readMetadata;
+
 import java.io.IOException;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexOptions;
@@ -86,11 +88,7 @@ final class VersionFieldReader extends Terms {
       final IndexInput clone = indexIn.clone();
       // System.out.println("start=" + indexStartFP + " field=" + fieldInfo.name);
       clone.seek(indexStartFP);
-      index =
-          new FST<>(
-              FST.readMetadata(clone, VersionBlockTreeTermsWriter.FST_OUTPUTS),
-              clone,
-              VersionBlockTreeTermsWriter.FST_OUTPUTS);
+      index = new FST<>(readMetadata(clone, VersionBlockTreeTermsWriter.FST_OUTPUTS), clone);
 
       /*
         if (false) {
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
@@ -614,7 +614,7 @@ public boolean load(DataInput input) throws IOException {
     count = input.readVLong();
     PairOutputs<Long, BytesRef> outputs =
         new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton());
-    this.fst = new FST<>(FST.readMetadata(input, outputs), input, outputs);
+    this.fst = new FST<>(FST.readMetadata(input, outputs), input);
     maxAnalyzedPathsForOneInput = input.readVInt();
     hasPayloads = input.readByte() == 1;
     return true;
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggester.java
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/fst/FSTTester.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/fst/FSTTester.java

Original file line number	Diff line number	Diff line change
`@@ -195,7 +195,7 @@ final class TermsReader extends Terms {`
`195`	`195`	`this.docCount = docCount;`
`196`	`196`	`OffHeapFSTStore offHeapFSTStore = new OffHeapFSTStore();`
`197`	`197`	`FSTTermOutputs outputs = new FSTTermOutputs(fieldInfo);`
`198`		`- this.dict = new FST<>(FST.readMetadata(in, outputs), in, outputs, offHeapFSTStore);`
	`198`	`+ this.dict = new FST<>(FST.readMetadata(in, outputs), in, offHeapFSTStore);`
`199`	`199`	`in.skipBytes(offHeapFSTStore.size());`
`200`	`200`	`}`
`201`	`201`