Add tests

Author: dungba88

lucene/core/src/test/org/apache/lucene/util/fst/TestDataOutputFSTWriter.java

@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.util.fst;
+
+import org.apache.lucene.store.OutputStreamDataOutput;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+
+import java.io.ByteArrayOutputStream;
+import java.util.Arrays;
+
+public class TestDataOutputFSTWriter extends LuceneTestCase {
+
+  public void testRandom() throws Exception {
+
+    final int iters = atLeast(10);
+    final int maxBytes = TEST_NIGHTLY ? 200000 : 20000;
+    for (int iter = 0; iter < iters; iter++) {
+      final int numBytes = TestUtil.nextInt(random(), 1, maxBytes);
+      final byte[] expected = new byte[numBytes];
+      final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+      final DataOutputFSTWriter bytes = new DataOutputFSTWriter(new OutputStreamDataOutput(baos), null);
+      if (VERBOSE) {
+        System.out.println("TEST: iter=" + iter + " numBytes=" + numBytes);
+      }
+
+      int pos = 0;
+      while (pos < numBytes) {
+        int op = random().nextInt(2);
+        if (VERBOSE) {
+          System.out.println("  cycle pos=" + pos);
+        }
+        switch (op) {
+          case 0:
+            {
+              // write random byte
+              byte b = (byte) random().nextInt(256);
+              if (VERBOSE) {
+                System.out.println("    writeByte b=" + b);
+              }
+
+              expected[pos++] = b;
+              bytes.writeByte(b);
+            }
+            break;
+
+          case 1:
+            {
+              // write random byte[]
+              int len = random().nextInt(Math.min(numBytes - pos, 100));
+              byte[] temp = new byte[len];
+              random().nextBytes(temp);
+              if (VERBOSE) {
+                System.out.println("    writeBytes len=" + len + " bytes=" + Arrays.toString(temp));
+              }
+              System.arraycopy(temp, 0, expected, pos, temp.length);
+              bytes.writeBytes(temp, 0, temp.length);
+              pos += len;
+            }
+            break;
+        }
+
+        assertEquals(pos, bytes.size());
+      }
+      for (int i = 0; i < numBytes; i++) {
+        assertEquals("byte @ index=" + i, expected[i], baos.toByteArray()[i]);
+      }
+    }
+  }
+}

lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java

@@ -22,9 +22,12 @@
 
 import java.io.BufferedReader;
 import java.io.ByteArrayOutputStream;
+import java.io.File;
 import java.io.IOException;
+import java.io.RandomAccessFile;
 import java.io.StringWriter;
 import java.io.Writer;
+import java.nio.channels.FileChannel;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -104,6 +107,55 @@ public void tearDown() throws Exception {
     super.tearDown();
   }
 
+  public void testFSAWithFileChannel() throws IOException {
+    File file = createTempFile().toFile();
+    try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) {
+      FileChannel fileChannel = raf.getChannel();
+      String[] strings2 =
+          new String[] {
+              "station", "commotion", "elation", "elastic", "plastic", "stop", "ftop", "ftation"
+          };
+      IntsRef[] terms2 = new IntsRef[strings2.length];
+      for (int inputMode = 0; inputMode < 2; inputMode++) {
+        if (VERBOSE) {
+          System.out.println("TEST: inputMode=" + inputModeToString(inputMode));
+        }
+
+        for (int idx = 0; idx < strings2.length; idx++) {
+          terms2[idx] = toIntsRef(strings2[idx], inputMode);
+        }
+        Arrays.sort(terms2);
+
+        // Test pre-determined FST sizes to make sure we haven't lost minimality (at least on this
+        // trivial set of terms):
+
+        // FSA
+        {
+          final Outputs<Object> outputs = NoOutputs.getSingleton();
+          final Object NO_OUTPUT = outputs.getNoOutput();
+          final List<FSTTester.InputOutput<Object>> pairs = new ArrayList<>(terms2.length);
+          for (IntsRef term : terms2) {
+            pairs.add(new FSTTester.InputOutput<>(term, NO_OUTPUT));
+          }
+          FSTTester<Object> tester = new FSTTester<>(random(), dir, inputMode, pairs, outputs) {
+
+            @Override
+            protected FSTCompiler.Builder<Object> getFSTBuilder() {
+              return super.getFSTBuilder().fstWriter(new FileChannelFSTWriter(fileChannel, 1024));
+            }
+          };
+          FST<Object> fst = tester.doTest();
+          assertNotNull(fst);
+          assertEquals(22, tester.nodeCount);
+          assertEquals(27, tester.arcCount);
+          for (IntsRef intsRef : terms2) {
+            assertNotNull(Util.get(fst, intsRef));
+          }
+        }
+      }
+    }
+  }
+
   public void testBasicFSA() throws IOException {
     String[] strings =
         new String[] {

lucene/test-framework/src/java/org/apache/lucene/tests/util/fst/FSTTester.java

@@ -255,10 +255,7 @@ private T randomAcceptedWord(FST<T> fst, IntsRefBuilder in) throws IOException {
 
   public FST<T> doTest() throws IOException {
 
-    final FSTCompiler<T> fstCompiler =
-        new FSTCompiler.Builder<>(
-                inputMode == 0 ? FST.INPUT_TYPE.BYTE1 : FST.INPUT_TYPE.BYTE4, outputs)
-            .build();
+    final FSTCompiler<T> fstCompiler = getFSTBuilder().build();
 
     for (InputOutput<T> pair : pairs) {
       if (pair.output instanceof List) {
@@ -316,6 +313,10 @@ public FST<T> doTest() throws IOException {
     return fst;
   }
 
+  protected FSTCompiler.Builder<T> getFSTBuilder() {
+    return new FSTCompiler.Builder<>(inputMode == 0 ? FST.INPUT_TYPE.BYTE1 : FST.INPUT_TYPE.BYTE4, outputs);
+  }
+
   protected boolean outputsEqual(T a, T b) {
     return a.equals(b);
   }