Skip to content

Commit b569193

Browse files
committed
Cleanup FSTDataOutputWriter
# Conflicts: # lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java # lucene/core/src/java/org/apache/lucene/util/fst/FSTDataOutputWriter.java
1 parent 9bd5f02 commit b569193

File tree

3 files changed

+44
-104
lines changed

3 files changed

+44
-104
lines changed

lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java

Lines changed: 39 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import java.io.IOException;
3434
import org.apache.lucene.store.ByteArrayDataOutput;
3535
import org.apache.lucene.store.DataOutput;
36+
import org.apache.lucene.util.Accountable;
3637
import org.apache.lucene.util.ArrayUtil;
3738
import org.apache.lucene.util.IntsRef;
3839
import org.apache.lucene.util.IntsRefBuilder;
@@ -135,13 +136,7 @@ public class FSTCompiler<T> {
135136
*/
136137
// TODO: remove this? Builder API should be the only entry point?
137138
public FSTCompiler(FST.INPUT_TYPE inputType, Outputs<T> outputs) {
138-
this(
139-
inputType,
140-
32.0,
141-
outputs,
142-
true,
143-
new FSTDataOutputWriter(getLegacyDataOutput(DEFAULT_BLOCK_BITS)),
144-
1f);
139+
this(inputType, 32.0, outputs, true, getLegacyDataOutput(DEFAULT_BLOCK_BITS), 1f);
145140
}
146141

147142
static DataOutput getLegacyDataOutput(int blockBits) {
@@ -153,24 +148,24 @@ private FSTCompiler(
153148
double suffixRAMLimitMB,
154149
Outputs<T> outputs,
155150
boolean allowFixedLengthArcs,
156-
FSTDataOutputWriter fstWriter,
151+
DataOutput dataOutput,
157152
float directAddressingMaxOversizingFactor) {
158153
this.allowFixedLengthArcs = allowFixedLengthArcs;
159154
this.directAddressingMaxOversizingFactor = directAddressingMaxOversizingFactor;
160155
// pad: ensure no node gets address 0 which is reserved to mean
161156
// the stop state w/ no arcs
162157
try {
163-
fstWriter.getDataOutput().writeByte((byte) 0);
158+
dataOutput.writeByte((byte) 0);
164159
numBytesWritten++;
165160
} catch (IOException e) {
166161
throw new RuntimeException(e);
167162
}
168-
this.dataOutput = fstWriter.getDataOutput();
163+
this.dataOutput = dataOutput;
169164
fst =
170165
new FST<>(
171166
new FST.FSTMetadata<>(inputType, null, -1, VERSION_CURRENT, 0),
172167
outputs,
173-
fstWriter.getReader());
168+
toFSTReader(dataOutput));
174169
if (suffixRAMLimitMB < 0) {
175170
throw new IllegalArgumentException("ramLimitMB must be >= 0; got: " + suffixRAMLimitMB);
176171
} else if (suffixRAMLimitMB > 0) {
@@ -188,6 +183,37 @@ private FSTCompiler(
188183
}
189184
}
190185

186+
// Get the respective FSTReader of the DataOutput
187+
private FSTReader toFSTReader(DataOutput dataOutput) {
188+
if (dataOutput instanceof FSTReader) {
189+
return (FSTReader) dataOutput;
190+
}
191+
return new FSTReader() {
192+
@Override
193+
public long size() {
194+
return numBytesWritten;
195+
}
196+
197+
@Override
198+
public FST.BytesReader getReverseBytesReader() {
199+
return null;
200+
}
201+
202+
@Override
203+
public void writeTo(DataOutput out) {
204+
throw new UnsupportedOperationException("writeTo(DataOutput) is not supported");
205+
}
206+
207+
@Override
208+
public long ramBytesUsed() {
209+
if (dataOutput instanceof Accountable) {
210+
return ((Accountable) dataOutput).ramBytesUsed();
211+
}
212+
return 0;
213+
}
214+
};
215+
}
216+
191217
/**
192218
* Fluent-style constructor for FST {@link FSTCompiler}.
193219
*
@@ -309,7 +335,7 @@ public FSTCompiler<T> build() {
309335
suffixRAMLimitMB,
310336
outputs,
311337
allowFixedLengthArcs,
312-
new FSTDataOutputWriter(dataOutput),
338+
dataOutput,
313339
directAddressingMaxOversizingFactor);
314340
return fstCompiler;
315341
}
@@ -904,6 +930,7 @@ void finish(long newStartNode) {
904930
}
905931
fst.metadata.startNode = newStartNode;
906932
fst.metadata.numBytes = numBytesWritten;
933+
scratchBytes.truncate(0);
907934
}
908935

909936
private boolean validOutput(T output) {

lucene/core/src/java/org/apache/lucene/util/fst/FSTDataOutputWriter.java

Lines changed: 0 additions & 86 deletions
This file was deleted.

lucene/core/src/test/org/apache/lucene/util/fst/TestFSTDataOutputWriter.java

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import java.util.List;
2727
import java.util.Random;
2828
import org.apache.lucene.store.DataInput;
29+
import org.apache.lucene.store.DataOutput;
2930
import org.apache.lucene.store.Directory;
3031
import org.apache.lucene.store.InputStreamDataInput;
3132
import org.apache.lucene.store.OutputStreamDataOutput;
@@ -63,9 +64,7 @@ public void testRandom() throws Exception {
6364
final int numBytes = TestUtil.nextInt(random(), 1, maxBytes);
6465
final byte[] expected = new byte[numBytes];
6566
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
66-
final FSTDataOutputWriter fstWriter =
67-
new FSTDataOutputWriter(new OutputStreamDataOutput(baos));
68-
FSTReader reader = fstWriter.getReader();
67+
final DataOutput dataOutput = new OutputStreamDataOutput(baos);
6968
if (VERBOSE) {
7069
System.out.println("TEST: iter=" + iter + " numBytes=" + numBytes);
7170
}
@@ -86,7 +85,7 @@ public void testRandom() throws Exception {
8685
}
8786

8887
expected[pos++] = b;
89-
fstWriter.writeByte(b);
88+
dataOutput.writeByte(b);
9089
}
9190
break;
9291

@@ -100,13 +99,13 @@ public void testRandom() throws Exception {
10099
System.out.println(" writeBytes len=" + len + " bytes=" + Arrays.toString(temp));
101100
}
102101
System.arraycopy(temp, 0, expected, pos, temp.length);
103-
fstWriter.writeBytes(temp, 0, temp.length);
102+
dataOutput.writeBytes(temp, 0, temp.length);
104103
pos += len;
105104
}
106105
break;
107106
}
108107

109-
assertEquals(pos, reader.size());
108+
assertEquals(pos, baos.toByteArray().length);
110109
}
111110
for (int i = 0; i < numBytes; i++) {
112111
assertEquals("byte @ index=" + i, expected[i], baos.toByteArray()[i]);

0 commit comments

Comments
 (0)