Skip to content

Commit a404e18

Browse files
committed
Allow FST builder to use different writer (#12543)
1 parent 6caf5b0 commit a404e18

File tree

7 files changed

+457
-49
lines changed

7 files changed

+457
-49
lines changed

lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
// TODO: merge with PagedBytes, except PagedBytes doesn't
2727
// let you read while writing which FST needs
2828

29-
class BytesStore extends DataOutput implements FSTReader {
29+
class BytesStore extends DataOutput implements FSTWriter {
3030

3131
private static final long BASE_RAM_BYTES_USED =
3232
RamUsageEstimator.shallowSizeOfInstance(BytesStore.class)
@@ -359,6 +359,7 @@ public void truncate(long newLen) {
359359
assert newLen == getPosition();
360360
}
361361

362+
@Override
362363
public void finish() {
363364
if (current != null) {
364365
byte[] lastBuffer = new byte[nextWrite];
@@ -368,6 +369,18 @@ public void finish() {
368369
}
369370
}
370371

372+
/** Writes all of our bytes to the target {@link FSTWriter}. */
373+
public void writeTo(FSTWriter out) throws IOException {
374+
// TODO: if the FSTWriter is also BytesStore we are doing double write
375+
// once to reverse the bytes and once to write to the BytesStore
376+
// maybe we should combine it into reverseAndWriteTo()?
377+
reverse(0, getPosition() - 1);
378+
finish();
379+
for (byte[] block : blocks) {
380+
out.writeBytes(block, 0, block.length);
381+
}
382+
}
383+
371384
/** Writes all of our bytes to the target {@link DataOutput}. */
372385
@Override
373386
public void writeTo(DataOutput out) throws IOException {
Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.lucene.util.fst;
18+
19+
import java.io.Closeable;
20+
import java.io.IOException;
21+
import org.apache.lucene.store.DataOutput;
22+
import org.apache.lucene.util.Accountable;
23+
import org.apache.lucene.util.RamUsageEstimator;
24+
25+
/**
26+
* An {@link FSTWriter} which write to a {@link DataOutput}. It only supports writing to the {@link
27+
* DataOutput} and not reading from it To read, you must construct a corresponding {@link
28+
* org.apache.lucene.store.DataInput} and use the {@link FSTStore} to read
29+
*
30+
* @lucene.experimental
31+
*/
32+
public class DataOutputFSTWriter implements FSTWriter {
33+
34+
private static final long BASE_RAM_BYTES_USED =
35+
RamUsageEstimator.shallowSizeOfInstance(DataOutputFSTWriter.class);
36+
37+
private final DataOutput dataOutput;
38+
39+
private long size = 0L;
40+
41+
/**
42+
* ctor
43+
*
44+
* @param dataOutput the data output to write to
45+
*/
46+
public DataOutputFSTWriter(DataOutput dataOutput) {
47+
this.dataOutput = dataOutput;
48+
}
49+
50+
@Override
51+
public long ramBytesUsed() {
52+
long size = BASE_RAM_BYTES_USED;
53+
if (dataOutput instanceof Accountable) {
54+
size += ((Accountable) dataOutput).ramBytesUsed();
55+
}
56+
return size;
57+
}
58+
59+
@Override
60+
public long size() {
61+
return size;
62+
}
63+
64+
@Override
65+
public void writeByte(byte b) throws IOException {
66+
size++;
67+
dataOutput.writeByte(b);
68+
}
69+
70+
@Override
71+
public void writeBytes(byte[] b, int offset, int length) throws IOException {
72+
size += length;
73+
dataOutput.writeBytes(b, offset, length);
74+
}
75+
76+
@Override
77+
public void finish() throws IOException {
78+
if (dataOutput instanceof Closeable) {
79+
((Closeable) dataOutput).close();
80+
}
81+
}
82+
83+
@Override
84+
public void writeTo(DataOutput out) throws IOException {
85+
throw new UnsupportedOperationException(
86+
"writeTo(DataOutput) is not supported by DataOutputFSTWriter");
87+
}
88+
89+
@Override
90+
public FST.BytesReader getReverseBytesReader() {
91+
return null;
92+
}
93+
}

0 commit comments

Comments
 (0)