Skip to content

Commit d0c8ff5

Browse files
dnhatndnhatn
authored
Refactor TSDB doc_values util allow introduce new codec (#115042)
This PR refactors the doc_values utils used in the TSDB codec to allow sharing between the current codec and the new codec.
1 parent e927aaa commit d0c8ff5

File tree

8 files changed

+49
-48
lines changed

8 files changed

+49
-48
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/AbstractDocValuesForUtilBenchmark.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ public abstract class AbstractDocValuesForUtilBenchmark {
2121
protected final int blockSize;
2222

2323
public AbstractDocValuesForUtilBenchmark() {
24-
this.forUtil = new DocValuesForUtil();
24+
this.forUtil = new DocValuesForUtil(ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE);
2525
this.blockSize = ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE;
2626
}
2727

‎benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/DecodeBenchmark.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
import org.apache.lucene.store.ByteArrayDataInput;
1313
import org.apache.lucene.store.ByteArrayDataOutput;
1414
import org.apache.lucene.store.DataOutput;
15-
import org.elasticsearch.index.codec.tsdb.DocValuesForUtil;
1615
import org.openjdk.jmh.infra.Blackhole;
1716

1817
import java.io.IOException;
@@ -44,7 +43,7 @@ public void setupInvocation(int bitsPerValue) {
4443

4544
@Override
4645
public void benchmark(int bitsPerValue, Blackhole bh) throws IOException {
47-
DocValuesForUtil.decode(bitsPerValue, this.dataInput, this.output);
46+
forUtil.decode(bitsPerValue, this.dataInput, this.output);
4847
bh.consume(this.output);
4948
}
5049
}

‎server/src/main/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtil.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,12 @@ public class DocValuesForUtil {
2121
private static final int BITS_IN_FIVE_BYTES = 5 * Byte.SIZE;
2222
private static final int BITS_IN_SIX_BYTES = 6 * Byte.SIZE;
2323
private static final int BITS_IN_SEVEN_BYTES = 7 * Byte.SIZE;
24-
private static final int blockSize = ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE;
24+
private final int blockSize;
2525
private final byte[] encoded = new byte[1024];
2626

27-
public DocValuesForUtil() {}
27+
public DocValuesForUtil(int numericBlockSize) {
28+
this.blockSize = numericBlockSize;
29+
}
2830

2931
public static int roundBits(int bitsPerValue) {
3032
if (bitsPerValue > 24 && bitsPerValue <= 32) {
@@ -67,7 +69,7 @@ private void encodeFiveSixOrSevenBytesPerValue(long[] in, int bitsPerValue, fina
6769
out.writeBytes(this.encoded, bytesPerValue * in.length);
6870
}
6971

70-
public static void decode(int bitsPerValue, final DataInput in, long[] out) throws IOException {
72+
public void decode(int bitsPerValue, final DataInput in, long[] out) throws IOException {
7173
if (bitsPerValue <= 24) {
7274
ForUtil.decode(bitsPerValue, in, out);
7375
} else if (bitsPerValue <= 32) {
@@ -81,7 +83,7 @@ public static void decode(int bitsPerValue, final DataInput in, long[] out) thro
8183
}
8284
}
8385

84-
private static void decodeFiveSixOrSevenBytesPerValue(int bitsPerValue, final DataInput in, long[] out) throws IOException {
86+
private void decodeFiveSixOrSevenBytesPerValue(int bitsPerValue, final DataInput in, long[] out) throws IOException {
8587
// NOTE: we expect multibyte values to be written "least significant byte" first
8688
int bytesPerValue = bitsPerValue / Byte.SIZE;
8789
long mask = (1L << bitsPerValue) - 1;

‎server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesConsumer.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ private long[] writeField(FieldInfo field, DocValuesProducer valuesProducer, lon
144144
if (maxOrd != 1) {
145145
final long[] buffer = new long[ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE];
146146
int bufferSize = 0;
147-
final ES87TSDBDocValuesEncoder encoder = new ES87TSDBDocValuesEncoder();
147+
final TSDBDocValuesEncoder encoder = new TSDBDocValuesEncoder(ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE);
148148
values = valuesProducer.getSortedNumeric(field);
149149
final int bitsPerOrd = maxOrd >= 0 ? PackedInts.bitsRequired(maxOrd - 1) : -1;
150150
for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {

‎server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -965,7 +965,7 @@ public long longValue() {
965965

966966
private final int maxDoc = ES87TSDBDocValuesProducer.this.maxDoc;
967967
private int doc = -1;
968-
private final ES87TSDBDocValuesEncoder decoder = new ES87TSDBDocValuesEncoder();
968+
private final TSDBDocValuesEncoder decoder = new TSDBDocValuesEncoder(ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE);
969969
private long currentBlockIndex = -1;
970970
private final long[] currentBlock = new long[ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE];
971971

@@ -1030,7 +1030,7 @@ public long longValue() throws IOException {
10301030
);
10311031
return new NumericDocValues() {
10321032

1033-
private final ES87TSDBDocValuesEncoder decoder = new ES87TSDBDocValuesEncoder();
1033+
private final TSDBDocValuesEncoder decoder = new TSDBDocValuesEncoder(ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE);
10341034
private long currentBlockIndex = -1;
10351035
private final long[] currentBlock = new long[ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE];
10361036

@@ -1092,7 +1092,7 @@ private NumericValues getValues(NumericEntry entry, final long maxOrd) throws IO
10921092
final int bitsPerOrd = maxOrd >= 0 ? PackedInts.bitsRequired(maxOrd - 1) : -1;
10931093
return new NumericValues() {
10941094

1095-
private final ES87TSDBDocValuesEncoder decoder = new ES87TSDBDocValuesEncoder();
1095+
private final TSDBDocValuesEncoder decoder = new TSDBDocValuesEncoder(ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE);
10961096
private long currentBlockIndex = -1;
10971097
private final long[] currentBlock = new long[ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE];
10981098

‎server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesEncoder.java‎ renamed to ‎server/src/main/java/org/elasticsearch/index/codec/tsdb/TSDBDocValuesEncoder.java‎

Lines changed: 23 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,8 @@
4444
* </li>
4545
* </ul>
4646
*
47-
* Notice that encoding and decoding are written in a nested way, for instance {@link ES87TSDBDocValuesEncoder#deltaEncode} calling
48-
* {@link ES87TSDBDocValuesEncoder#removeOffset} and so on. This allows us to easily introduce new encoding schemes or remove existing
47+
* Notice that encoding and decoding are written in a nested way, for instance {@link TSDBDocValuesEncoder#deltaEncode} calling
48+
* {@link TSDBDocValuesEncoder#removeOffset} and so on. This allows us to easily introduce new encoding schemes or remove existing
4949
* (non-effective) encoding schemes in a backward-compatible way.
5050
*
5151
* A token is used as a bitmask to represent which encoding is applied and allows us to detect the applied encoding scheme at decoding time.
@@ -54,11 +54,13 @@
5454
*
5555
* Of course, decoding follows the opposite order with respect to encoding.
5656
*/
57-
public class ES87TSDBDocValuesEncoder {
57+
public class TSDBDocValuesEncoder {
5858
private final DocValuesForUtil forUtil;
59+
private final int numericBlockSize;
5960

60-
public ES87TSDBDocValuesEncoder() {
61-
this.forUtil = new DocValuesForUtil();
61+
public TSDBDocValuesEncoder(int numericBlockSize) {
62+
this.forUtil = new DocValuesForUtil(numericBlockSize);
63+
this.numericBlockSize = numericBlockSize;
6264
}
6365

6466
/**
@@ -68,7 +70,7 @@ public ES87TSDBDocValuesEncoder() {
6870
private void deltaEncode(int token, int tokenBits, long[] in, DataOutput out) throws IOException {
6971
int gts = 0;
7072
int lts = 0;
71-
for (int i = 1; i < ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE; ++i) {
73+
for (int i = 1; i < numericBlockSize; ++i) {
7274
if (in[i] > in[i - 1]) {
7375
gts++;
7476
} else if (in[i] < in[i - 1]) {
@@ -79,7 +81,7 @@ private void deltaEncode(int token, int tokenBits, long[] in, DataOutput out) th
7981
final boolean doDeltaCompression = (gts == 0 && lts >= 2) || (lts == 0 && gts >= 2);
8082
long first = 0;
8183
if (doDeltaCompression) {
82-
for (int i = ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE - 1; i > 0; --i) {
84+
for (int i = numericBlockSize - 1; i > 0; --i) {
8385
in[i] -= in[i - 1];
8486
}
8587
// Avoid setting in[0] to 0 in case there is a minimum interval between
@@ -115,7 +117,7 @@ private void removeOffset(int token, int tokenBits, long[] in, DataOutput out) t
115117
}
116118

117119
if (min != 0) {
118-
for (int i = 0; i < ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE; ++i) {
120+
for (int i = 0; i < numericBlockSize; ++i) {
119121
in[i] -= min;
120122
}
121123
token = (token << 1) | 0x01;
@@ -143,7 +145,7 @@ private void gcdEncode(int token, int tokenBits, long[] in, DataOutput out) thro
143145
}
144146
final boolean doGcdCompression = Long.compareUnsigned(gcd, 1) > 0;
145147
if (doGcdCompression) {
146-
for (int i = 0; i < ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE; ++i) {
148+
for (int i = 0; i < numericBlockSize; ++i) {
147149
in[i] /= gcd;
148150
}
149151
token = (token << 1) | 0x01;
@@ -174,7 +176,7 @@ private void forEncode(int token, int tokenBits, long[] in, DataOutput out) thro
174176
* Encode the given longs using a combination of delta-coding, GCD factorization and bit packing.
175177
*/
176178
void encode(long[] in, DataOutput out) throws IOException {
177-
assert in.length == ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE;
179+
assert in.length == numericBlockSize;
178180

179181
deltaEncode(0, 0, in, out);
180182
}
@@ -192,7 +194,7 @@ void encode(long[] in, DataOutput out) throws IOException {
192194
* </ul>
193195
*/
194196
void encodeOrdinals(long[] in, DataOutput out, int bitsPerOrd) throws IOException {
195-
assert in.length == ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE;
197+
assert in.length == numericBlockSize;
196198
int numRuns = 1;
197199
long firstValue = in[0];
198200
long previousValue = firstValue;
@@ -259,7 +261,7 @@ void encodeOrdinals(long[] in, DataOutput out, int bitsPerOrd) throws IOExceptio
259261
}
260262

261263
void decodeOrdinals(DataInput in, long[] out, int bitsPerOrd) throws IOException {
262-
assert out.length == ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE : out.length;
264+
assert out.length == numericBlockSize : out.length;
263265

264266
long v1 = in.readVLong();
265267
int encoding = Long.numberOfTrailingZeros(~v1);
@@ -275,7 +277,7 @@ void decodeOrdinals(DataInput in, long[] out, int bitsPerOrd) throws IOException
275277
Arrays.fill(out, runLen, out.length, v2);
276278
} else if (encoding == 2) {
277279
// bit-packed
278-
DocValuesForUtil.decode(bitsPerOrd, in, out);
280+
forUtil.decode(bitsPerOrd, in, out);
279281
} else if (encoding == 3) {
280282
// cycle encoding
281283
int cycleLength = (int) v1;
@@ -293,13 +295,13 @@ void decodeOrdinals(DataInput in, long[] out, int bitsPerOrd) throws IOException
293295

294296
/** Decode longs that have been encoded with {@link #encode}. */
295297
void decode(DataInput in, long[] out) throws IOException {
296-
assert out.length == ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE : out.length;
298+
assert out.length == numericBlockSize : out.length;
297299

298300
final int token = in.readVInt();
299301
final int bitsPerValue = token >>> 3;
300302

301303
if (bitsPerValue != 0) {
302-
DocValuesForUtil.decode(bitsPerValue, in, out);
304+
forUtil.decode(bitsPerValue, in, out);
303305
} else {
304306
Arrays.fill(out, 0L);
305307
}
@@ -330,21 +332,21 @@ void decode(DataInput in, long[] out) throws IOException {
330332
}
331333

332334
// this loop should auto-vectorize
333-
private static void mul(long[] arr, long m) {
334-
for (int i = 0; i < ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE; ++i) {
335+
private void mul(long[] arr, long m) {
336+
for (int i = 0; i < numericBlockSize; ++i) {
335337
arr[i] *= m;
336338
}
337339
}
338340

339341
// this loop should auto-vectorize
340-
private static void add(long[] arr, long min) {
341-
for (int i = 0; i < ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE; ++i) {
342+
private void add(long[] arr, long min) {
343+
for (int i = 0; i < numericBlockSize; ++i) {
342344
arr[i] += min;
343345
}
344346
}
345347

346-
private static void deltaDecode(long[] arr) {
347-
for (int i = 1; i < ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE; ++i) {
348+
private void deltaDecode(long[] arr) {
349+
for (int i = 1; i < numericBlockSize; ++i) {
348350
arr[i] += arr[i - 1];
349351
}
350352
}

‎server/src/test/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtilTests.java‎

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -31,17 +31,18 @@
3131
import java.util.Random;
3232

3333
public class DocValuesForUtilTests extends LuceneTestCase {
34+
int NUMERIC_BLOCK_SIZE = 1 << 7;
3435

3536
public void testEncodeDecode() throws IOException {
3637
final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);
37-
final long[] values = new long[iterations * ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE];
38+
final long[] values = new long[iterations * NUMERIC_BLOCK_SIZE];
3839
final int[] bpvs = new int[iterations];
3940

4041
for (int i = 0; i < iterations; ++i) {
4142
final int bpv = TestUtil.nextInt(random(), 1, 64);
4243
bpvs[i] = DocValuesForUtil.roundBits(bpv);
43-
for (int j = 0; j < ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE; ++j) {
44-
values[i * ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE + j] = bpv == 64
44+
for (int j = 0; j < NUMERIC_BLOCK_SIZE; ++j) {
45+
values[i * NUMERIC_BLOCK_SIZE + j] = bpv == 64
4546
? random().nextLong()
4647
: TestUtil.nextLong(random(), 0, PackedInts.maxValue(bpv));
4748
}
@@ -53,12 +54,12 @@ public void testEncodeDecode() throws IOException {
5354
{
5455
// encode
5556
IndexOutput out = d.createOutput("test.bin", IOContext.DEFAULT);
56-
final DocValuesForUtil forUtil = new DocValuesForUtil();
57+
final DocValuesForUtil forUtil = new DocValuesForUtil(NUMERIC_BLOCK_SIZE);
5758

5859
for (int i = 0; i < iterations; ++i) {
59-
long[] source = new long[ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE];
60-
for (int j = 0; j < ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE; ++j) {
61-
source[j] = values[i * ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE + j];
60+
long[] source = new long[NUMERIC_BLOCK_SIZE];
61+
for (int j = 0; j < NUMERIC_BLOCK_SIZE; ++j) {
62+
source[j] = values[i * NUMERIC_BLOCK_SIZE + j];
6263
}
6364
out.writeByte((byte) bpvs[i]);
6465
forUtil.encode(source, bpvs[i], out);
@@ -70,17 +71,14 @@ public void testEncodeDecode() throws IOException {
7071
{
7172
// decode
7273
IndexInput in = d.openInput("test.bin", IOContext.READONCE);
73-
final long[] restored = new long[ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE];
74+
final DocValuesForUtil forUtil = new DocValuesForUtil(NUMERIC_BLOCK_SIZE);
75+
final long[] restored = new long[NUMERIC_BLOCK_SIZE];
7476
for (int i = 0; i < iterations; ++i) {
7577
final int bitsPerValue = in.readByte();
76-
DocValuesForUtil.decode(bitsPerValue, in, restored);
78+
forUtil.decode(bitsPerValue, in, restored);
7779
assertArrayEquals(
7880
Arrays.toString(restored),
79-
ArrayUtil.copyOfSubArray(
80-
values,
81-
i * ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE,
82-
(i + 1) * ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE
83-
),
81+
ArrayUtil.copyOfSubArray(values, i * NUMERIC_BLOCK_SIZE, (i + 1) * NUMERIC_BLOCK_SIZE),
8482
restored
8583
);
8684
}

‎server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesEncoderTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,11 @@
2323

2424
public class ES87TSDBDocValuesEncoderTests extends LuceneTestCase {
2525

26-
private final ES87TSDBDocValuesEncoder encoder;
26+
private final TSDBDocValuesEncoder encoder;
2727
private final int blockSize = ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE;
2828

2929
public ES87TSDBDocValuesEncoderTests() {
30-
this.encoder = new ES87TSDBDocValuesEncoder();
30+
this.encoder = new TSDBDocValuesEncoder(blockSize);
3131
}
3232

3333
public void testRandomValues() throws IOException {

0 commit comments

Comments
 (0)