Skip to content

Prepare tsdb doc values format for merging optimizations. #125933

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Apr 2, 2025
Merged

Prepare tsdb doc values format for merging optimizations. #125933

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
3d33572
Prepare tsdb doc values format for merging optimizations.
martijnvg Mar 31, 2025
fa2fb4f
fix check style
martijnvg Mar 31, 2025
705f0fb
improve test
martijnvg Mar 31, 2025
cb44dde
Moved docsWithFieldOffset, docsWithFieldLength, jumpTableEntryCount, …
martijnvg Mar 31, 2025
aa9f39d
Merge remote-tracking branch 'es/main' into es87_tsdb_doc_values_form…
martijnvg Mar 31, 2025
7597820
spotless
martijnvg Mar 31, 2025
32677c1
fix precommit
martijnvg Mar 31, 2025
5aabfca
Merge remote-tracking branch 'es/main' into es87_tsdb_doc_values_form…
martijnvg Apr 1, 2025
378ee29
Fork tsdb doc values change into a new codec: ES819TSDB
martijnvg Apr 1, 2025
405d56c
Merge remote-tracking branch 'es/main' into es87_tsdb_doc_values_form…
martijnvg Apr 2, 2025
622da42
Move ES87TSDBDocValuesConsumer to test code.
martijnvg Apr 2, 2025
e5dcfbf
Merge remote-tracking branch 'es/main' into es87_tsdb_doc_values_form…
martijnvg Apr 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 21 additions & 20 deletions server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesConsumer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,26 +125,8 @@ private long[] writeField(FieldInfo field, DocValuesProducer valuesProducer, lon
numValues += count;
}

if (numDocsWithValue == 0) { // meta[-2, 0]: No documents with values
meta.writeLong(-2); // docsWithFieldOffset
meta.writeLong(0L); // docsWithFieldLength
meta.writeShort((short) -1); // jumpTableEntryCount
meta.writeByte((byte) -1); // denseRankPower
} else if (numDocsWithValue == maxDoc) { // meta[-1, 0]: All documents have values
meta.writeLong(-1); // docsWithFieldOffset
meta.writeLong(0L); // docsWithFieldLength
meta.writeShort((short) -1); // jumpTableEntryCount
meta.writeByte((byte) -1); // denseRankPower
} else { // meta[data.offset, data.length]: IndexedDISI structure for documents with values
long offset = data.getFilePointer();
meta.writeLong(offset); // docsWithFieldOffset
values = valuesProducer.getSortedNumeric(field);
final short jumpTableEntryCount = IndexedDISI.writeBitSet(values, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER);
meta.writeLong(data.getFilePointer() - offset); // docsWithFieldLength
meta.writeShort(jumpTableEntryCount);
meta.writeByte(IndexedDISI.DEFAULT_DENSE_RANK_POWER);
}
meta.writeLong(numValues);
meta.writeInt(numDocsWithValue);

if (numValues > 0) {
// Special case for maxOrd of 1, signal -1 that no blocks will be written
Expand Down Expand Up @@ -206,6 +188,26 @@ private long[] writeField(FieldInfo field, DocValuesProducer valuesProducer, lon
meta.writeLong(valuesDataLength);
}

if (numDocsWithValue == 0) { // meta[-2, 0]: No documents with values
meta.writeLong(-2); // docsWithFieldOffset
meta.writeLong(0L); // docsWithFieldLength
meta.writeShort((short) -1); // jumpTableEntryCount
meta.writeByte((byte) -1); // denseRankPower
} else if (numDocsWithValue == maxDoc) { // meta[-1, 0]: All documents have values
meta.writeLong(-1); // docsWithFieldOffset
meta.writeLong(0L); // docsWithFieldLength
meta.writeShort((short) -1); // jumpTableEntryCount
meta.writeByte((byte) -1); // denseRankPower
} else { // meta[data.offset, data.length]: IndexedDISI structure for documents with values
long offset = data.getFilePointer();
meta.writeLong(offset); // docsWithFieldOffset
values = valuesProducer.getSortedNumeric(field);
final short jumpTableEntryCount = IndexedDISI.writeBitSet(values, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER);
meta.writeLong(data.getFilePointer() - offset); // docsWithFieldLength
meta.writeShort(jumpTableEntryCount);
meta.writeByte(IndexedDISI.DEFAULT_DENSE_RANK_POWER);
}

return new long[] { numDocsWithValue, numValues };
}

Expand Down Expand Up @@ -495,7 +497,6 @@ private void writeSortedNumericField(FieldInfo field, DocValuesProducer valuesPr
long numValues = stats[1];
assert numValues >= numDocsWithField;

meta.writeInt(numDocsWithField);
if (numValues > numDocsWithField) {
long start = data.getFilePointer();
meta.writeLong(start);
Expand Down
13 changes: 10 additions & 3 deletions server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,21 @@ public class ES87TSDBDocValuesFormat extends org.apache.lucene.codecs.DocValuesF
static final String DATA_EXTENSION = "dvd";
static final String META_CODEC = "ES87TSDBDocValuesMetadata";
static final String META_EXTENSION = "dvm";
static final int VERSION_START = 0;
static final int VERSION_CURRENT = VERSION_START;
static final byte NUMERIC = 0;
static final byte BINARY = 1;
static final byte SORTED = 2;
static final byte SORTED_SET = 3;
static final byte SORTED_NUMERIC = 4;

// Version constants:
static final int VERSION_START = 0;
// Second versions has two changes:
// 1) Moved numDocsWithField metadata statistic from SortedNumericEntry to NumericEntry and
// 2) Moved docsWithFieldOffset, docsWithFieldLength, jumpTableEntryCount, denseRankPower metadata properties in the format to be after
// values metadata.
static final int VERSION_TWO = 1;
static final int VERSION_CURRENT = VERSION_TWO;

static final int TERMS_DICT_BLOCK_LZ4_SHIFT = 6;
static final int TERMS_DICT_BLOCK_LZ4_SIZE = 1 << TERMS_DICT_BLOCK_LZ4_SHIFT;
static final int TERMS_DICT_BLOCK_LZ4_MASK = TERMS_DICT_BLOCK_LZ4_SIZE - 1;
Expand Down Expand Up @@ -75,7 +82,7 @@ public class ES87TSDBDocValuesFormat extends org.apache.lucene.codecs.DocValuesF
}
}

private final int skipIndexIntervalSize;
final int skipIndexIntervalSize;

/** Default constructor. */
public ES87TSDBDocValuesFormat() {
Expand Down
69 changes: 41 additions & 28 deletions server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
import static org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat.SKIP_INDEX_MAX_LEVEL;
import static org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat.TERMS_DICT_BLOCK_LZ4_SHIFT;

public class ES87TSDBDocValuesProducer extends DocValuesProducer {
final class ES87TSDBDocValuesProducer extends DocValuesProducer {
private final IntObjectHashMap<NumericEntry> numerics;
private final IntObjectHashMap<BinaryEntry> binaries;
private final IntObjectHashMap<SortedEntry> sorted;
Expand All @@ -58,7 +58,7 @@ public class ES87TSDBDocValuesProducer extends DocValuesProducer {
private final IntObjectHashMap<DocValuesSkipperEntry> skippers;
private final IndexInput data;
private final int maxDoc;
private final int version;
final int version;
private final boolean merging;

ES87TSDBDocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension)
Expand Down Expand Up @@ -88,7 +88,7 @@ public class ES87TSDBDocValuesProducer extends DocValuesProducer {
state.segmentSuffix
);

readFields(in, state.fieldInfos);
readFields(in, state.fieldInfos, version);

} catch (Throwable exception) {
priorE = exception;
Expand Down Expand Up @@ -359,7 +359,7 @@ public long cost() {
};
}

private abstract class BaseSortedDocValues extends SortedDocValues {
abstract class BaseSortedDocValues extends SortedDocValues {

final SortedEntry entry;
final TermsEnum termsEnum;
Expand Down Expand Up @@ -395,7 +395,7 @@ public TermsEnum termsEnum() throws IOException {
}
}

private abstract static class BaseSortedSetDocValues extends SortedSetDocValues {
abstract static class BaseSortedSetDocValues extends SortedSetDocValues {

final SortedSetEntry entry;
final IndexInput data;
Expand Down Expand Up @@ -860,7 +860,7 @@ public void close() throws IOException {
data.close();
}

private void readFields(IndexInput meta, FieldInfos infos) throws IOException {
private void readFields(IndexInput meta, FieldInfos infos, int version) throws IOException {
for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) {
FieldInfo info = infos.fieldInfo(fieldNumber);
if (info == null) {
Expand All @@ -871,24 +871,24 @@ private void readFields(IndexInput meta, FieldInfos infos) throws IOException {
skippers.put(info.number, readDocValueSkipperMeta(meta));
}
if (type == ES87TSDBDocValuesFormat.NUMERIC) {
numerics.put(info.number, readNumeric(meta));
numerics.put(info.number, readNumeric(meta, version));
} else if (type == ES87TSDBDocValuesFormat.BINARY) {
binaries.put(info.number, readBinary(meta));
} else if (type == ES87TSDBDocValuesFormat.SORTED) {
sorted.put(info.number, readSorted(meta));
sorted.put(info.number, readSorted(meta, version));
} else if (type == ES87TSDBDocValuesFormat.SORTED_SET) {
sortedSets.put(info.number, readSortedSet(meta));
sortedSets.put(info.number, readSortedSet(meta, version));
} else if (type == ES87TSDBDocValuesFormat.SORTED_NUMERIC) {
sortedNumerics.put(info.number, readSortedNumeric(meta));
sortedNumerics.put(info.number, readSortedNumeric(meta, version));
} else {
throw new CorruptIndexException("invalid type: " + type, meta);
}
}
}

private static NumericEntry readNumeric(IndexInput meta) throws IOException {
private static NumericEntry readNumeric(IndexInput meta, int version) throws IOException {
NumericEntry entry = new NumericEntry();
readNumeric(meta, entry);
readNumeric(meta, entry, version);
return entry;
}

Expand All @@ -903,12 +903,17 @@ private static DocValuesSkipperEntry readDocValueSkipperMeta(IndexInput meta) th
return new DocValuesSkipperEntry(offset, length, minValue, maxValue, docCount, maxDocID);
}

private static void readNumeric(IndexInput meta, NumericEntry entry) throws IOException {
entry.docsWithFieldOffset = meta.readLong();
entry.docsWithFieldLength = meta.readLong();
entry.jumpTableEntryCount = meta.readShort();
entry.denseRankPower = meta.readByte();
private static void readNumeric(IndexInput meta, NumericEntry entry, int version) throws IOException {
if (version < ES87TSDBDocValuesFormat.VERSION_TWO) {
Copy link
Member

@dnhatn dnhatn Mar 31, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since these branches are far away, should we fork a new consumer and producer instead?

Copy link
Member Author

@martijnvg martijnvg Apr 1, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done: 378ee29

entry.docsWithFieldOffset = meta.readLong();
entry.docsWithFieldLength = meta.readLong();
entry.jumpTableEntryCount = meta.readShort();
entry.denseRankPower = meta.readByte();
}
entry.numValues = meta.readLong();
if (version >= ES87TSDBDocValuesFormat.VERSION_TWO) {
entry.numDocsWithField = meta.readInt();
}
if (entry.numValues > 0) {
final int indexBlockShift = meta.readInt();
// Special case, -1 means there are no blocks, so no need to load the metadata for it
Expand All @@ -925,6 +930,12 @@ private static void readNumeric(IndexInput meta, NumericEntry entry) throws IOEx
entry.valuesOffset = meta.readLong();
entry.valuesLength = meta.readLong();
}
if (version >= ES87TSDBDocValuesFormat.VERSION_TWO) {
entry.docsWithFieldOffset = meta.readLong();
entry.docsWithFieldLength = meta.readLong();
entry.jumpTableEntryCount = meta.readShort();
entry.denseRankPower = meta.readByte();
}
}

private BinaryEntry readBinary(IndexInput meta) throws IOException {
Expand All @@ -951,15 +962,17 @@ private BinaryEntry readBinary(IndexInput meta) throws IOException {
return entry;
}

private static SortedNumericEntry readSortedNumeric(IndexInput meta) throws IOException {
private static SortedNumericEntry readSortedNumeric(IndexInput meta, int version) throws IOException {
SortedNumericEntry entry = new SortedNumericEntry();
readSortedNumeric(meta, entry);
readSortedNumeric(meta, entry, version);
return entry;
}

private static SortedNumericEntry readSortedNumeric(IndexInput meta, SortedNumericEntry entry) throws IOException {
readNumeric(meta, entry);
entry.numDocsWithField = meta.readInt();
private static SortedNumericEntry readSortedNumeric(IndexInput meta, SortedNumericEntry entry, int version) throws IOException {
readNumeric(meta, entry, version);
if (version < ES87TSDBDocValuesFormat.VERSION_TWO) {
entry.numDocsWithField = meta.readInt();
}
if (entry.numDocsWithField != entry.numValues) {
entry.addressesOffset = meta.readLong();
final int blockShift = meta.readVInt();
Expand All @@ -969,29 +982,29 @@ private static SortedNumericEntry readSortedNumeric(IndexInput meta, SortedNumer
return entry;
}

private SortedEntry readSorted(IndexInput meta) throws IOException {
private SortedEntry readSorted(IndexInput meta, int version) throws IOException {
SortedEntry entry = new SortedEntry();
entry.ordsEntry = new NumericEntry();
readNumeric(meta, entry.ordsEntry);
readNumeric(meta, entry.ordsEntry, version);
entry.termsDictEntry = new TermsDictEntry();
readTermDict(meta, entry.termsDictEntry);
return entry;
}

private SortedSetEntry readSortedSet(IndexInput meta) throws IOException {
private SortedSetEntry readSortedSet(IndexInput meta, int version) throws IOException {
SortedSetEntry entry = new SortedSetEntry();
byte multiValued = meta.readByte();
switch (multiValued) {
case 0: // singlevalued
entry.singleValueEntry = readSorted(meta);
entry.singleValueEntry = readSorted(meta, version);
return entry;
case 1: // multivalued
break;
default:
throw new CorruptIndexException("Invalid multiValued flag: " + multiValued, meta);
}
entry.ordsEntry = new SortedNumericEntry();
readSortedNumeric(meta, entry.ordsEntry);
readSortedNumeric(meta, entry.ordsEntry, version);
entry.termsDictEntry = new TermsDictEntry();
readTermDict(meta, entry.termsDictEntry);
return entry;
Expand Down Expand Up @@ -1421,6 +1434,7 @@ private static class NumericEntry {
short jumpTableEntryCount;
byte denseRankPower;
long numValues;
int numDocsWithField;
long indexOffset;
long indexLength;
DirectMonotonicReader.Meta indexMeta;
Expand All @@ -1444,7 +1458,6 @@ private static class BinaryEntry {
}

private static class SortedNumericEntry extends NumericEntry {
int numDocsWithField;
DirectMonotonicReader.Meta addressesMeta;
long addressesOffset;
long addressesLength;
Expand Down
Loading