Skip to content

Commit 969e71a

Browse files
martijnvgmartijnvg
committed
Add make use of SortedSetDocValues#termsEnum() to load terms if ordinals are dense.
Also added block loader for dimension fields.
1 parent 2f1aa71 commit 969e71a

File tree

11 files changed

+211
-38
lines changed

11 files changed

+211
-38
lines changed

‎server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesProducer.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1325,7 +1325,7 @@ public void loadBlock(BlockLoader.SingletonLongBuilder builder, BlockLoader.Docs
13251325
}
13261326

13271327
@Override
1328-
public void loadBlock(BlockLoader.TSIDOrdinalsBuilder builder, BlockLoader.Docs docs, int offset)
1328+
public void loadBlock(BlockLoader.TSSingletonOrdinalsBuilder builder, BlockLoader.Docs docs, int offset)
13291329
throws IOException {
13301330
assert maxOrd >= 0;
13311331
doc = docs.get(docs.count() - 1);

‎server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/SingletonDocValuesBlockLoader.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ public interface SingletonDocValuesBlockLoader {
1717

1818
void loadBlock(BlockLoader.SingletonLongBuilder builder, BlockLoader.Docs docs, int offset) throws IOException;
1919

20-
void loadBlock(BlockLoader.TSIDOrdinalsBuilder builder, BlockLoader.Docs docs, int offset) throws IOException;
20+
void loadBlock(BlockLoader.TSSingletonOrdinalsBuilder builder, BlockLoader.Docs docs, int offset) throws IOException;
2121

2222
int docID();
2323

‎server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -423,7 +423,7 @@ interface BlockFactory {
423423
*/
424424
SingletonOrdinalsBuilder singletonOrdinalsBuilder(SortedDocValues ordinals, int count);
425425

426-
TSIDOrdinalsBuilder tsidOrdinalsBuilder(SortedDocValues ordinals, int count);
426+
TSSingletonOrdinalsBuilder tsSingletonOrdinalsBuilder(boolean isPrimaryIndexSortField, SortedDocValues ordinals, int count);
427427

428428
/**
429429
* Build a reader for reading {@link SortedSetDocValues}
@@ -523,11 +523,11 @@ interface SingletonOrdinalsBuilder extends Builder {
523523
SingletonOrdinalsBuilder appendOrd(int value);
524524
}
525525

526-
interface TSIDOrdinalsBuilder extends Builder {
526+
interface TSSingletonOrdinalsBuilder extends Builder {
527527

528-
TSIDOrdinalsBuilder appendOrd(long value);
528+
TSSingletonOrdinalsBuilder appendOrd(long value);
529529

530-
TSIDOrdinalsBuilder appendOrds(long[] values, int from, int length);
530+
TSSingletonOrdinalsBuilder appendOrds(long[] values, int from, int length);
531531

532532
}
533533

‎server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -794,7 +794,11 @@ NamedAnalyzer normalizer() {
794794
@Override
795795
public BlockLoader blockLoader(BlockLoaderContext blContext) {
796796
if (hasDocValues() && (blContext.fieldExtractPreference() != FieldExtractPreference.STORED || isSyntheticSource)) {
797-
return new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(name());
797+
if (isDimension) {
798+
return new TSDimensionBlockLoader(name());
799+
} else {
800+
return new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(name());
801+
}
798802
}
799803
if (isStored()) {
800804
return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(name());

‎server/src/main/java/org/elasticsearch/index/mapper/TSDimensionBlockLoader.java‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.mapper;
11+
12+
import org.apache.lucene.index.DocValues;
13+
import org.apache.lucene.index.LeafReaderContext;
14+
import org.apache.lucene.index.SortedDocValues;
15+
import org.apache.lucene.index.SortedSetDocValues;
16+
import org.elasticsearch.index.codec.tsdb.es819.BlockAwareSortedDocValues;
17+
import org.elasticsearch.index.codec.tsdb.es819.SingletonDocValuesBlockLoader;
18+
import org.elasticsearch.search.fetch.StoredFieldsSpec;
19+
20+
import java.io.IOException;
21+
22+
public class TSDimensionBlockLoader implements BlockLoader {
23+
24+
private final String fieldName;
25+
private final BlockLoader fallback;
26+
27+
public TSDimensionBlockLoader(String fieldName) {
28+
this.fieldName = fieldName;
29+
this.fallback = new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(fieldName);
30+
}
31+
32+
@Override
33+
public Builder builder(BlockFactory factory, int expectedCount) {
34+
return factory.bytesRefs(expectedCount);
35+
}
36+
37+
@Override
38+
public ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) throws IOException {
39+
var singleton = DocValues.unwrapSingleton(context.reader().getSortedSetDocValues(fieldName));
40+
if (singleton instanceof BlockAwareSortedDocValues b) {
41+
return new TSDimensions(b);
42+
}
43+
return fallback.columnAtATimeReader(context);
44+
}
45+
46+
@Override
47+
public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOException {
48+
return fallback.rowStrideReader(context);
49+
}
50+
51+
@Override
52+
public StoredFieldsSpec rowStrideStoredFieldSpec() {
53+
return StoredFieldsSpec.NO_REQUIREMENTS;
54+
}
55+
56+
@Override
57+
public boolean supportsOrdinals() {
58+
return true;
59+
}
60+
61+
@Override
62+
public SortedSetDocValues ordinals(LeafReaderContext context) throws IOException {
63+
return DocValues.getSortedSet(context.reader(), fieldName);
64+
}
65+
66+
public String toString() {
67+
return "TSIDBlockLoader[" + fieldName + "]";
68+
}
69+
70+
public static final class TSDimensions implements ColumnAtATimeReader {
71+
private final Thread creationThread;
72+
private final SortedDocValues sorted;
73+
private final SingletonDocValuesBlockLoader blockLoader;
74+
75+
TSDimensions(BlockAwareSortedDocValues sorted) {
76+
this.creationThread = Thread.currentThread();
77+
this.sorted = sorted;
78+
this.blockLoader = sorted.getSingletonBlockLoader();
79+
}
80+
81+
@Override
82+
public Block read(BlockFactory factory, Docs docs, int offset) throws IOException {
83+
try (var builder = factory.tsSingletonOrdinalsBuilder(false, sorted, docs.count() - offset)) {
84+
blockLoader.loadBlock(builder, docs, offset);
85+
return builder.build();
86+
}
87+
}
88+
89+
@Override
90+
public boolean canReuse(int startingDocID) {
91+
return creationThread == Thread.currentThread() && blockLoader.docID() <= startingDocID;
92+
}
93+
94+
@Override
95+
public String toString() {
96+
return "TSDimensionBlockLoader.TSDimensions";
97+
}
98+
}
99+
}

‎server/src/main/java/org/elasticsearch/index/mapper/TSIDBlockLoader.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ public static final class TSIDs implements ColumnAtATimeReader {
6868

6969
@Override
7070
public Block read(BlockFactory factory, Docs docs, int offset) throws IOException {
71-
try (TSIDOrdinalsBuilder builder = factory.tsidOrdinalsBuilder(sorted, docs.count() - offset)) {
71+
try (TSSingletonOrdinalsBuilder builder = factory.tsSingletonOrdinalsBuilder(true, sorted, docs.count() - offset)) {
7272
blockLoader.loadBlock(builder, docs, offset);
7373
return builder.build();
7474
}

‎server/src/test/java/org/elasticsearch/index/mapper/TSIDBulkBlockLoadingTests.java‎ renamed to ‎server/src/test/java/org/elasticsearch/index/mapper/TSBulkBlockLoadingTests.java‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,16 +25,25 @@
2525
import org.elasticsearch.index.IndexVersion;
2626
import org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat;
2727

28+
import java.io.IOException;
2829
import java.util.Locale;
2930
import java.util.stream.IntStream;
3031

3132
import static org.hamcrest.Matchers.equalTo;
3233
import static org.hamcrest.Matchers.instanceOf;
3334
import static org.mockito.Mockito.mock;
3435

35-
public class TSIDBulkBlockLoadingTests extends MapperServiceTestCase {
36+
public class TSBulkBlockLoadingTests extends MapperServiceTestCase {
3637

37-
public void testManyValues() throws Exception {
38+
public void testManyTSIDs() throws IOException {
39+
doTestManyValues(TimeSeriesIdFieldMapper.NAME, TSIDBlockLoader.TSIDs.class);
40+
}
41+
42+
public void testManyDimensions() throws IOException {
43+
doTestManyValues("host_name", TSDimensionBlockLoader.TSDimensions.class);
44+
}
45+
46+
public void doTestManyValues(String fieldName, Class<?> expectedColumnReader) throws IOException {
3847
final String mappings = """
3948
{
4049
"_doc" : {
@@ -61,25 +70,25 @@ public void testManyValues() throws Exception {
6170
int uniqueTsidEvery = 200;
6271
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
6372
iwc.setLeafSorter(DataStream.TIMESERIES_LEAF_READERS_SORTER);
64-
iwc.setIndexSort(new Sort(new SortField(TimeSeriesIdFieldMapper.NAME, SortField.Type.STRING, false)));
73+
iwc.setIndexSort(new Sort(new SortField(fieldName, SortField.Type.STRING, false)));
6574
iwc.setCodec(TestUtil.alwaysDocValuesFormat(new ES819TSDBDocValuesFormat()));
6675
try (IndexWriter iw = new IndexWriter(directory, iwc)) {
6776
for (int i = from; i < to; i++) {
6877
LuceneDocument doc = new LuceneDocument();
6978
int tsid = i / uniqueTsidEvery;
70-
doc.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, new BytesRef(String.format(Locale.ROOT, "%04d", tsid))));
79+
doc.add(new SortedDocValuesField(fieldName, new BytesRef(String.format(Locale.ROOT, "%04d", tsid))));
7180
iw.addDocument(doc);
7281
}
7382
iw.forceMerge(1);
7483
}
7584
var mockBlockContext = mock(MappedFieldType.BlockLoaderContext.class);
76-
var blockLoader = mapperService.fieldType(TimeSeriesIdFieldMapper.NAME).blockLoader(mockBlockContext);
85+
var blockLoader = mapperService.fieldType(fieldName).blockLoader(mockBlockContext);
7786
try (DirectoryReader reader = DirectoryReader.open(directory)) {
7887
LeafReaderContext context = reader.leaves().get(0);
7988
{
8089
// One big doc block
8190
var columnReader = blockLoader.columnAtATimeReader(context);
82-
assertThat(columnReader, instanceOf(TSIDBlockLoader.TSIDs.class));
91+
assertThat(columnReader, instanceOf(expectedColumnReader));
8392
var docBlock = TestBlock.docs(IntStream.range(from, to).toArray());
8493
var block = (TestBlock) columnReader.read(TestBlock.factory(), docBlock, 0);
8594
assertThat(block.size(), equalTo(to - from));
@@ -93,7 +102,7 @@ public void testManyValues() throws Exception {
93102
// Smaller doc blocks
94103
int docBlockSize = 1000;
95104
var columnReader = blockLoader.columnAtATimeReader(context);
96-
assertThat(columnReader, instanceOf(TSIDBlockLoader.TSIDs.class));
105+
assertThat(columnReader, instanceOf(expectedColumnReader));
97106
for (int i = from; i < to; i += docBlockSize) {
98107
var docBlock = TestBlock.docs(IntStream.range(i, i + docBlockSize).toArray());
99108
var block = (TestBlock) columnReader.read(TestBlock.factory(), docBlock, 0);
@@ -108,7 +117,7 @@ public void testManyValues() throws Exception {
108117
{
109118
// One smaller doc block:
110119
var columnReader = blockLoader.columnAtATimeReader(context);
111-
assertThat(columnReader, instanceOf(TSIDBlockLoader.TSIDs.class));
120+
assertThat(columnReader, instanceOf(expectedColumnReader));
112121
var docBlock = TestBlock.docs(IntStream.range(1010, 2020).toArray());
113122
var block = (TestBlock) columnReader.read(TestBlock.factory(), docBlock, 0);
114123
assertThat(block.size(), equalTo(1010));
@@ -121,7 +130,7 @@ public void testManyValues() throws Exception {
121130
{
122131
// Read two tiny blocks:
123132
var columnReader = blockLoader.columnAtATimeReader(context);
124-
assertThat(columnReader, instanceOf(TSIDBlockLoader.TSIDs.class));
133+
assertThat(columnReader, instanceOf(expectedColumnReader));
125134
var docBlock = TestBlock.docs(IntStream.range(32, 64).toArray());
126135
var block = (TestBlock) columnReader.read(TestBlock.factory(), docBlock, 0);
127136
assertThat(block.size(), equalTo(32));

‎test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -272,20 +272,24 @@ public BlockLoader.Block constantBytes(BytesRef value, int count) {
272272
}
273273

274274
@Override
275-
public BlockLoader.TSIDOrdinalsBuilder tsidOrdinalsBuilder(SortedDocValues ordinals, int expectedCount) {
275+
public BlockLoader.TSSingletonOrdinalsBuilder tsSingletonOrdinalsBuilder(
276+
boolean isPrimaryIndexSortField,
277+
SortedDocValues ordinals,
278+
int expectedCount
279+
) {
276280
final long[] ords = new long[expectedCount];
277-
return new BlockLoader.TSIDOrdinalsBuilder() {
281+
return new BlockLoader.TSSingletonOrdinalsBuilder() {
278282

279283
int count;
280284

281285
@Override
282-
public BlockLoader.TSIDOrdinalsBuilder appendOrd(long value) {
286+
public BlockLoader.TSSingletonOrdinalsBuilder appendOrd(long value) {
283287
ords[count++] = value;
284288
return this;
285289
}
286290

287291
@Override
288-
public BlockLoader.TSIDOrdinalsBuilder appendOrds(long[] values, int from, int length) {
292+
public BlockLoader.TSSingletonOrdinalsBuilder appendOrds(long[] values, int from, int length) {
289293
try {
290294
System.arraycopy(values, from, ords, count, length);
291295
} catch (ArrayIndexOutOfBoundsException e) {

‎x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/DelegatingBlockLoaderFactory.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,8 +87,12 @@ public BlockLoader.Builder nulls(int expectedCount) {
8787
}
8888

8989
@Override
90-
public BlockLoader.TSIDOrdinalsBuilder tsidOrdinalsBuilder(SortedDocValues ordinals, int count) {
91-
return new TSIDOrdinalsBuilder(factory, ordinals, count);
90+
public BlockLoader.TSSingletonOrdinalsBuilder tsSingletonOrdinalsBuilder(
91+
boolean isPrimaryIndexSortField,
92+
SortedDocValues ordinals,
93+
int count
94+
) {
95+
return new TSSingletonOrdinalsBuilder(isPrimaryIndexSortField, factory, ordinals, count);
9296
}
9397

9498
@Override

0 commit comments

Comments
 (0)