Skip to content

Commit 2eb25be

Browse files
nik9000nik9000
authored
ESQL: Most remaining block loads from MV_MIN and MV_MAX (elastic#137820)
Implements most remaining block loaders for MV_MIN and MV_MAX. Once elastic#137382 is in we can push MV_MIN and MV_MAX into the block loaders for most field types. This is compelling it significantly reduces the amount of data loaded when using MV_MIN and MV_MAX.
1 parent 02d48fb commit 2eb25be

File tree

47 files changed

+1864
-433
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+1864
-433
lines changed

‎server/src/main/java/module-info.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -500,6 +500,7 @@
500500
exports org.elasticsearch.index.codec.vectors.cluster to org.elasticsearch.test.knn;
501501
exports org.elasticsearch.index.codec.vectors.es93 to org.elasticsearch.test.knn;
502502
exports org.elasticsearch.search.crossproject;
503-
exports org.elasticsearch.index.mapper.blockloader.docvalues;
504503
exports org.elasticsearch.index.mapper.blockloader;
504+
exports org.elasticsearch.index.mapper.blockloader.docvalues;
505+
exports org.elasticsearch.index.mapper.blockloader.docvalues.fn;
505506
}

‎server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@
6161
import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
6262
import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig;
6363
import org.elasticsearch.index.mapper.blockloader.docvalues.BytesRefsFromOrdsBlockLoader;
64-
import org.elasticsearch.index.mapper.blockloader.docvalues.Utf8CodePointsFromOrdsBlockLoader;
64+
import org.elasticsearch.index.mapper.blockloader.docvalues.fn.Utf8CodePointsFromOrdsBlockLoader;
6565
import org.elasticsearch.index.query.AutomatonQueryWithDescription;
6666
import org.elasticsearch.index.query.SearchExecutionContext;
6767
import org.elasticsearch.index.similarity.SimilarityProvider;

‎server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractBooleansBlockLoader.java‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.mapper.blockloader.docvalues;
11+
12+
import org.apache.lucene.index.DocValues;
13+
import org.apache.lucene.index.LeafReaderContext;
14+
import org.apache.lucene.index.NumericDocValues;
15+
import org.apache.lucene.index.SortedNumericDocValues;
16+
17+
import java.io.IOException;
18+
19+
/**
20+
* Loads {@code boolean}s from doc values.
21+
*/
22+
public abstract class AbstractBooleansBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader {
23+
protected final String fieldName;
24+
25+
public AbstractBooleansBlockLoader(String fieldName) {
26+
this.fieldName = fieldName;
27+
}
28+
29+
@Override
30+
public final BooleanBuilder builder(BlockFactory factory, int expectedCount) {
31+
return factory.booleans(expectedCount);
32+
}
33+
34+
@Override
35+
public AllReader reader(LeafReaderContext context) throws IOException {
36+
SortedNumericDocValues docValues = context.reader().getSortedNumericDocValues(fieldName);
37+
if (docValues != null) {
38+
NumericDocValues singleton = DocValues.unwrapSingleton(docValues);
39+
if (singleton != null) {
40+
return singletonReader(singleton);
41+
}
42+
return sortedReader(docValues);
43+
}
44+
NumericDocValues singleton = context.reader().getNumericDocValues(fieldName);
45+
if (singleton != null) {
46+
return singletonReader(singleton);
47+
}
48+
return new ConstantNullsReader();
49+
}
50+
51+
protected abstract AllReader singletonReader(NumericDocValues docValues);
52+
53+
protected abstract AllReader sortedReader(SortedNumericDocValues docValues);
54+
55+
public static class Singleton extends BlockDocValuesReader {
56+
private final NumericDocValues numericDocValues;
57+
58+
public Singleton(NumericDocValues numericDocValues) {
59+
this.numericDocValues = numericDocValues;
60+
}
61+
62+
@Override
63+
public Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException {
64+
try (BooleanBuilder builder = factory.booleansFromDocValues(docs.count() - offset)) {
65+
int lastDoc = -1;
66+
for (int i = offset; i < docs.count(); i++) {
67+
int doc = docs.get(i);
68+
if (doc < lastDoc) {
69+
throw new IllegalStateException("docs within same block must be in order");
70+
}
71+
if (numericDocValues.advanceExact(doc)) {
72+
builder.appendBoolean(numericDocValues.longValue() != 0);
73+
} else {
74+
builder.appendNull();
75+
}
76+
lastDoc = doc;
77+
}
78+
return builder.build();
79+
}
80+
}
81+
82+
@Override
83+
public void read(int docId, StoredFields storedFields, Builder builder) throws IOException {
84+
BooleanBuilder blockBuilder = (BooleanBuilder) builder;
85+
if (numericDocValues.advanceExact(docId)) {
86+
blockBuilder.appendBoolean(numericDocValues.longValue() != 0);
87+
} else {
88+
blockBuilder.appendNull();
89+
}
90+
}
91+
92+
@Override
93+
public int docId() {
94+
return numericDocValues.docID();
95+
}
96+
97+
@Override
98+
public String toString() {
99+
return "BooleansFromDocValues.Singleton";
100+
}
101+
}
102+
103+
static class Sorted extends BlockDocValuesReader {
104+
private final SortedNumericDocValues numericDocValues;
105+
106+
Sorted(SortedNumericDocValues numericDocValues) {
107+
this.numericDocValues = numericDocValues;
108+
}
109+
110+
@Override
111+
public Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException {
112+
try (BooleanBuilder builder = factory.booleansFromDocValues(docs.count() - offset)) {
113+
for (int i = offset; i < docs.count(); i++) {
114+
int doc = docs.get(i);
115+
read(doc, builder);
116+
}
117+
return builder.build();
118+
}
119+
}
120+
121+
@Override
122+
public void read(int docId, StoredFields storedFields, Builder builder) throws IOException {
123+
read(docId, (BooleanBuilder) builder);
124+
}
125+
126+
private void read(int doc, BooleanBuilder builder) throws IOException {
127+
if (false == numericDocValues.advanceExact(doc)) {
128+
builder.appendNull();
129+
return;
130+
}
131+
int count = numericDocValues.docValueCount();
132+
if (count == 1) {
133+
builder.appendBoolean(numericDocValues.nextValue() != 0);
134+
return;
135+
}
136+
builder.beginPositionEntry();
137+
for (int v = 0; v < count; v++) {
138+
builder.appendBoolean(numericDocValues.nextValue() != 0);
139+
}
140+
builder.endPositionEntry();
141+
}
142+
143+
@Override
144+
public int docId() {
145+
return numericDocValues.docID();
146+
}
147+
148+
@Override
149+
public String toString() {
150+
return "BooleansFromDocValues.Sorted";
151+
}
152+
}
153+
}

‎server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractBytesRefsFromOrdsBlockLoader.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,10 @@
2020
/**
2121
* Loads {@code keyword} style fields that are stored as a lookup table.
2222
*/
23-
abstract class AbstractBytesRefsFromOrdsBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader {
23+
public abstract class AbstractBytesRefsFromOrdsBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader {
2424
protected final String fieldName;
2525

26-
AbstractBytesRefsFromOrdsBlockLoader(String fieldName) {
26+
public AbstractBytesRefsFromOrdsBlockLoader(String fieldName) {
2727
this.fieldName = fieldName;
2828
}
2929

@@ -56,7 +56,7 @@ public final AllReader reader(LeafReaderContext context) throws IOException {
5656
protected static class Singleton extends BlockDocValuesReader {
5757
private final SortedDocValues ordinals;
5858

59-
Singleton(SortedDocValues ordinals) {
59+
public Singleton(SortedDocValues ordinals) {
6060
this.ordinals = ordinals;
6161
}
6262

‎server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractDoublesFromDocValuesBlockLoader.java‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.mapper.blockloader.docvalues;
11+
12+
import org.apache.lucene.index.DocValues;
13+
import org.apache.lucene.index.LeafReaderContext;
14+
import org.apache.lucene.index.NumericDocValues;
15+
import org.apache.lucene.index.SortedNumericDocValues;
16+
17+
import java.io.IOException;
18+
19+
/**
20+
* Loads {@code double}s from doc values.
21+
*/
22+
public abstract class AbstractDoublesFromDocValuesBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader {
23+
protected final String fieldName;
24+
private final BlockDocValuesReader.ToDouble toDouble;
25+
26+
public AbstractDoublesFromDocValuesBlockLoader(String fieldName, BlockDocValuesReader.ToDouble toDouble) {
27+
this.fieldName = fieldName;
28+
this.toDouble = toDouble;
29+
}
30+
31+
@Override
32+
public final Builder builder(BlockFactory factory, int expectedCount) {
33+
return factory.doubles(expectedCount);
34+
}
35+
36+
@Override
37+
public final AllReader reader(LeafReaderContext context) throws IOException {
38+
SortedNumericDocValues docValues = context.reader().getSortedNumericDocValues(fieldName);
39+
if (docValues != null) {
40+
NumericDocValues singleton = DocValues.unwrapSingleton(docValues);
41+
if (singleton != null) {
42+
return singletonReader(singleton, toDouble);
43+
}
44+
return sortedReader(docValues, toDouble);
45+
}
46+
NumericDocValues singleton = context.reader().getNumericDocValues(fieldName);
47+
if (singleton != null) {
48+
return singletonReader(singleton, toDouble);
49+
}
50+
return new ConstantNullsReader();
51+
}
52+
53+
protected abstract AllReader singletonReader(NumericDocValues docValues, BlockDocValuesReader.ToDouble toDouble);
54+
55+
protected abstract AllReader sortedReader(SortedNumericDocValues docValues, BlockDocValuesReader.ToDouble toDouble);
56+
57+
public static class Singleton extends BlockDocValuesReader implements BlockDocValuesReader.NumericDocValuesAccessor {
58+
private final NumericDocValues docValues;
59+
private final ToDouble toDouble;
60+
61+
public Singleton(NumericDocValues docValues, ToDouble toDouble) {
62+
this.docValues = docValues;
63+
this.toDouble = toDouble;
64+
}
65+
66+
@Override
67+
public Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException {
68+
if (docValues instanceof OptionalColumnAtATimeReader direct) {
69+
Block result = direct.tryRead(factory, docs, offset, nullsFiltered, toDouble, false);
70+
if (result != null) {
71+
return result;
72+
}
73+
}
74+
try (DoubleBuilder builder = factory.doublesFromDocValues(docs.count() - offset)) {
75+
for (int i = offset; i < docs.count(); i++) {
76+
int doc = docs.get(i);
77+
if (docValues.advanceExact(doc)) {
78+
builder.appendDouble(toDouble.convert(docValues.longValue()));
79+
} else {
80+
builder.appendNull();
81+
}
82+
}
83+
return builder.build();
84+
}
85+
}
86+
87+
@Override
88+
public void read(int docId, StoredFields storedFields, Builder builder) throws IOException {
89+
DoubleBuilder blockBuilder = (DoubleBuilder) builder;
90+
if (docValues.advanceExact(docId)) {
91+
blockBuilder.appendDouble(toDouble.convert(docValues.longValue()));
92+
} else {
93+
blockBuilder.appendNull();
94+
}
95+
}
96+
97+
@Override
98+
public int docId() {
99+
return docValues.docID();
100+
}
101+
102+
@Override
103+
public String toString() {
104+
return "DoublesFromDocValues.Singleton";
105+
}
106+
107+
@Override
108+
public NumericDocValues numericDocValues() {
109+
return docValues;
110+
}
111+
}
112+
113+
public static class Sorted extends BlockDocValuesReader {
114+
private final SortedNumericDocValues docValues;
115+
private final ToDouble toDouble;
116+
117+
Sorted(SortedNumericDocValues docValues, ToDouble toDouble) {
118+
this.docValues = docValues;
119+
this.toDouble = toDouble;
120+
}
121+
122+
@Override
123+
public Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException {
124+
try (DoubleBuilder builder = factory.doublesFromDocValues(docs.count() - offset)) {
125+
for (int i = offset; i < docs.count(); i++) {
126+
int doc = docs.get(i);
127+
read(doc, builder);
128+
}
129+
return builder.build();
130+
}
131+
}
132+
133+
@Override
134+
public void read(int docId, StoredFields storedFields, Builder builder) throws IOException {
135+
read(docId, (DoubleBuilder) builder);
136+
}
137+
138+
private void read(int doc, DoubleBuilder builder) throws IOException {
139+
if (false == docValues.advanceExact(doc)) {
140+
builder.appendNull();
141+
return;
142+
}
143+
int count = docValues.docValueCount();
144+
if (count == 1) {
145+
builder.appendDouble(toDouble.convert(docValues.nextValue()));
146+
return;
147+
}
148+
builder.beginPositionEntry();
149+
for (int v = 0; v < count; v++) {
150+
builder.appendDouble(toDouble.convert(docValues.nextValue()));
151+
}
152+
builder.endPositionEntry();
153+
}
154+
155+
@Override
156+
public int docId() {
157+
return docValues.docID();
158+
}
159+
160+
@Override
161+
public String toString() {
162+
return "DoublesFromDocValues.Sorted";
163+
}
164+
}
165+
}

‎server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractIntsFromDocValuesBlockLoader.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
public abstract class AbstractIntsFromDocValuesBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader {
2323
protected final String fieldName;
2424

25-
AbstractIntsFromDocValuesBlockLoader(String fieldName) {
25+
protected AbstractIntsFromDocValuesBlockLoader(String fieldName) {
2626
this.fieldName = fieldName;
2727
}
2828

@@ -55,7 +55,7 @@ public final AllReader reader(LeafReaderContext context) throws IOException {
5555
public static class Singleton extends BlockDocValuesReader implements BlockDocValuesReader.NumericDocValuesAccessor {
5656
private final NumericDocValues numericDocValues;
5757

58-
Singleton(NumericDocValues numericDocValues) {
58+
public Singleton(NumericDocValues numericDocValues) {
5959
this.numericDocValues = numericDocValues;
6060
}
6161

0 commit comments

Comments
 (0)