Skip to content

Commit 83d180d

Browse files
lcawllcawl
authored
Merge branch 'main' into dsl-tutorial
2 parents 59b1bab + e248da6 commit 83d180d

File tree

31 files changed

+379
-76
lines changed

31 files changed

+379
-76
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -142,11 +142,26 @@ static void selfTest() {
142142
private static List<ValuesSourceReaderOperator.FieldInfo> fields(String name) {
143143
return switch (name) {
144144
case "3_stored_keywords" -> List.of(
145-
new ValuesSourceReaderOperator.FieldInfo("keyword_1", ElementType.BYTES_REF, shardIdx -> blockLoader("stored_keyword_1")),
146-
new ValuesSourceReaderOperator.FieldInfo("keyword_2", ElementType.BYTES_REF, shardIdx -> blockLoader("stored_keyword_2")),
147-
new ValuesSourceReaderOperator.FieldInfo("keyword_3", ElementType.BYTES_REF, shardIdx -> blockLoader("stored_keyword_3"))
145+
new ValuesSourceReaderOperator.FieldInfo(
146+
"keyword_1",
147+
ElementType.BYTES_REF,
148+
false,
149+
shardIdx -> blockLoader("stored_keyword_1")
150+
),
151+
new ValuesSourceReaderOperator.FieldInfo(
152+
"keyword_2",
153+
ElementType.BYTES_REF,
154+
false,
155+
shardIdx -> blockLoader("stored_keyword_2")
156+
),
157+
new ValuesSourceReaderOperator.FieldInfo(
158+
"keyword_3",
159+
ElementType.BYTES_REF,
160+
false,
161+
shardIdx -> blockLoader("stored_keyword_3")
162+
)
148163
);
149-
default -> List.of(new ValuesSourceReaderOperator.FieldInfo(name, elementType(name), shardIdx -> blockLoader(name)));
164+
default -> List.of(new ValuesSourceReaderOperator.FieldInfo(name, elementType(name), false, shardIdx -> blockLoader(name)));
150165
};
151166
}
152167

‎muted-tests.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -603,6 +603,9 @@ tests:
603603
- class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
604604
method: test {csv-spec:change_point.Values null column}
605605
issue: https://github.com/elastic/elasticsearch/issues/133334
606+
- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
607+
method: test {p0=search/110_field_collapsing/field collapsing}
608+
issue: https://github.com/elastic/elasticsearch/issues/133361
606609

607610
# Examples:
608611
#

‎server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,12 @@ protected void writeExtent(BlockLoader.IntBuilder builder, Extent extent) {
9898
public BlockLoader.AllReader reader(LeafReaderContext context) throws IOException {
9999
return new BlockLoader.AllReader() {
100100
@Override
101-
public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs, int offset) throws IOException {
101+
public BlockLoader.Block read(
102+
BlockLoader.BlockFactory factory,
103+
BlockLoader.Docs docs,
104+
int offset,
105+
boolean nullsFiltered
106+
) throws IOException {
102107
var binaryDocValues = context.reader().getBinaryDocValues(fieldName);
103108
var reader = new GeometryDocValueReader();
104109
try (var builder = factory.ints(docs.count() - offset)) {

‎server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ static class SingletonLongs extends BlockDocValuesReader {
130130
}
131131

132132
@Override
133-
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset) throws IOException {
133+
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException {
134134
if (numericDocValues instanceof BlockLoader.OptionalColumnAtATimeReader direct) {
135135
BlockLoader.Block result = direct.tryRead(factory, docs, offset);
136136
if (result != null) {
@@ -179,7 +179,7 @@ static class Longs extends BlockDocValuesReader {
179179
}
180180

181181
@Override
182-
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset) throws IOException {
182+
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException {
183183
try (BlockLoader.LongBuilder builder = factory.longsFromDocValues(docs.count() - offset)) {
184184
for (int i = offset; i < docs.count(); i++) {
185185
int doc = docs.get(i);
@@ -260,7 +260,7 @@ private static class SingletonInts extends BlockDocValuesReader {
260260
}
261261

262262
@Override
263-
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset) throws IOException {
263+
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException {
264264
try (BlockLoader.IntBuilder builder = factory.intsFromDocValues(docs.count() - offset)) {
265265
for (int i = offset; i < docs.count(); i++) {
266266
int doc = docs.get(i);
@@ -303,7 +303,7 @@ private static class Ints extends BlockDocValuesReader {
303303
}
304304

305305
@Override
306-
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset) throws IOException {
306+
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException {
307307
try (BlockLoader.IntBuilder builder = factory.intsFromDocValues(docs.count() - offset)) {
308308
for (int i = offset; i < docs.count(); i++) {
309309
int doc = docs.get(i);
@@ -397,7 +397,7 @@ private static class SingletonDoubles extends BlockDocValuesReader {
397397
}
398398

399399
@Override
400-
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset) throws IOException {
400+
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException {
401401
try (BlockLoader.DoubleBuilder builder = factory.doublesFromDocValues(docs.count() - offset)) {
402402
for (int i = offset; i < docs.count(); i++) {
403403
int doc = docs.get(i);
@@ -442,7 +442,7 @@ private static class Doubles extends BlockDocValuesReader {
442442
}
443443

444444
@Override
445-
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset) throws IOException {
445+
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException {
446446
try (BlockLoader.DoubleBuilder builder = factory.doublesFromDocValues(docs.count() - offset)) {
447447
for (int i = offset; i < docs.count(); i++) {
448448
int doc = docs.get(i);
@@ -540,7 +540,7 @@ private abstract static class DenseVectorValuesBlockReader<T extends KnnVectorVa
540540
}
541541

542542
@Override
543-
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset) throws IOException {
543+
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException {
544544
// Doubles from doc values ensures that the values are in order
545545
try (BlockLoader.FloatBuilder builder = factory.denseVectors(docs.count() - offset, dimensions)) {
546546
for (int i = offset; i < docs.count(); i++) {
@@ -710,7 +710,7 @@ private BlockLoader.Block readSingleDoc(BlockFactory factory, int docId) throws
710710
}
711711

712712
@Override
713-
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset) throws IOException {
713+
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException {
714714
if (docs.count() - offset == 1) {
715715
return readSingleDoc(factory, docs.get(offset));
716716
}
@@ -761,7 +761,7 @@ private static class Ordinals extends BlockDocValuesReader {
761761
}
762762

763763
@Override
764-
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset) throws IOException {
764+
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException {
765765
if (docs.count() - offset == 1) {
766766
return readSingleDoc(factory, docs.get(offset));
767767
}
@@ -875,7 +875,7 @@ private static class BytesRefsFromBinary extends BlockDocValuesReader {
875875
}
876876

877877
@Override
878-
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset) throws IOException {
878+
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException {
879879
try (BlockLoader.BytesRefBuilder builder = factory.bytesRefs(docs.count() - offset)) {
880880
for (int i = offset; i < docs.count(); i++) {
881881
int doc = docs.get(i);
@@ -988,7 +988,7 @@ public void read(int docId, BlockLoader.StoredFields storedFields, Builder build
988988
}
989989

990990
@Override
991-
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset) throws IOException {
991+
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException {
992992
try (BlockLoader.FloatBuilder builder = factory.denseVectors(docs.count() - offset, dimensions)) {
993993
for (int i = offset; i < docs.count(); i++) {
994994
int doc = docs.get(i);
@@ -1099,7 +1099,7 @@ private static class SingletonBooleans extends BlockDocValuesReader {
10991099
}
11001100

11011101
@Override
1102-
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset) throws IOException {
1102+
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException {
11031103
try (BlockLoader.BooleanBuilder builder = factory.booleansFromDocValues(docs.count() - offset)) {
11041104
int lastDoc = -1;
11051105
for (int i = offset; i < docs.count(); i++) {
@@ -1147,7 +1147,7 @@ private static class Booleans extends BlockDocValuesReader {
11471147
}
11481148

11491149
@Override
1150-
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset) throws IOException {
1150+
public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException {
11511151
try (BlockLoader.BooleanBuilder builder = factory.booleansFromDocValues(docs.count() - offset)) {
11521152
for (int i = offset; i < docs.count(); i++) {
11531153
int doc = docs.get(i);

‎server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,15 @@ interface Reader {
4343
interface ColumnAtATimeReader extends Reader {
4444
/**
4545
* Reads the values of all documents in {@code docs}.
46-
*/
47-
BlockLoader.Block read(BlockFactory factory, Docs docs, int offset) throws IOException;
46+
*
47+
* @param nullsFiltered if {@code true}, then target docs are guaranteed to have a value for the field;
48+
* otherwise, the guarantee is unknown. This enables optimizations for block loaders,
49+
* treating the field as dense (every document has value) even if it is sparse in
50+
* the index. For example, "FROM index | WHERE x != null | STATS sum(x)", after filtering out
51+
* documents without value for field x, all target documents returned from the source operator
52+
* will have a value for field x whether x is dense or sparse in the index.
53+
*/
54+
BlockLoader.Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException;
4855
}
4956

5057
/**
@@ -166,7 +173,7 @@ public String toString() {
166173
*/
167174
class ConstantNullsReader implements AllReader {
168175
@Override
169-
public Block read(BlockFactory factory, Docs docs, int offset) throws IOException {
176+
public Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException {
170177
return factory.constantNulls(docs.count() - offset);
171178
}
172179

@@ -200,7 +207,7 @@ public Builder builder(BlockFactory factory, int expectedCount) {
200207
public ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) {
201208
return new ColumnAtATimeReader() {
202209
@Override
203-
public Block read(BlockFactory factory, Docs docs, int offset) {
210+
public Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) {
204211
return factory.constantBytes(value, docs.count() - offset);
205212
}
206213

@@ -278,8 +285,8 @@ public ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) throws
278285
}
279286
return new ColumnAtATimeReader() {
280287
@Override
281-
public Block read(BlockFactory factory, Docs docs, int offset) throws IOException {
282-
return reader.read(factory, docs, offset);
288+
public Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException {
289+
return reader.read(factory, docs, offset, nullsFiltered);
283290
}
284291

285292
@Override

‎server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptBlockDocValuesReader.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,8 @@ public int docId() {
4949
}
5050

5151
@Override
52-
public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs, int offset) throws IOException {
52+
public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs, int offset, boolean nullsFiltered)
53+
throws IOException {
5354
// Note that we don't emit falses before trues so we conform to the doc values contract and can use booleansFromDocValues
5455
try (BlockLoader.BooleanBuilder builder = factory.booleans(docs.count() - offset)) {
5556
for (int i = offset; i < docs.count(); i++) {

‎server/src/main/java/org/elasticsearch/index/mapper/DateScriptBlockDocValuesReader.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,8 @@ public int docId() {
4949
}
5050

5151
@Override
52-
public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs, int offset) throws IOException {
52+
public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs, int offset, boolean nullsFiltered)
53+
throws IOException {
5354
// Note that we don't sort the values sort, so we can't use factory.longsFromDocValues
5455
try (BlockLoader.LongBuilder builder = factory.longs(docs.count() - offset)) {
5556
for (int i = offset; i < docs.count(); i++) {

‎server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptBlockDocValuesReader.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,8 @@ public int docId() {
4949
}
5050

5151
@Override
52-
public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs, int offset) throws IOException {
52+
public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs, int offset, boolean nullsFiltered)
53+
throws IOException {
5354
// Note that we don't sort the values sort, so we can't use factory.doublesFromDocValues
5455
try (BlockLoader.DoubleBuilder builder = factory.doubles(docs.count() - offset)) {
5556
for (int i = offset; i < docs.count(); i++) {

‎server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -339,6 +339,7 @@ public boolean newDynamicStringField(DocumentParserContext context, String name)
339339
new TextFieldMapper.Builder(
340340
name,
341341
indexSettings.getIndexVersionCreated(),
342+
indexSettings.getMode(),
342343
context.indexAnalyzers(),
343344
SourceFieldMapper.isSynthetic(indexSettings),
344345
false

‎server/src/main/java/org/elasticsearch/index/mapper/IpScriptBlockDocValuesReader.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,8 @@ public int docId() {
4949
}
5050

5151
@Override
52-
public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs, int offset) throws IOException {
52+
public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs, int offset, boolean nullsFiltered)
53+
throws IOException {
5354
// Note that we don't pre-sort our output so we can't use bytesRefsFromDocValues
5455
try (BlockLoader.BytesRefBuilder builder = factory.bytesRefs(docs.count() - offset)) {
5556
for (int i = offset; i < docs.count(); i++) {

0 commit comments

Comments
 (0)