Skip to content

Commit 4f6bacf

Browse files
martijnvgmartijnvg
committed
Improve reading ignored source in esql
* Parse sub field fields once in IgnoredSourceRowStrideReader instead of for each doc that gets read. * Introduce a dedicated StoredFieldLoader for ignored source (IgnoredSourceFieldLoader). Which optimizes reading stored fields just for ignored source by avoiding relatively expensive set stuff (see CustomFieldsVisitor) and make use aborting loading stored fields (`Status.STOP`) when ignored source is read.
1 parent 9513583 commit 4f6bacf

File tree

3 files changed

+154
-20
lines changed

3 files changed

+154
-20
lines changed

‎server/src/main/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoader.java‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.fieldvisitor;
11+
12+
import org.apache.lucene.index.FieldInfo;
13+
import org.apache.lucene.index.LeafReader;
14+
import org.apache.lucene.index.LeafReaderContext;
15+
import org.apache.lucene.index.StoredFieldVisitor;
16+
import org.apache.lucene.util.BytesRef;
17+
import org.elasticsearch.common.CheckedBiConsumer;
18+
import org.elasticsearch.common.bytes.BytesReference;
19+
import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader;
20+
import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
21+
import org.elasticsearch.search.fetch.StoredFieldsSpec;
22+
23+
import java.io.IOException;
24+
import java.util.ArrayList;
25+
import java.util.List;
26+
import java.util.Map;
27+
28+
class IgnoredSourceFieldLoader extends StoredFieldLoader {
29+
30+
@Override
31+
public LeafStoredFieldLoader getLoader(LeafReaderContext ctx, int[] docs) throws IOException {
32+
var reader = sequentialReader(ctx);
33+
var visitor = new SFV();
34+
return new LeafStoredFieldLoader() {
35+
36+
private int doc = -1;
37+
38+
@Override
39+
public void advanceTo(int doc) throws IOException {
40+
if (doc != this.doc) {
41+
visitor.reset();
42+
reader.accept(doc, visitor);
43+
this.doc = doc;
44+
}
45+
}
46+
47+
@Override
48+
public BytesReference source() {
49+
return null;
50+
}
51+
52+
@Override
53+
public String id() {
54+
return null;
55+
}
56+
57+
@Override
58+
public String routing() {
59+
return null;
60+
}
61+
62+
@Override
63+
public Map<String, List<Object>> storedFields() {
64+
return Map.of(IgnoredSourceFieldMapper.NAME, visitor.values);
65+
}
66+
};
67+
}
68+
69+
@Override
70+
public List<String> fieldsToLoad() {
71+
return List.of(IgnoredSourceFieldMapper.NAME);
72+
}
73+
74+
static class SFV extends StoredFieldVisitor {
75+
76+
boolean processing;
77+
final List<Object> values = new ArrayList<>();
78+
79+
@Override
80+
public Status needsField(FieldInfo fieldInfo) throws IOException {
81+
if (IgnoredSourceFieldMapper.NAME.equals(fieldInfo.name)) {
82+
processing = true;
83+
return Status.YES;
84+
} else if (processing) {
85+
return Status.STOP;
86+
}
87+
88+
return Status.NO;
89+
}
90+
91+
@Override
92+
public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
93+
values.add(new BytesRef(value));
94+
}
95+
96+
void reset() {
97+
values.clear();
98+
processing = false;
99+
}
100+
101+
}
102+
103+
static boolean supports(StoredFieldsSpec spec) {
104+
return spec.requiresSource() == false
105+
&& spec.requiresMetadata() == false
106+
&& spec.requiredStoredFields().size() == 1
107+
&& spec.requiredStoredFields().contains(IgnoredSourceFieldMapper.NAME);
108+
}
109+
110+
// TODO: use provided one
111+
private static CheckedBiConsumer<Integer, StoredFieldVisitor, IOException> sequentialReader(LeafReaderContext ctx) throws IOException {
112+
LeafReader leafReader = ctx.reader();
113+
if (leafReader instanceof SequentialStoredFieldsLeafReader lf) {
114+
return lf.getSequentialStoredFieldsReader()::document;
115+
}
116+
return leafReader.storedFields()::document;
117+
}
118+
}

‎server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,9 @@ public static StoredFieldLoader fromSpec(StoredFieldsSpec spec) {
5050
if (spec.noRequirements()) {
5151
return StoredFieldLoader.empty();
5252
}
53+
if (IgnoredSourceFieldLoader.supports(spec)) {
54+
return new IgnoredSourceFieldLoader();
55+
}
5356
return create(spec.requiresSource(), spec.requiredStoredFields());
5457
}
5558

@@ -91,6 +94,10 @@ public static StoredFieldLoader fromSpecSequential(StoredFieldsSpec spec) {
9194
if (spec.noRequirements()) {
9295
return StoredFieldLoader.empty();
9396
}
97+
if (IgnoredSourceFieldLoader.supports(spec)) {
98+
return new IgnoredSourceFieldLoader();
99+
}
100+
94101
List<String> fieldsToLoad = fieldsToLoad(spec.requiresSource(), spec.requiredStoredFields());
95102
return new StoredFieldLoader() {
96103
@Override

‎server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java‎

Lines changed: 29 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,12 @@
3939
public abstract class FallbackSyntheticSourceBlockLoader implements BlockLoader {
4040
private final Reader<?> reader;
4141
private final String fieldName;
42+
private final Set<String> fieldPaths;
4243

4344
protected FallbackSyntheticSourceBlockLoader(Reader<?> reader, String fieldName) {
4445
this.reader = reader;
4546
this.fieldName = fieldName;
47+
this.fieldPaths = splitIntoFieldPaths(fieldName);
4648
}
4749

4850
@Override
@@ -52,7 +54,7 @@ public ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) throws
5254

5355
@Override
5456
public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOException {
55-
return new IgnoredSourceRowStrideReader<>(fieldName, reader);
57+
return new IgnoredSourceRowStrideReader<>(fieldName, reader, fieldPaths);
5658
}
5759

5860
@Override
@@ -70,7 +72,31 @@ public SortedSetDocValues ordinals(LeafReaderContext context) throws IOException
7072
throw new UnsupportedOperationException();
7173
}
7274

73-
private record IgnoredSourceRowStrideReader<T>(String fieldName, Reader<T> reader) implements RowStrideReader {
75+
static Set<String> splitIntoFieldPaths(String fieldName) {
76+
var paths = new HashSet<String>();
77+
paths.add("_doc");
78+
var current = new StringBuilder();
79+
for (var part : fieldName.split("\\.")) {
80+
if (current.isEmpty() == false) {
81+
current.append('.');
82+
}
83+
current.append(part);
84+
paths.add(current.toString());
85+
}
86+
return paths;
87+
}
88+
89+
private static final class IgnoredSourceRowStrideReader<T> implements RowStrideReader {
90+
private final String fieldName;
91+
private final Reader<T> reader;
92+
private final Set<String> fieldPaths;
93+
94+
private IgnoredSourceRowStrideReader(String fieldName, Reader<T> reader, Set<String> fieldPaths) {
95+
this.fieldName = fieldName;
96+
this.reader = reader;
97+
this.fieldPaths = fieldPaths;
98+
}
99+
74100
@Override
75101
public void read(int docId, StoredFields storedFields, Builder builder) throws IOException {
76102
var ignoredSource = storedFields.storedFields().get(IgnoredSourceFieldMapper.NAME);
@@ -80,26 +106,9 @@ public void read(int docId, StoredFields storedFields, Builder builder) throws I
80106
}
81107

82108
Map<String, List<IgnoredSourceFieldMapper.NameValue>> valuesForFieldAndParents = new HashMap<>();
83-
84-
// Contains name of the field and all its parents
85-
Set<String> fieldNames = new HashSet<>() {
86-
{
87-
add("_doc");
88-
}
89-
};
90-
91-
var current = new StringBuilder();
92-
for (String part : fieldName.split("\\.")) {
93-
if (current.isEmpty() == false) {
94-
current.append('.');
95-
}
96-
current.append(part);
97-
fieldNames.add(current.toString());
98-
}
99-
100109
for (Object value : ignoredSource) {
101110
IgnoredSourceFieldMapper.NameValue nameValue = IgnoredSourceFieldMapper.decode(value);
102-
if (fieldNames.contains(nameValue.name())) {
111+
if (fieldPaths.contains(nameValue.name())) {
103112
valuesForFieldAndParents.computeIfAbsent(nameValue.name(), k -> new ArrayList<>()).add(nameValue);
104113
}
105114
}

0 commit comments

Comments
 (0)