Skip to content

Commit 2e1f047

Browse files
JeremyDahlgrenJeremyDahlgren
authored
Merge branch 'main' into fix/127466
2 parents 37ba3d6 + 5964ad7 commit 2e1f047

File tree

17 files changed

+414
-35
lines changed

17 files changed

+414
-35
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.apache.lucene.util.NumericUtils;
2626
import org.elasticsearch.common.breaker.NoopCircuitBreaker;
2727
import org.elasticsearch.common.lucene.Lucene;
28+
import org.elasticsearch.common.settings.Settings;
2829
import org.elasticsearch.common.util.BigArrays;
2930
import org.elasticsearch.compute.data.BlockFactory;
3031
import org.elasticsearch.compute.data.BytesRefBlock;
@@ -50,6 +51,7 @@
5051
import org.elasticsearch.index.mapper.MappedFieldType;
5152
import org.elasticsearch.index.mapper.NumberFieldMapper;
5253
import org.elasticsearch.search.lookup.SearchLookup;
54+
import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
5355
import org.openjdk.jmh.annotations.Benchmark;
5456
import org.openjdk.jmh.annotations.BenchmarkMode;
5557
import org.openjdk.jmh.annotations.Fork;
@@ -335,7 +337,7 @@ public void benchmark() {
335337
fields(name),
336338
List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> {
337339
throw new UnsupportedOperationException("can't load _source here");
338-
})),
340+
}, EsqlPlugin.STORED_FIELDS_SEQUENTIAL_PROPORTION.getDefault(Settings.EMPTY))),
339341
0
340342
);
341343
long sum = 0;

‎docs/changelog/127348.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 127348
2+
summary: Speed loading stored fields
3+
area: ES|QL
4+
type: enhancement
5+
issues: []

‎docs/changelog/127527.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 127527
2+
summary: "No, line noise isn't a valid ip"
3+
area: ES|QL
4+
type: bug
5+
issues: []

‎docs/reference/elasticsearch/index-settings/index-modules.md‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ $$$index-codec$$$ `index.codec`
4949

5050
$$$index-mode-setting$$$ `index.mode`
5151
: The `index.mode` setting is used to control settings applied in specific domains like ingestion of time series data or logs. Different mutually exclusive modes exist, which are used to apply settings or default values controlling indexing of documents, sorting and other parameters whose value affects indexing or query performance.
52-
52+
5353
**Example**
5454

5555
```console
@@ -248,3 +248,8 @@ $$$index-final-pipeline$$$
248248

249249
$$$index-hidden$$$ `index.hidden`
250250
: Indicates whether the index should be hidden by default. Hidden indices are not returned by default when using a wildcard expression. This behavior is controlled per request through the use of the `expand_wildcards` parameter. Possible values are `true` and `false` (default).
251+
252+
$$$index-esql-stored-fields-sequential-proportion$$$
253+
254+
`index.esql.stored_fields_sequential_proportion`
255+
: Tuning parameter for deciding when {{esql}} will load [Stored fields](/reference/elasticsearch/rest-apis/retrieve-selected-fields.md#stored-fields) using a strategy tuned for loading dense sequence of documents. Allows values between 0.0 and 1.0 and defaults to 0.2. Indices with documents smaller than 10kb may see speed improvements loading `text` fields by setting this lower.

‎x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ public String describe() {
107107
*/
108108
public record FieldInfo(String name, ElementType type, IntFunction<BlockLoader> blockLoader) {}
109109

110-
public record ShardContext(IndexReader reader, Supplier<SourceLoader> newSourceLoader) {}
110+
public record ShardContext(IndexReader reader, Supplier<SourceLoader> newSourceLoader, double storedFieldsSequentialProportion) {}
111111

112112
private final FieldWork[] fields;
113113
private final List<ShardContext> shardContexts;
@@ -247,8 +247,9 @@ private void loadFromSingleLeaf(Block[] blocks, int shard, int segment, BlockLoa
247247
}
248248

249249
SourceLoader sourceLoader = null;
250+
ShardContext shardContext = shardContexts.get(shard);
250251
if (storedFieldsSpec.requiresSource()) {
251-
sourceLoader = shardContexts.get(shard).newSourceLoader.get();
252+
sourceLoader = shardContext.newSourceLoader.get();
252253
storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(true, false, sourceLoader.requiredStoredFields()));
253254
}
254255

@@ -261,7 +262,7 @@ private void loadFromSingleLeaf(Block[] blocks, int shard, int segment, BlockLoa
261262
);
262263
}
263264
StoredFieldLoader storedFieldLoader;
264-
if (useSequentialStoredFieldsReader(docs)) {
265+
if (useSequentialStoredFieldsReader(docs, shardContext.storedFieldsSequentialProportion())) {
265266
storedFieldLoader = StoredFieldLoader.fromSpecSequential(storedFieldsSpec);
266267
trackStoredFields(storedFieldsSpec, true);
267268
} else {
@@ -438,9 +439,13 @@ public void close() {
438439
* Is it more efficient to use a sequential stored field reader
439440
* when reading stored fields for the documents contained in {@code docIds}?
440441
*/
441-
private boolean useSequentialStoredFieldsReader(BlockLoader.Docs docs) {
442+
private boolean useSequentialStoredFieldsReader(BlockLoader.Docs docs, double storedFieldsSequentialProportion) {
442443
int count = docs.count();
443-
return count >= SEQUENTIAL_BOUNDARY && docs.get(count - 1) - docs.get(0) == count - 1;
444+
if (count < SEQUENTIAL_BOUNDARY) {
445+
return false;
446+
}
447+
int range = docs.get(count - 1) - docs.get(0);
448+
return range * storedFieldsSequentialProportion <= count;
444449
}
445450

446451
private void trackStoredFields(StoredFieldsSpec spec, boolean sequential) {

‎x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,7 @@ public String toString() {
198198
operators.add(
199199
new OrdinalsGroupingOperator(
200200
shardIdx -> new KeywordFieldMapper.KeywordFieldType("g").blockLoader(mockBlContext()),
201-
List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> SourceLoader.FROM_STORED_SOURCE)),
201+
List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> SourceLoader.FROM_STORED_SOURCE, 0.2)),
202202
ElementType.BYTES_REF,
203203
0,
204204
gField,

‎x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneQueryEvaluatorTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,7 @@ private List<Page> runQuery(Set<String> values, Query query, boolean shuffleDocs
208208
),
209209
List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> {
210210
throw new UnsupportedOperationException();
211-
})),
211+
}, 0.2)),
212212
0
213213
)
214214
);

‎x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -200,7 +200,7 @@ private MapperService mapperService(String indexKey) {
200200
private List<ValuesSourceReaderOperator.ShardContext> initShardContexts() {
201201
return INDICES.keySet()
202202
.stream()
203-
.map(index -> new ValuesSourceReaderOperator.ShardContext(reader(index), () -> SourceLoader.FROM_STORED_SOURCE))
203+
.map(index -> new ValuesSourceReaderOperator.ShardContext(reader(index), () -> SourceLoader.FROM_STORED_SOURCE, 0.2))
204204
.toList();
205205
}
206206

@@ -1297,7 +1297,7 @@ public void testWithNulls() throws IOException {
12971297
LuceneOperator.NO_LIMIT,
12981298
false // no scoring
12991299
);
1300-
var vsShardContext = new ValuesSourceReaderOperator.ShardContext(reader(indexKey), () -> SourceLoader.FROM_STORED_SOURCE);
1300+
var vsShardContext = new ValuesSourceReaderOperator.ShardContext(reader(indexKey), () -> SourceLoader.FROM_STORED_SOURCE, 0.2);
13011301
try (
13021302
Driver driver = TestDriverFactory.create(
13031303
driverContext,
@@ -1415,7 +1415,7 @@ public void testDescriptionOfMany() throws IOException {
14151415

14161416
ValuesSourceReaderOperator.Factory factory = new ValuesSourceReaderOperator.Factory(
14171417
cases.stream().map(c -> c.info).toList(),
1418-
List.of(new ValuesSourceReaderOperator.ShardContext(reader(indexKey), () -> SourceLoader.FROM_STORED_SOURCE)),
1418+
List.of(new ValuesSourceReaderOperator.ShardContext(reader(indexKey), () -> SourceLoader.FROM_STORED_SOURCE, 0.2)),
14191419
0
14201420
);
14211421
assertThat(factory.describe(), equalTo("ValuesSourceReaderOperator[fields = [" + cases.size() + " fields]]"));
@@ -1443,7 +1443,9 @@ public void testManyShards() throws IOException {
14431443
List<ValuesSourceReaderOperator.ShardContext> readerShardContexts = new ArrayList<>();
14441444
for (int s = 0; s < shardCount; s++) {
14451445
contexts.add(new LuceneSourceOperatorTests.MockShardContext(readers[s], s));
1446-
readerShardContexts.add(new ValuesSourceReaderOperator.ShardContext(readers[s], () -> SourceLoader.FROM_STORED_SOURCE));
1446+
readerShardContexts.add(
1447+
new ValuesSourceReaderOperator.ShardContext(readers[s], () -> SourceLoader.FROM_STORED_SOURCE, 0.2)
1448+
);
14471449
}
14481450
var luceneFactory = new LuceneSourceOperator.Factory(
14491451
contexts,

‎x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java‎

Lines changed: 68 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,8 @@ public class ValuesSourceReaderOperatorTests extends OperatorTestCase {
114114
{ false, true, true },
115115
{ false, false, true, true } };
116116

117+
static final double STORED_FIELDS_SEQUENTIAL_PROPORTIONS = 0.2;
118+
117119
private Directory directory = newDirectory();
118120
private MapperService mapperService;
119121
private IndexReader reader;
@@ -147,7 +149,16 @@ static Operator.OperatorFactory factory(IndexReader reader, String name, Element
147149
fail("unexpected shardIdx [" + shardIdx + "]");
148150
}
149151
return loader;
150-
})), List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> SourceLoader.FROM_STORED_SOURCE)), 0);
152+
})),
153+
List.of(
154+
new ValuesSourceReaderOperator.ShardContext(
155+
reader,
156+
() -> SourceLoader.FROM_STORED_SOURCE,
157+
STORED_FIELDS_SEQUENTIAL_PROPORTIONS
158+
)
159+
),
160+
0
161+
);
151162
}
152163

153164
@Override
@@ -443,7 +454,13 @@ public void testManySingleDocPages() {
443454
operators.add(
444455
new ValuesSourceReaderOperator.Factory(
445456
List.of(testCase.info, fieldInfo(mapperService.fieldType("key"), ElementType.INT)),
446-
List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> SourceLoader.FROM_STORED_SOURCE)),
457+
List.of(
458+
new ValuesSourceReaderOperator.ShardContext(
459+
reader,
460+
() -> SourceLoader.FROM_STORED_SOURCE,
461+
STORED_FIELDS_SEQUENTIAL_PROPORTIONS
462+
)
463+
),
447464
0
448465
).get(driverContext)
449466
);
@@ -549,7 +566,13 @@ private void loadSimpleAndAssert(
549566
operators.add(
550567
new ValuesSourceReaderOperator.Factory(
551568
List.of(fieldInfo(mapperService.fieldType("key"), ElementType.INT)),
552-
List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> SourceLoader.FROM_STORED_SOURCE)),
569+
List.of(
570+
new ValuesSourceReaderOperator.ShardContext(
571+
reader,
572+
() -> SourceLoader.FROM_STORED_SOURCE,
573+
STORED_FIELDS_SEQUENTIAL_PROPORTIONS
574+
)
575+
),
553576
0
554577
).get(driverContext)
555578
);
@@ -561,7 +584,13 @@ private void loadSimpleAndAssert(
561584
operators.add(
562585
new ValuesSourceReaderOperator.Factory(
563586
b.stream().map(i -> i.info).toList(),
564-
List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> SourceLoader.FROM_STORED_SOURCE)),
587+
List.of(
588+
new ValuesSourceReaderOperator.ShardContext(
589+
reader,
590+
() -> SourceLoader.FROM_STORED_SOURCE,
591+
STORED_FIELDS_SEQUENTIAL_PROPORTIONS
592+
)
593+
),
565594
0
566595
).get(driverContext)
567596
);
@@ -651,7 +680,13 @@ private void testLoadAllStatus(boolean allInOnePage) {
651680
.map(
652681
i -> new ValuesSourceReaderOperator.Factory(
653682
List.of(i.info),
654-
List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> SourceLoader.FROM_STORED_SOURCE)),
683+
List.of(
684+
new ValuesSourceReaderOperator.ShardContext(
685+
reader,
686+
() -> SourceLoader.FROM_STORED_SOURCE,
687+
STORED_FIELDS_SEQUENTIAL_PROPORTIONS
688+
)
689+
),
655690
0
656691
).get(driverContext)
657692
)
@@ -1417,7 +1452,13 @@ public void testNullsShared() {
14171452
new ValuesSourceReaderOperator.FieldInfo("null1", ElementType.NULL, shardIdx -> BlockLoader.CONSTANT_NULLS),
14181453
new ValuesSourceReaderOperator.FieldInfo("null2", ElementType.NULL, shardIdx -> BlockLoader.CONSTANT_NULLS)
14191454
),
1420-
List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> SourceLoader.FROM_STORED_SOURCE)),
1455+
List.of(
1456+
new ValuesSourceReaderOperator.ShardContext(
1457+
reader,
1458+
() -> SourceLoader.FROM_STORED_SOURCE,
1459+
STORED_FIELDS_SEQUENTIAL_PROPORTIONS
1460+
)
1461+
),
14211462
0
14221463
).get(driverContext)
14231464
),
@@ -1462,7 +1503,13 @@ private void testSequentialStoredFields(boolean sequential, int docCount) throws
14621503
fieldInfo(mapperService.fieldType("key"), ElementType.INT),
14631504
fieldInfo(storedTextField("stored_text"), ElementType.BYTES_REF)
14641505
),
1465-
List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> SourceLoader.FROM_STORED_SOURCE)),
1506+
List.of(
1507+
new ValuesSourceReaderOperator.ShardContext(
1508+
reader,
1509+
() -> SourceLoader.FROM_STORED_SOURCE,
1510+
STORED_FIELDS_SEQUENTIAL_PROPORTIONS
1511+
)
1512+
),
14661513
0
14671514
).get(driverContext);
14681515
List<Page> results = drive(op, source.iterator(), driverContext);
@@ -1490,7 +1537,13 @@ public void testDescriptionOfMany() throws IOException {
14901537

14911538
ValuesSourceReaderOperator.Factory factory = new ValuesSourceReaderOperator.Factory(
14921539
cases.stream().map(c -> c.info).toList(),
1493-
List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> SourceLoader.FROM_STORED_SOURCE)),
1540+
List.of(
1541+
new ValuesSourceReaderOperator.ShardContext(
1542+
reader,
1543+
() -> SourceLoader.FROM_STORED_SOURCE,
1544+
STORED_FIELDS_SEQUENTIAL_PROPORTIONS
1545+
)
1546+
),
14941547
0
14951548
);
14961549
assertThat(factory.describe(), equalTo("ValuesSourceReaderOperator[fields = [" + cases.size() + " fields]]"));
@@ -1517,7 +1570,13 @@ public void testManyShards() throws IOException {
15171570
List<ValuesSourceReaderOperator.ShardContext> readerShardContexts = new ArrayList<>();
15181571
for (int s = 0; s < shardCount; s++) {
15191572
contexts.add(new LuceneSourceOperatorTests.MockShardContext(readers[s], s));
1520-
readerShardContexts.add(new ValuesSourceReaderOperator.ShardContext(readers[s], () -> SourceLoader.FROM_STORED_SOURCE));
1573+
readerShardContexts.add(
1574+
new ValuesSourceReaderOperator.ShardContext(
1575+
readers[s],
1576+
() -> SourceLoader.FROM_STORED_SOURCE,
1577+
STORED_FIELDS_SEQUENTIAL_PROPORTIONS
1578+
)
1579+
);
15211580
}
15221581
var luceneFactory = new LuceneSourceOperator.Factory(
15231582
contexts,

0 commit comments

Comments
 (0)