Skip to content

Commit 21d47f9

Browse files
martijnvgmartijnvg
committed
Improve block loader for source only runtime fields.
By using the FallbackSyntheticSourceBlockLoader instead of generic LongScriptBlockLoader.
1 parent 3eefef7 commit 21d47f9

File tree

4 files changed

+178
-8
lines changed

4 files changed

+178
-8
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ public abstract class AbstractScriptFieldType<LeafFactory> extends MappedFieldTy
4949
protected final Script script;
5050
private final Function<SearchLookup, LeafFactory> factory;
5151
private final boolean isResultDeterministic;
52-
private final boolean isParsedFromSource;
52+
protected final boolean isParsedFromSource;
5353

5454
protected AbstractScriptFieldType(
5555
String name,

‎server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030

3131
import java.time.ZoneId;
3232
import java.util.Collection;
33+
import java.util.List;
3334
import java.util.Map;
3435
import java.util.Set;
3536
import java.util.function.Function;
@@ -109,7 +110,31 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) {
109110

110111
@Override
111112
public BlockLoader blockLoader(BlockLoaderContext blContext) {
112-
return new LongScriptBlockDocValuesReader.LongScriptBlockLoader(leafFactory(blContext.lookup()));
113+
var indexSettings = blContext.indexSettings();
114+
if (isParsedFromSource && indexSettings.getIndexMappingSourceMode() == SourceFieldMapper.Mode.SYNTHETIC) {
115+
var reader = new NumberType.NumberFallbackSyntheticSourceReader(NumberType.LONG, null, true) {
116+
@Override
117+
public void writeToBlock(List<Number> values, BlockLoader.Builder blockBuilder) {
118+
var builder = (BlockLoader.LongBuilder) blockBuilder;
119+
for (var value : values) {
120+
builder.appendLong(value.longValue());
121+
}
122+
}
123+
};
124+
125+
return new FallbackSyntheticSourceBlockLoader(
126+
reader,
127+
name(),
128+
IgnoredSourceFieldMapper.ignoredSourceFormat(indexSettings.getIndexVersionCreated())
129+
) {
130+
@Override
131+
public Builder builder(BlockFactory factory, int expectedCount) {
132+
return factory.longs(expectedCount);
133+
}
134+
};
135+
} else {
136+
return new LongScriptBlockDocValuesReader.LongScriptBlockLoader(leafFactory(blContext.lookup()));
137+
}
113138
}
114139

115140
@Override

‎server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,11 @@
2929
import org.apache.lucene.store.Directory;
3030
import org.apache.lucene.tests.index.RandomIndexWriter;
3131
import org.apache.lucene.util.BytesRef;
32+
import org.elasticsearch.common.bytes.BytesArray;
3233
import org.elasticsearch.common.geo.ShapeRelation;
3334
import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery;
35+
import org.elasticsearch.common.settings.Settings;
36+
import org.elasticsearch.common.xcontent.XContentHelper;
3437
import org.elasticsearch.index.IndexVersion;
3538
import org.elasticsearch.index.fielddata.LongScriptFieldData;
3639
import org.elasticsearch.index.fielddata.ScriptDocValues;
@@ -42,6 +45,10 @@
4245
import org.elasticsearch.script.ScriptFactory;
4346
import org.elasticsearch.script.ScriptType;
4447
import org.elasticsearch.search.MultiValueMode;
48+
import org.elasticsearch.search.lookup.SearchLookup;
49+
import org.elasticsearch.xcontent.XContentFactory;
50+
import org.elasticsearch.xcontent.XContentParserConfiguration;
51+
import org.elasticsearch.xcontent.XContentType;
4552

4653
import java.io.IOException;
4754
import java.util.ArrayList;
@@ -52,6 +59,8 @@
5259
import static org.hamcrest.Matchers.containsInAnyOrder;
5360
import static org.hamcrest.Matchers.equalTo;
5461
import static org.hamcrest.Matchers.greaterThan;
62+
import static org.hamcrest.Matchers.instanceOf;
63+
import static org.hamcrest.Matchers.nullValue;
5564

5665
public class LongScriptFieldTypeTests extends AbstractNonTextScriptFieldTypeTestCase {
5766

@@ -302,6 +311,81 @@ public void testBlockLoader() throws IOException {
302311
}
303312
}
304313

314+
public void testBlockLoaderSourceOnlyRuntimeField() throws IOException {
315+
try (
316+
Directory directory = newDirectory();
317+
RandomIndexWriter iw = new RandomIndexWriter(random(), directory, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE))
318+
) {
319+
iw.addDocuments(
320+
List.of(
321+
List.of(new StoredField("_source", new BytesRef("{\"test\": [1]}"))),
322+
List.of(new StoredField("_source", new BytesRef("{\"test\": [2]}")))
323+
)
324+
);
325+
try (DirectoryReader reader = iw.getReader()) {
326+
LongScriptFieldType fieldType = simpleSourceOnlyMappedFieldType();
327+
328+
// Assert implementations:
329+
BlockLoader loader = fieldType.blockLoader(blContext(Settings.EMPTY));
330+
// ignored source doesn't support column at a time loading:
331+
var columnAtATimeLoader = loader.columnAtATimeReader(reader.leaves().getFirst());
332+
assertThat(columnAtATimeLoader, instanceOf(LongScriptBlockDocValuesReader.class));
333+
var rowStrideReader = loader.rowStrideReader(reader.leaves().getFirst());
334+
assertThat(rowStrideReader, instanceOf(LongScriptBlockDocValuesReader.class));
335+
336+
// Assert values:
337+
assertThat(blockLoaderReadValuesFromColumnAtATimeReader(reader, fieldType, 0), equalTo(List.of(1L, 2L)));
338+
assertThat(blockLoaderReadValuesFromColumnAtATimeReader(reader, fieldType, 1), equalTo(List.of(2L)));
339+
assertThat(blockLoaderReadValuesFromRowStrideReader(reader, fieldType), equalTo(List.of(1L, 2L)));
340+
}
341+
}
342+
}
343+
344+
public void testBlockLoaderSourceOnlyRuntimeFieldWithSyntheticSource() throws IOException {
345+
var settings = Settings.builder().put("index.mapping.source.mode", "synthetic").build();
346+
try (
347+
Directory directory = newDirectory();
348+
RandomIndexWriter iw = new RandomIndexWriter(random(), directory, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE))
349+
) {
350+
351+
var document1 = createDocumentWithIgnoredSource("[1]");
352+
var document2 = createDocumentWithIgnoredSource("[2]");
353+
354+
iw.addDocuments(List.of(document1, document2));
355+
try (DirectoryReader reader = iw.getReader()) {
356+
LongScriptFieldType fieldType = simpleSourceOnlyMappedFieldType();
357+
358+
// Assert implementations:
359+
BlockLoader loader = fieldType.blockLoader(blContext(settings));
360+
// ignored source doesn't support column at a time loading:
361+
var columnAtATimeLoader = loader.columnAtATimeReader(reader.leaves().getFirst());
362+
assertThat(columnAtATimeLoader, nullValue());
363+
var rowStrideReader = loader.rowStrideReader(reader.leaves().getFirst());
364+
assertThat(
365+
rowStrideReader.getClass().getName(),
366+
equalTo("org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader$IgnoredSourceRowStrideReader")
367+
);
368+
369+
// Assert values:
370+
assertThat(blockLoaderReadValuesFromRowStrideReader(settings, reader, fieldType), equalTo(List.of(1L, 2L)));
371+
}
372+
}
373+
}
374+
375+
private static LuceneDocument createDocumentWithIgnoredSource(String bytes) throws IOException {
376+
var doc = new LuceneDocument();
377+
var parser = XContentHelper.createParser(
378+
XContentParserConfiguration.EMPTY,
379+
new BytesArray(bytes),
380+
XContentFactory.xContent(XContentType.JSON).type()
381+
);
382+
parser.nextToken();
383+
var nameValue = new IgnoredSourceFieldMapper.NameValue("test", 0, XContentDataHelper.encodeToken(parser), doc);
384+
var ignoredSourceFormat = IgnoredSourceFieldMapper.ignoredSourceFormat(IndexVersion.current());
385+
ignoredSourceFormat.writeIgnoredFields(List.of(nameValue));
386+
return doc;
387+
}
388+
305389
@Override
306390
protected Query randomTermsQuery(MappedFieldType ft, SearchExecutionContext ctx) {
307391
return ft.termsQuery(List.of(randomLong()), ctx);
@@ -312,6 +396,10 @@ protected LongScriptFieldType simpleMappedFieldType() {
312396
return build("read_foo", Map.of(), OnScriptError.FAIL);
313397
}
314398

399+
private LongScriptFieldType simpleSourceOnlyMappedFieldType() {
400+
return build("read_test", Map.of(), OnScriptError.FAIL);
401+
}
402+
315403
@Override
316404
protected LongScriptFieldType loopFieldType() {
317405
return build("loop", Map.of(), OnScriptError.FAIL);
@@ -329,6 +417,32 @@ protected LongScriptFieldType build(String code, Map<String, Object> params, OnS
329417

330418
private static LongFieldScript.Factory factory(Script script) {
331419
switch (script.getIdOrCode()) {
420+
case "read_test":
421+
return new LongFieldScript.Factory() {
422+
@Override
423+
public LongFieldScript.LeafFactory newFactory(
424+
String fieldName,
425+
Map<String, Object> params,
426+
SearchLookup lookup,
427+
OnScriptError onScriptError
428+
) {
429+
return (ctx) -> new LongFieldScript(fieldName, params, lookup, onScriptError, ctx) {
430+
@Override
431+
@SuppressWarnings("unchecked")
432+
public void execute() {
433+
Map<String, Object> source = (Map<String, Object>) this.getParams().get("_source");
434+
for (Object foo : (List<?>) source.get("test")) {
435+
emit(((Number) foo).longValue());
436+
}
437+
};
438+
};
439+
}
440+
441+
@Override
442+
public boolean isParsedFromSource() {
443+
return true;
444+
}
445+
};
332446
case "read_foo":
333447
return (fieldName, params, lookup, onScriptError) -> (ctx) -> new LongFieldScript(
334448
fieldName,

‎test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java‎

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,17 @@
2222
import org.apache.lucene.tests.index.RandomIndexWriter;
2323
import org.apache.lucene.util.BytesRef;
2424
import org.elasticsearch.ElasticsearchException;
25+
import org.elasticsearch.cluster.metadata.IndexMetadata;
2526
import org.elasticsearch.common.Strings;
2627
import org.elasticsearch.common.bytes.BytesReference;
2728
import org.elasticsearch.common.geo.ShapeRelation;
29+
import org.elasticsearch.common.settings.Settings;
2830
import org.elasticsearch.common.xcontent.XContentHelper;
2931
import org.elasticsearch.index.IndexSettings;
32+
import org.elasticsearch.index.IndexVersion;
3033
import org.elasticsearch.index.fielddata.FieldDataContext;
3134
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
35+
import org.elasticsearch.index.fieldvisitor.StoredFieldLoader;
3236
import org.elasticsearch.index.query.ExistsQueryBuilder;
3337
import org.elasticsearch.index.query.SearchExecutionContext;
3438
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
@@ -37,6 +41,7 @@
3741
import org.elasticsearch.script.ScriptFactory;
3842
import org.elasticsearch.search.lookup.SearchLookup;
3943
import org.elasticsearch.search.lookup.SourceProvider;
44+
import org.elasticsearch.test.ESTestCase;
4045
import org.elasticsearch.xcontent.ToXContent;
4146
import org.elasticsearch.xcontent.XContentBuilder;
4247
import org.elasticsearch.xcontent.XContentFactory;
@@ -465,7 +470,16 @@ public final void testIsParsedFromSource() throws IOException {
465470

466471
protected final List<Object> blockLoaderReadValuesFromColumnAtATimeReader(DirectoryReader reader, MappedFieldType fieldType, int offset)
467472
throws IOException {
468-
BlockLoader loader = fieldType.blockLoader(blContext());
473+
return blockLoaderReadValuesFromColumnAtATimeReader(Settings.EMPTY, reader, fieldType, offset);
474+
}
475+
476+
protected final List<Object> blockLoaderReadValuesFromColumnAtATimeReader(
477+
Settings settings,
478+
DirectoryReader reader,
479+
MappedFieldType fieldType,
480+
int offset
481+
) throws IOException {
482+
BlockLoader loader = fieldType.blockLoader(blContext(settings));
469483
List<Object> all = new ArrayList<>();
470484
for (LeafReaderContext ctx : reader.leaves()) {
471485
TestBlock block = (TestBlock) loader.columnAtATimeReader(ctx).read(TestBlock.factory(), TestBlock.docs(ctx), offset, false);
@@ -478,13 +492,28 @@ protected final List<Object> blockLoaderReadValuesFromColumnAtATimeReader(Direct
478492

479493
protected final List<Object> blockLoaderReadValuesFromRowStrideReader(DirectoryReader reader, MappedFieldType fieldType)
480494
throws IOException {
481-
BlockLoader loader = fieldType.blockLoader(blContext());
495+
return blockLoaderReadValuesFromRowStrideReader(Settings.EMPTY, reader, fieldType);
496+
}
497+
498+
protected final List<Object> blockLoaderReadValuesFromRowStrideReader(
499+
Settings settings,
500+
DirectoryReader reader,
501+
MappedFieldType fieldType
502+
) throws IOException {
503+
BlockLoader loader = fieldType.blockLoader(blContext(settings));
482504
List<Object> all = new ArrayList<>();
483505
for (LeafReaderContext ctx : reader.leaves()) {
484506
BlockLoader.RowStrideReader blockReader = loader.rowStrideReader(ctx);
485507
BlockLoader.Builder builder = loader.builder(TestBlock.factory(), ctx.reader().numDocs());
508+
509+
assert loader.rowStrideStoredFieldSpec().requiresSource() == false;
510+
BlockLoaderStoredFieldsFromLeafLoader storedFields = new BlockLoaderStoredFieldsFromLeafLoader(
511+
StoredFieldLoader.fromSpec(loader.rowStrideStoredFieldSpec()).getLoader(ctx, null),
512+
null
513+
);
486514
for (int i = 0; i < ctx.reader().numDocs(); i++) {
487-
blockReader.read(i, null, builder);
515+
storedFields.advanceTo(i);
516+
blockReader.read(i, storedFields, builder);
488517
}
489518
TestBlock block = (TestBlock) builder.build();
490519
for (int i = 0; i < block.size(); i++) {
@@ -494,16 +523,18 @@ protected final List<Object> blockLoaderReadValuesFromRowStrideReader(DirectoryR
494523
return all;
495524
}
496525

497-
private MappedFieldType.BlockLoaderContext blContext() {
526+
protected MappedFieldType.BlockLoaderContext blContext(Settings settings) {
527+
String indexName = "test_index";
528+
var imd = IndexMetadata.builder(indexName).settings(ESTestCase.indexSettings(IndexVersion.current(), 1, 1).put(settings)).build();
498529
return new MappedFieldType.BlockLoaderContext() {
499530
@Override
500531
public String indexName() {
501-
throw new UnsupportedOperationException();
532+
return indexName;
502533
}
503534

504535
@Override
505536
public IndexSettings indexSettings() {
506-
throw new UnsupportedOperationException();
537+
return new IndexSettings(imd, settings);
507538
}
508539

509540
@Override

0 commit comments

Comments
 (0)