Skip to content

Commit f5fe89a

Browse files
martijnvgmartijnvg
committed
Deal with the situation that a field is both a normal field and runtime field.
1 parent 53e4b72 commit f5fe89a

File tree

6 files changed

+126
-12
lines changed

6 files changed

+126
-12
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,11 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) {
111111
@Override
112112
public BlockLoader blockLoader(BlockLoaderContext blContext) {
113113
var indexSettings = blContext.indexSettings();
114-
if (isParsedFromSource && indexSettings.getIndexMappingSourceMode() == SourceFieldMapper.Mode.SYNTHETIC) {
114+
if (isParsedFromSource && indexSettings.getIndexMappingSourceMode() == SourceFieldMapper.Mode.SYNTHETIC
115+
// A runtime and normal field can share the same name.
116+
// In that case there is no ignored source entry, and so we need to fail back to LongScriptBlockLoader.
117+
// We could optimize this, but at this stage feels like a rare scenario.
118+
&& blContext.lookup().onlyMappedAsRuntimeField(name())) {
115119
var reader = new NumberType.NumberFallbackSyntheticSourceReader(NumberType.LONG, null, true) {
116120
@Override
117121
public void writeToBlock(List<Number> values, BlockLoader.Builder blockBuilder) {

‎server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -528,6 +528,7 @@ public void setLookupProviders(
528528
// as well as runtime fields loaded from _source that do need a source provider as part of executing the query
529529
this.lookup = new SearchLookup(
530530
this::getFieldType,
531+
fieldName -> mappingLookup.getMapper(fieldName) == null,
531532
(fieldType, searchLookup, fielddataOperation) -> indexFieldDataLookup.apply(
532533
fieldType,
533534
new FieldDataContext(

‎server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ public class SearchLookup implements SourceProvider {
4646
private final Set<String> fieldChain;
4747
private final SourceProvider sourceProvider;
4848
private final Function<String, MappedFieldType> fieldTypeLookup;
49+
private final Function<String, Boolean> onlyMappedAsRuntimeField;
4950
private final TriFunction<
5051
MappedFieldType,
5152
Supplier<SearchLookup>,
@@ -64,7 +65,7 @@ public SearchLookup(
6465
TriFunction<MappedFieldType, Supplier<SearchLookup>, MappedFieldType.FielddataOperation, IndexFieldData<?>> fieldDataLookup,
6566
SourceProvider sourceProvider
6667
) {
67-
this(fieldTypeLookup, fieldDataLookup, sourceProvider, LeafFieldLookupProvider.fromStoredFields());
68+
this(fieldTypeLookup, fieldName -> false, fieldDataLookup, sourceProvider, LeafFieldLookupProvider.fromStoredFields());
6869
}
6970

7071
/**
@@ -76,11 +77,13 @@ public SearchLookup(
7677
*/
7778
public SearchLookup(
7879
Function<String, MappedFieldType> fieldTypeLookup,
80+
Function<String, Boolean> onlyMappedAsRuntimeField,
7981
TriFunction<MappedFieldType, Supplier<SearchLookup>, MappedFieldType.FielddataOperation, IndexFieldData<?>> fieldDataLookup,
8082
SourceProvider sourceProvider,
8183
Function<LeafReaderContext, LeafFieldLookupProvider> fieldLookupProvider
8284
) {
8385
this.fieldTypeLookup = fieldTypeLookup;
86+
this.onlyMappedAsRuntimeField = onlyMappedAsRuntimeField;
8487
this.fieldChain = Collections.emptySet();
8588
this.sourceProvider = sourceProvider;
8689
this.fieldDataLookup = fieldDataLookup;
@@ -98,6 +101,7 @@ private SearchLookup(SearchLookup searchLookup, Set<String> fieldChain) {
98101
this.fieldChain = Collections.unmodifiableSet(fieldChain);
99102
this.sourceProvider = searchLookup.sourceProvider;
100103
this.fieldTypeLookup = searchLookup.fieldTypeLookup;
104+
this.onlyMappedAsRuntimeField = searchLookup.onlyMappedAsRuntimeField;
101105
this.fieldDataLookup = searchLookup.fieldDataLookup;
102106
this.fieldLookupProvider = searchLookup.fieldLookupProvider;
103107
}
@@ -106,6 +110,7 @@ private SearchLookup(SearchLookup searchLookup, SourceProvider sourceProvider) {
106110
this.fieldChain = searchLookup.fieldChain;
107111
this.sourceProvider = sourceProvider;
108112
this.fieldTypeLookup = searchLookup.fieldTypeLookup;
113+
this.onlyMappedAsRuntimeField = searchLookup.onlyMappedAsRuntimeField;
109114
this.fieldDataLookup = searchLookup.fieldDataLookup;
110115
this.fieldLookupProvider = searchLookup.fieldLookupProvider;
111116
}
@@ -144,6 +149,13 @@ public MappedFieldType fieldType(String fieldName) {
144149
return fieldTypeLookup.apply(fieldName);
145150
}
146151

152+
/**
153+
* @return whether a field is only mapped as runtime field. A runtime and normal field can share the same name.
154+
*/
155+
public boolean onlyMappedAsRuntimeField(String fieldName) {
156+
return onlyMappedAsRuntimeField.apply(fieldName);
157+
}
158+
147159
public IndexFieldData<?> getForField(MappedFieldType fieldType, MappedFieldType.FielddataOperation options) {
148160
return fieldDataLookup.apply(fieldType, () -> forkAndTrackFieldReferences(fieldType.name()), options);
149161
}

‎server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,8 @@ public void testBlockLoaderSourceOnlyRuntimeField() throws IOException {
326326
LongScriptFieldType fieldType = simpleSourceOnlyMappedFieldType();
327327

328328
// Assert implementations:
329-
BlockLoader loader = fieldType.blockLoader(blContext(Settings.EMPTY));
329+
BlockLoader loader = fieldType.blockLoader(blContext(Settings.EMPTY, true));
330+
assertThat(loader, instanceOf(LongScriptBlockDocValuesReader.LongScriptBlockLoader.class));
330331
// ignored source doesn't support column at a time loading:
331332
var columnAtATimeLoader = loader.columnAtATimeReader(reader.leaves().getFirst());
332333
assertThat(columnAtATimeLoader, instanceOf(LongScriptBlockDocValuesReader.class));
@@ -356,7 +357,8 @@ public void testBlockLoaderSourceOnlyRuntimeFieldWithSyntheticSource() throws IO
356357
LongScriptFieldType fieldType = simpleSourceOnlyMappedFieldType();
357358

358359
// Assert implementations:
359-
BlockLoader loader = fieldType.blockLoader(blContext(settings));
360+
BlockLoader loader = fieldType.blockLoader(blContext(settings, true));
361+
assertThat(loader, instanceOf(FallbackSyntheticSourceBlockLoader.class));
360362
// ignored source doesn't support column at a time loading:
361363
var columnAtATimeLoader = loader.columnAtATimeReader(reader.leaves().getFirst());
362364
assertThat(columnAtATimeLoader, nullValue());
@@ -367,7 +369,7 @@ public void testBlockLoaderSourceOnlyRuntimeFieldWithSyntheticSource() throws IO
367369
);
368370

369371
// Assert values:
370-
assertThat(blockLoaderReadValuesFromRowStrideReader(settings, reader, fieldType), equalTo(List.of(1L, 2L)));
372+
assertThat(blockLoaderReadValuesFromRowStrideReader(settings, reader, fieldType, true), equalTo(List.of(1L, 2L)));
371373
}
372374
}
373375
}

‎server/src/test/java/org/elasticsearch/index/mapper/RuntimeFieldSourceProviderOptimizationTests.java‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,26 +9,35 @@
99

1010
package org.elasticsearch.index.mapper;
1111

12+
import org.apache.lucene.document.Document;
1213
import org.apache.lucene.index.DirectoryReader;
1314
import org.apache.lucene.index.IndexWriter;
1415
import org.apache.lucene.index.IndexWriterConfig;
1516
import org.apache.lucene.index.LeafReaderContext;
1617
import org.apache.lucene.search.IndexSearcher;
1718
import org.apache.lucene.store.Directory;
19+
import org.elasticsearch.cluster.metadata.IndexMetadata;
1820
import org.elasticsearch.common.bytes.BytesArray;
1921
import org.elasticsearch.common.settings.Settings;
22+
import org.elasticsearch.index.IndexSettings;
23+
import org.elasticsearch.index.IndexVersion;
2024
import org.elasticsearch.index.fielddata.LongScriptDocValues;
2125
import org.elasticsearch.index.fielddata.LongScriptFieldData;
2226
import org.elasticsearch.script.LongFieldScript;
27+
import org.elasticsearch.search.lookup.SearchLookup;
2328
import org.elasticsearch.test.ESSingleNodeTestCase;
29+
import org.elasticsearch.test.ESTestCase;
2430
import org.elasticsearch.xcontent.XContentType;
2531

2632
import java.io.IOException;
2733
import java.util.Locale;
2834
import java.util.Map;
35+
import java.util.Set;
2936

3037
import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
3138
import static org.hamcrest.Matchers.equalTo;
39+
import static org.hamcrest.Matchers.instanceOf;
40+
import static org.hamcrest.Matchers.notNullValue;
3241

3342
/**
3443
* Tests that source provider optimization that filters _source based on the same of source only runtime fields kick in.
@@ -142,4 +151,79 @@ public void testWithoutSourceProviderOptimization() throws IOException {
142151
}
143152
}
144153

154+
public void testNormalAndRuntimeFieldWithSameName() throws IOException {
155+
var mapping = jsonBuilder().startObject().startObject("runtime");
156+
mapping.startObject("field").field("type", "long").endObject();
157+
mapping.startObject("field2").field("type", "long").endObject();
158+
mapping.endObject().startObject("properties");
159+
mapping.startObject("field").field("type", "long").endObject();
160+
mapping.endObject().endObject();
161+
162+
var settings = Settings.builder().put("index.mapping.source.mode", "synthetic").build();
163+
var indexService = createIndex("test-index", settings, mapping);
164+
var fieldType1 = indexService.mapperService().fieldType("field");
165+
assertThat(fieldType1, notNullValue());
166+
var fieldType2 = indexService.mapperService().fieldType("field2");
167+
assertThat(fieldType2, notNullValue());
168+
169+
// Assert implementations:
170+
try (Directory directory = newDirectory(); IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) {
171+
iw.addDocument(new Document());
172+
try (var indexReader = DirectoryReader.open(iw)) {
173+
var searcher = new IndexSearcher(indexReader);
174+
var context = indexService.newSearchExecutionContext(0, 0, searcher, () -> 1L, null, Map.of());
175+
176+
// field name 'field' is both mapped as runtime and normal field and so LongScriptBlockLoader is expected:
177+
BlockLoader loader = fieldType1.blockLoader(blContext(settings, context.lookup()));
178+
assertThat(loader, instanceOf(LongScriptBlockDocValuesReader.LongScriptBlockLoader.class));
179+
180+
// field name 'field2' is just mapped as runtime field and so FallbackSyntheticSourceBlockLoader is expected:
181+
BlockLoader loader2 = fieldType2.blockLoader(blContext(settings, context.lookup()));
182+
assertThat(loader2, instanceOf(FallbackSyntheticSourceBlockLoader.class));
183+
}
184+
}
185+
186+
}
187+
188+
static MappedFieldType.BlockLoaderContext blContext(Settings settings, SearchLookup lookup) {
189+
String indexName = "test_index";
190+
var imd = IndexMetadata.builder(indexName).settings(ESTestCase.indexSettings(IndexVersion.current(), 1, 1).put(settings)).build();
191+
return new MappedFieldType.BlockLoaderContext() {
192+
@Override
193+
public String indexName() {
194+
return indexName;
195+
}
196+
197+
@Override
198+
public IndexSettings indexSettings() {
199+
return new IndexSettings(imd, settings);
200+
}
201+
202+
@Override
203+
public MappedFieldType.FieldExtractPreference fieldExtractPreference() {
204+
return MappedFieldType.FieldExtractPreference.NONE;
205+
}
206+
207+
@Override
208+
public SearchLookup lookup() {
209+
return lookup;
210+
}
211+
212+
@Override
213+
public Set<String> sourcePaths(String name) {
214+
throw new UnsupportedOperationException();
215+
}
216+
217+
@Override
218+
public String parentField(String field) {
219+
throw new UnsupportedOperationException();
220+
}
221+
222+
@Override
223+
public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() {
224+
return FieldNamesFieldMapper.FieldNamesFieldType.get(true);
225+
}
226+
};
227+
}
228+
145229
}

‎test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.elasticsearch.script.Script;
4040
import org.elasticsearch.script.ScriptContext;
4141
import org.elasticsearch.script.ScriptFactory;
42+
import org.elasticsearch.search.lookup.LeafFieldLookupProvider;
4243
import org.elasticsearch.search.lookup.SearchLookup;
4344
import org.elasticsearch.search.lookup.SourceProvider;
4445
import org.elasticsearch.test.ESTestCase;
@@ -297,13 +298,15 @@ protected boolean supportsRangeQueries() {
297298
protected static SearchExecutionContext mockContext(boolean allowExpensiveQueries, MappedFieldType mappedFieldType) {
298299
return mockContext(
299300
allowExpensiveQueries,
301+
false,
300302
mappedFieldType,
301303
SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP)
302304
);
303305
}
304306

305307
protected static SearchExecutionContext mockContext(
306308
boolean allowExpensiveQueries,
309+
boolean fieldOnlyMappedAsRuntimeField,
307310
MappedFieldType mappedFieldType,
308311
SourceProvider sourceProvider
309312
) {
@@ -314,9 +317,11 @@ protected static SearchExecutionContext mockContext(
314317
when(context.allowExpensiveQueries()).thenReturn(allowExpensiveQueries);
315318
SearchLookup lookup = new SearchLookup(
316319
context::getFieldType,
320+
(fieldName) -> fieldOnlyMappedAsRuntimeField,
317321
(mft, lookupSupplier, fdo) -> mft.fielddataBuilder(new FieldDataContext("test", null, lookupSupplier, context::sourcePath, fdo))
318322
.build(null, null),
319-
sourceProvider
323+
sourceProvider,
324+
LeafFieldLookupProvider.fromStoredFields()
320325
);
321326
when(context.lookup()).thenReturn(lookup);
322327
when(context.getForField(any(), any())).then(args -> {
@@ -479,7 +484,7 @@ protected final List<Object> blockLoaderReadValuesFromColumnAtATimeReader(
479484
MappedFieldType fieldType,
480485
int offset
481486
) throws IOException {
482-
BlockLoader loader = fieldType.blockLoader(blContext(settings));
487+
BlockLoader loader = fieldType.blockLoader(blContext(settings, true));
483488
List<Object> all = new ArrayList<>();
484489
for (LeafReaderContext ctx : reader.leaves()) {
485490
TestBlock block = (TestBlock) loader.columnAtATimeReader(ctx).read(TestBlock.factory(), TestBlock.docs(ctx), offset, false);
@@ -492,15 +497,16 @@ protected final List<Object> blockLoaderReadValuesFromColumnAtATimeReader(
492497

493498
protected final List<Object> blockLoaderReadValuesFromRowStrideReader(DirectoryReader reader, MappedFieldType fieldType)
494499
throws IOException {
495-
return blockLoaderReadValuesFromRowStrideReader(Settings.EMPTY, reader, fieldType);
500+
return blockLoaderReadValuesFromRowStrideReader(Settings.EMPTY, reader, fieldType, false);
496501
}
497502

498503
protected final List<Object> blockLoaderReadValuesFromRowStrideReader(
499504
Settings settings,
500505
DirectoryReader reader,
501-
MappedFieldType fieldType
506+
MappedFieldType fieldType,
507+
boolean fieldOnlyMappedAsRuntimeField
502508
) throws IOException {
503-
BlockLoader loader = fieldType.blockLoader(blContext(settings));
509+
BlockLoader loader = fieldType.blockLoader(blContext(settings, fieldOnlyMappedAsRuntimeField));
504510
List<Object> all = new ArrayList<>();
505511
for (LeafReaderContext ctx : reader.leaves()) {
506512
BlockLoader.RowStrideReader blockReader = loader.rowStrideReader(ctx);
@@ -523,7 +529,7 @@ protected final List<Object> blockLoaderReadValuesFromRowStrideReader(
523529
return all;
524530
}
525531

526-
protected MappedFieldType.BlockLoaderContext blContext(Settings settings) {
532+
protected MappedFieldType.BlockLoaderContext blContext(Settings settings, boolean fieldOnlyMappedAsRuntimeField) {
527533
String indexName = "test_index";
528534
var imd = IndexMetadata.builder(indexName).settings(ESTestCase.indexSettings(IndexVersion.current(), 1, 1).put(settings)).build();
529535
return new MappedFieldType.BlockLoaderContext() {
@@ -544,7 +550,12 @@ public MappedFieldType.FieldExtractPreference fieldExtractPreference() {
544550

545551
@Override
546552
public SearchLookup lookup() {
547-
return mockContext().lookup();
553+
return mockContext(
554+
true,
555+
fieldOnlyMappedAsRuntimeField,
556+
null,
557+
SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP)
558+
).lookup();
548559
}
549560

550561
@Override

0 commit comments

Comments
 (0)