Skip to content

Commit c46702d

Browse files
Kubik42Kubik42
authored
Improve block loader for source only runtime date fields (#135373)
* Improve block loader for source only runtime date fields * Update docs/changelog/135373.yaml
1 parent 057b5c8 commit c46702d

File tree

3 files changed

+212
-0
lines changed

3 files changed

+212
-0
lines changed

‎docs/changelog/135373.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 135373
2+
summary: Improve block loader for source only runtime date fields
3+
area: Mapping
4+
type: enhancement
5+
issues: []

‎server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,9 @@
3333
import org.elasticsearch.search.runtime.LongScriptFieldRangeQuery;
3434
import org.elasticsearch.search.runtime.LongScriptFieldTermQuery;
3535
import org.elasticsearch.search.runtime.LongScriptFieldTermsQuery;
36+
import org.elasticsearch.xcontent.XContentParser;
3637

38+
import java.io.IOException;
3739
import java.time.ZoneId;
3840
import java.time.ZoneOffset;
3941
import java.util.ArrayList;
@@ -183,9 +185,59 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) {
183185

184186
@Override
185187
public BlockLoader blockLoader(BlockLoaderContext blContext) {
188+
FallbackSyntheticSourceBlockLoader fallbackSyntheticSourceBlockLoader = fallbackSyntheticSourceBlockLoader(
189+
blContext,
190+
BlockLoader.BlockFactory::longs,
191+
this::fallbackSyntheticSourceBlockLoaderReader
192+
);
193+
194+
if (fallbackSyntheticSourceBlockLoader != null) {
195+
return fallbackSyntheticSourceBlockLoader;
196+
}
186197
return new DateScriptBlockDocValuesReader.DateScriptBlockLoader(leafFactory(blContext.lookup()));
187198
}
188199

200+
private FallbackSyntheticSourceBlockLoader.Reader<?> fallbackSyntheticSourceBlockLoaderReader() {
201+
return new FallbackSyntheticSourceBlockLoader.SingleValueReader<Long>(null) {
202+
@Override
203+
public void convertValue(Object value, List<Long> accumulator) {
204+
try {
205+
if (value instanceof Number) {
206+
accumulator.add(((Number) value).longValue());
207+
} else {
208+
// when the value is given a string formatted date; ex. 2020-07-22T16:09:41.355Z
209+
accumulator.add(dateTimeFormatter.parseMillis(value.toString()));
210+
}
211+
} catch (Exception e) {
212+
// ensure a malformed value doesn't crash
213+
}
214+
}
215+
216+
@Override
217+
public void writeToBlock(List<Long> values, BlockLoader.Builder blockBuilder) {
218+
var longBuilder = (BlockLoader.LongBuilder) blockBuilder;
219+
for (Long value : values) {
220+
longBuilder.appendLong(value);
221+
}
222+
}
223+
224+
@Override
225+
protected void parseNonNullValue(XContentParser parser, List<Long> accumulator) throws IOException {
226+
try {
227+
String dateAsStr = parser.textOrNull();
228+
229+
if (dateAsStr == null) {
230+
accumulator.add(dateTimeFormatter.parseMillis(null));
231+
} else {
232+
accumulator.add(dateTimeFormatter.parseMillis(dateAsStr));
233+
}
234+
} catch (Exception e) {
235+
// ensure a malformed value doesn't crash
236+
}
237+
}
238+
};
239+
}
240+
189241
@Override
190242
public DateScriptFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) {
191243
return new DateScriptFieldData.Builder(

‎server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.elasticsearch.common.Strings;
3636
import org.elasticsearch.common.geo.ShapeRelation;
3737
import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery;
38+
import org.elasticsearch.common.settings.Settings;
3839
import org.elasticsearch.common.time.DateFormatter;
3940
import org.elasticsearch.index.IndexVersion;
4041
import org.elasticsearch.index.fielddata.DateScriptFieldData;
@@ -47,6 +48,7 @@
4748
import org.elasticsearch.script.ScriptFactory;
4849
import org.elasticsearch.script.ScriptType;
4950
import org.elasticsearch.search.MultiValueMode;
51+
import org.elasticsearch.search.lookup.SearchLookup;
5052
import org.elasticsearch.xcontent.XContentBuilder;
5153

5254
import java.io.IOException;
@@ -55,6 +57,7 @@
5557
import java.time.ZonedDateTime;
5658
import java.time.temporal.ChronoUnit;
5759
import java.util.ArrayList;
60+
import java.util.Arrays;
5861
import java.util.List;
5962
import java.util.Map;
6063

@@ -65,9 +68,12 @@
6568
import static org.hamcrest.Matchers.containsString;
6669
import static org.hamcrest.Matchers.equalTo;
6770
import static org.hamcrest.Matchers.instanceOf;
71+
import static org.hamcrest.Matchers.nullValue;
6872

6973
public class DateScriptFieldTypeTests extends AbstractNonTextScriptFieldTypeTestCase {
7074

75+
private static final Long MALFORMED_DATE = null;
76+
7177
@Override
7278
protected ScriptFactory parseFromSource() {
7379
return DateFieldScript.PARSE_FROM_SOURCE;
@@ -502,6 +508,155 @@ public void testBlockLoader() throws IOException {
502508
}
503509
}
504510

511+
public void testBlockLoaderSourceOnlyRuntimeField() throws IOException {
512+
try (
513+
Directory directory = newDirectory();
514+
RandomIndexWriter iw = new RandomIndexWriter(random(), directory, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE))
515+
) {
516+
// given
517+
iw.addDocuments(
518+
List.of(
519+
List.of(new StoredField("_source", new BytesRef("{\"test\": [1595432181354]}"))),
520+
List.of(new StoredField("_source", new BytesRef("{\"test\": [\"2020-07-22T16:09:41.355Z\"]}"))),
521+
List.of(new StoredField("_source", new BytesRef("{\"test\": [null]}"))),
522+
List.of(new StoredField("_source", new BytesRef("{\"test\": []}"))),
523+
List.of(new StoredField("_source", new BytesRef("{\"test\": [\"malformed\"]}")))
524+
)
525+
);
526+
DateScriptFieldType fieldType = simpleSourceOnlyMappedFieldType();
527+
List<Long> expected = Arrays.asList(
528+
1595432181354L,
529+
Instant.parse("2020-07-22T16:09:41.355Z").toEpochMilli(),
530+
null,
531+
null,
532+
MALFORMED_DATE
533+
);
534+
535+
try (DirectoryReader reader = iw.getReader()) {
536+
// when
537+
BlockLoader loader = fieldType.blockLoader(blContext(Settings.EMPTY, true));
538+
539+
// then
540+
541+
// assert loader is of expected instance type
542+
assertThat(loader, instanceOf(DateScriptBlockDocValuesReader.DateScriptBlockLoader.class));
543+
544+
// ignored source doesn't support column at a time loading:
545+
var columnAtATimeLoader = loader.columnAtATimeReader(reader.leaves().getFirst());
546+
assertThat(columnAtATimeLoader, instanceOf(DateScriptBlockDocValuesReader.class));
547+
548+
var rowStrideReader = loader.rowStrideReader(reader.leaves().getFirst());
549+
assertThat(rowStrideReader, instanceOf(DateScriptBlockDocValuesReader.class));
550+
551+
// assert values
552+
assertThat(blockLoaderReadValuesFromColumnAtATimeReader(reader, fieldType, 0), equalTo(expected));
553+
assertThat(blockLoaderReadValuesFromRowStrideReader(reader, fieldType), equalTo(expected));
554+
}
555+
}
556+
}
557+
558+
public void testBlockLoaderSourceOnlyRuntimeFieldWithSyntheticSource() throws IOException {
559+
try (
560+
Directory directory = newDirectory();
561+
RandomIndexWriter iw = new RandomIndexWriter(random(), directory, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE))
562+
) {
563+
// given
564+
iw.addDocuments(
565+
List.of(
566+
createDocumentWithIgnoredSource("[1595432181354]"),
567+
createDocumentWithIgnoredSource("[\"2020-07-22T16:09:41.355Z\"]"),
568+
createDocumentWithIgnoredSource("[\"\"]"),
569+
createDocumentWithIgnoredSource("[\"malformed\"]")
570+
)
571+
);
572+
573+
Settings settings = Settings.builder().put("index.mapping.source.mode", "synthetic").build();
574+
DateScriptFieldType fieldType = simpleSourceOnlyMappedFieldType();
575+
List<Long> expected = Arrays.asList(
576+
1595432181354L,
577+
Instant.parse("2020-07-22T16:09:41.355Z").toEpochMilli(),
578+
null,
579+
MALFORMED_DATE
580+
);
581+
582+
try (DirectoryReader reader = iw.getReader()) {
583+
// when
584+
BlockLoader loader = fieldType.blockLoader(blContext(settings, true));
585+
586+
// then
587+
588+
// assert loader is of expected instance type
589+
assertThat(loader, instanceOf(FallbackSyntheticSourceBlockLoader.class));
590+
591+
// ignored source doesn't support column at a time loading:
592+
var columnAtATimeLoader = loader.columnAtATimeReader(reader.leaves().getFirst());
593+
assertThat(columnAtATimeLoader, nullValue());
594+
595+
var rowStrideReader = loader.rowStrideReader(reader.leaves().getFirst());
596+
assertThat(
597+
rowStrideReader.getClass().getName(),
598+
equalTo("org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader$IgnoredSourceRowStrideReader")
599+
);
600+
601+
// assert values
602+
assertThat(blockLoaderReadValuesFromRowStrideReader(settings, reader, fieldType, true), equalTo(expected));
603+
}
604+
}
605+
}
606+
607+
/**
608+
* Returns a source only mapped field type. This is useful, since the available build() function doesn't override isParsedFromSource()
609+
*/
610+
private DateScriptFieldType simpleSourceOnlyMappedFieldType() {
611+
Script script = new Script(ScriptType.INLINE, "test", "", emptyMap());
612+
DateFieldScript.Factory factory = new DateFieldScript.Factory() {
613+
@Override
614+
public DateFieldScript.LeafFactory newFactory(
615+
String fieldName,
616+
Map<String, Object> params,
617+
SearchLookup searchLookup,
618+
DateFormatter formatter,
619+
OnScriptError onScriptError
620+
) {
621+
return ctx -> new DateFieldScript(
622+
fieldName,
623+
params,
624+
searchLookup,
625+
DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
626+
onScriptError,
627+
ctx
628+
) {
629+
@Override
630+
@SuppressWarnings("unchecked")
631+
public void execute() {
632+
Map<String, Object> source = (Map<String, Object>) this.getParams().get("_source");
633+
for (Object timestamp : (List<?>) source.get("test")) {
634+
Parse parse = new Parse(this);
635+
try {
636+
emit(parse.parse(timestamp));
637+
} catch (Exception e) {
638+
// skip
639+
}
640+
}
641+
}
642+
};
643+
}
644+
645+
@Override
646+
public boolean isParsedFromSource() {
647+
return true;
648+
}
649+
};
650+
return new DateScriptFieldType(
651+
"test",
652+
factory,
653+
DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
654+
script,
655+
emptyMap(),
656+
OnScriptError.FAIL
657+
);
658+
}
659+
505660
@Override
506661
protected Query randomTermsQuery(MappedFieldType ft, SearchExecutionContext ctx) {
507662
return ft.termsQuery(randomList(1, 100, DateScriptFieldTypeTests::randomDate), ctx);

0 commit comments

Comments
 (0)