Skip to content

Commit 7aeb2a2

Browse files
lktslkts
committed
Prototype FallbackSyntheticSourceBlockLoader
1 parent da0640b commit 7aeb2a2

File tree

4 files changed

+281
-0
lines changed

4 files changed

+281
-0
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.mapper;
11+
12+
import org.apache.lucene.index.LeafReaderContext;
13+
import org.apache.lucene.index.SortedSetDocValues;
14+
import org.apache.lucene.util.BytesRef;
15+
import org.elasticsearch.search.fetch.StoredFieldsSpec;
16+
import org.elasticsearch.xcontent.XContentParser;
17+
import org.elasticsearch.xcontent.XContentParserConfiguration;
18+
19+
import java.io.IOException;
20+
import java.util.ArrayList;
21+
import java.util.List;
22+
import java.util.Optional;
23+
import java.util.Set;
24+
25+
/**
26+
* Block loader for fields that use fallback synthetic source implementation.
27+
* <br>
28+
* Usually fields have doc_values or stored fields and block loaders use them directly. In some cases neither is available
29+
* and we would fall back to (potentially synthetic) _source. However, in case of synthetic source, there is actually no need to
30+
* construct the entire _source. We know that there is no doc_values and stored fields, and therefore we will be using fallback synthetic
31+
* source. That is equivalent to just reading _ignored_source stored field directly and doing an in-place synthetic source just
32+
* for this field.
33+
* <br>
34+
* See {@link IgnoredSourceFieldMapper}.
35+
*/
36+
public abstract class FallbackSyntheticSourceBlockLoader implements BlockLoader {
37+
private final MappedFieldType.BlockLoaderContext blockLoaderContext;
38+
private final Reader reader;
39+
private final String fieldName;
40+
41+
protected FallbackSyntheticSourceBlockLoader(MappedFieldType.BlockLoaderContext blockLoaderContext, Reader reader, String fieldName) {
42+
this.blockLoaderContext = blockLoaderContext;
43+
this.reader = reader;
44+
this.fieldName = fieldName;
45+
}
46+
47+
@Override
48+
public ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) throws IOException {
49+
return null;
50+
}
51+
52+
@Override
53+
public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOException {
54+
return new IgnoredSourceRowStrideReader(fieldName, reader);
55+
}
56+
57+
@Override
58+
public StoredFieldsSpec rowStrideStoredFieldSpec() {
59+
return new StoredFieldsSpec(false, false, Set.of(IgnoredSourceFieldMapper.NAME));
60+
}
61+
62+
@Override
63+
public boolean supportsOrdinals() {
64+
return false;
65+
}
66+
67+
@Override
68+
public SortedSetDocValues ordinals(LeafReaderContext context) throws IOException {
69+
throw new UnsupportedOperationException();
70+
}
71+
72+
private static class IgnoredSourceRowStrideReader implements RowStrideReader {
73+
private final String fieldName;
74+
private final Reader reader;
75+
76+
private IgnoredSourceRowStrideReader(String fieldName, Reader reader) {
77+
this.fieldName = fieldName;
78+
this.reader = reader;
79+
}
80+
81+
@Override
82+
public void read(int docId, StoredFields storedFields, Builder builder) throws IOException {
83+
var ignoredSource = storedFields.storedFields().get(IgnoredSourceFieldMapper.NAME);
84+
if (ignoredSource == null) {
85+
return;
86+
}
87+
88+
boolean written = false;
89+
for (Object value : ignoredSource) {
90+
IgnoredSourceFieldMapper.NameValue nameValue = IgnoredSourceFieldMapper.decode(value);
91+
if (nameValue.name().equals(fieldName)) {
92+
// Leaf field is stored directly (not as a part of a parent object), let's try to decode it.
93+
Optional<Object> singleValue = XContentDataHelper.decode(nameValue.value());
94+
if (singleValue.isPresent()) {
95+
reader.readValue(singleValue.get(), builder);
96+
written = true;
97+
continue;
98+
}
99+
100+
// We have a value for this field but it's an array or an object
101+
var type = XContentDataHelper.decodeType(nameValue.value());
102+
assert type.isPresent();
103+
104+
var filterParserConfig = XContentParserConfiguration.EMPTY.withFiltering("", Set.of(fieldName), Set.of(), true);
105+
try (XContentParser parser = type.get().xContent().createParser(filterParserConfig, nameValue.value().bytes, nameValue.value().offset + 1, nameValue.value().length - 1)) {
106+
parser.nextToken();
107+
reader.parse(parser, builder);
108+
}
109+
written = true;
110+
}
111+
// It is possible that the field is stored as part of the parent object, we'll need to look at those too and use something
112+
// similar to reader.parse()
113+
}
114+
115+
if (written == false) {
116+
builder.appendNull();
117+
}
118+
}
119+
120+
@Override
121+
public boolean canReuse(int startingDocID) {
122+
return false;
123+
}
124+
}
125+
126+
public interface Reader {
127+
/**
128+
* Converts a raw stored value for this field to a value in a format suitable for block loader and appends it to block.
129+
* @param value raw decoded value from _ignored_source field (synthetic _source value)
130+
*/
131+
void readValue(Object value, Builder builder);
132+
133+
/**
134+
* Parses one or more complex values using a provided parser and appends results to a block.
135+
* @param parser parser of a value from _ignored_source field (synthetic _source value)
136+
*/
137+
void parse(XContentParser parser, Builder builder) throws IOException;
138+
}
139+
}

‎server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
import org.elasticsearch.search.runtime.StringScriptFieldTermQuery;
6565
import org.elasticsearch.search.runtime.StringScriptFieldWildcardQuery;
6666
import org.elasticsearch.xcontent.XContentBuilder;
67+
import org.elasticsearch.xcontent.XContentParser;
6768

6869
import java.io.IOException;
6970
import java.io.UncheckedIOException;
@@ -631,6 +632,40 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
631632
if (isStored()) {
632633
return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(name());
633634
}
635+
636+
if (isSyntheticSource) {
637+
var reader = new FallbackSyntheticSourceBlockLoader.Reader() {
638+
@Override
639+
public void readValue(Object value, BlockLoader.Builder builder) {
640+
assert value instanceof BytesRef;
641+
// TODO apply ignore_above/normalizer same as sourceValueFetcher()
642+
((BlockLoader.BytesRefBuilder )builder).appendBytesRef((BytesRef) value);
643+
}
644+
645+
@Override
646+
public void parse(XContentParser parser, BlockLoader.Builder builder) throws IOException {
647+
assert parser.currentToken() == XContentParser.Token.START_ARRAY;
648+
649+
var bytesRefBuilder = (BlockLoader.BytesRefBuilder ) builder;
650+
651+
bytesRefBuilder.beginPositionEntry();
652+
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
653+
assert parser.currentToken() == XContentParser.Token.VALUE_STRING;
654+
655+
bytesRefBuilder.appendBytesRef(new BytesRef(parser.charBuffer()));
656+
}
657+
bytesRefBuilder.endPositionEntry();
658+
}
659+
};
660+
661+
return new FallbackSyntheticSourceBlockLoader(blContext, reader, name()) {
662+
@Override
663+
public Builder builder(BlockFactory factory, int expectedCount) {
664+
return factory.bytesRefs(expectedCount);
665+
}
666+
};
667+
}
668+
634669
SourceValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name()));
635670
return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext));
636671
}

‎server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,28 @@ static void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
110110
}
111111
}
112112

113+
/**
114+
* Decode the value in the passed {@link BytesRef} in place and return it.
115+
* Returns {@link Optional#empty()} for complex values (objects and arrays).
116+
*/
117+
static Optional<Object> decode(BytesRef r) {
118+
return switch ((char) r.bytes[r.offset]) {
119+
case BINARY_ENCODING -> Optional.of(TypeUtils.EMBEDDED_OBJECT.decode(r));
120+
case CBOR_OBJECT_ENCODING, JSON_OBJECT_ENCODING, YAML_OBJECT_ENCODING, SMILE_OBJECT_ENCODING -> Optional.empty();
121+
case BIG_DECIMAL_ENCODING -> Optional.of(TypeUtils.BIG_DECIMAL.decode(r));
122+
case FALSE_ENCODING, TRUE_ENCODING -> Optional.of(TypeUtils.BOOLEAN.decode(r));
123+
case BIG_INTEGER_ENCODING -> Optional.of(TypeUtils.BIG_INTEGER.decode(r));
124+
case STRING_ENCODING -> Optional.of(TypeUtils.STRING.decode(r));
125+
case INTEGER_ENCODING -> Optional.of(TypeUtils.INTEGER.decode(r));
126+
case LONG_ENCODING -> Optional.of(TypeUtils.LONG.decode(r));
127+
case DOUBLE_ENCODING -> Optional.of(TypeUtils.DOUBLE.decode(r));
128+
case FLOAT_ENCODING -> Optional.of(TypeUtils.FLOAT.decode(r));
129+
case NULL_ENCODING -> Optional.ofNullable(TypeUtils.NULL.decode(r));
130+
case VOID_ENCODING -> Optional.of(TypeUtils.VOID.decode(r));
131+
default -> throw new IllegalArgumentException("Can't decode " + r);
132+
};
133+
}
134+
113135
/**
114136
* Determines if the given {@link BytesRef}, encoded with {@link XContentDataHelper#encodeToken(XContentParser)},
115137
* is an encoded object.
@@ -339,6 +361,11 @@ byte[] encode(XContentParser parser) throws IOException {
339361
return bytes;
340362
}
341363

364+
@Override
365+
Object decode(BytesRef r) {
366+
return new BytesRef(r.bytes, r.offset + 1, r.length - 1);
367+
}
368+
342369
@Override
343370
void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
344371
b.value(new BytesRef(r.bytes, r.offset + 1, r.length - 1).utf8ToString());
@@ -359,6 +386,11 @@ byte[] encode(XContentParser parser) throws IOException {
359386
return bytes;
360387
}
361388

389+
@Override
390+
Object decode(BytesRef r) {
391+
return ByteUtils.readIntLE(r.bytes, 1 + r.offset);
392+
}
393+
362394
@Override
363395
void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
364396
b.value(ByteUtils.readIntLE(r.bytes, 1 + r.offset));
@@ -379,6 +411,11 @@ byte[] encode(XContentParser parser) throws IOException {
379411
return bytes;
380412
}
381413

414+
@Override
415+
Object decode(BytesRef r) {
416+
return ByteUtils.readLongLE(r.bytes, 1 + r.offset);
417+
}
418+
382419
@Override
383420
void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
384421
b.value(ByteUtils.readLongLE(r.bytes, 1 + r.offset));
@@ -399,6 +436,11 @@ byte[] encode(XContentParser parser) throws IOException {
399436
return bytes;
400437
}
401438

439+
@Override
440+
Object decode(BytesRef r) {
441+
return ByteUtils.readDoubleLE(r.bytes, 1 + r.offset);
442+
}
443+
402444
@Override
403445
void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
404446
b.value(ByteUtils.readDoubleLE(r.bytes, 1 + r.offset));
@@ -419,6 +461,11 @@ byte[] encode(XContentParser parser) throws IOException {
419461
return bytes;
420462
}
421463

464+
@Override
465+
Object decode(BytesRef r) {
466+
return ByteUtils.readFloatLE(r.bytes, 1 + r.offset);
467+
}
468+
422469
@Override
423470
void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
424471
b.value(ByteUtils.readFloatLE(r.bytes, 1 + r.offset));
@@ -437,6 +484,11 @@ byte[] encode(XContentParser parser) throws IOException {
437484
return bytes;
438485
}
439486

487+
@Override
488+
Object decode(BytesRef r) {
489+
return new BigInteger(r.bytes, r.offset + 1, r.length - 1);
490+
}
491+
440492
@Override
441493
void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
442494
b.value(new BigInteger(r.bytes, r.offset + 1, r.length - 1));
@@ -455,6 +507,15 @@ byte[] encode(XContentParser parser) throws IOException {
455507
return bytes;
456508
}
457509

510+
@Override
511+
Object decode(BytesRef r) {
512+
if (r.length < 5) {
513+
throw new IllegalArgumentException("Can't decode " + r);
514+
}
515+
int scale = ByteUtils.readIntLE(r.bytes, r.offset + 1);
516+
return new BigDecimal(new BigInteger(r.bytes, r.offset + 5, r.length - 5), scale);
517+
}
518+
458519
@Override
459520
void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
460521
if (r.length < 5) {
@@ -477,6 +538,15 @@ byte[] encode(XContentParser parser) throws IOException {
477538
return bytes;
478539
}
479540

541+
@Override
542+
Object decode(BytesRef r) {
543+
if (r.length != 1) {
544+
throw new IllegalArgumentException("Can't decode " + r);
545+
}
546+
assert r.bytes[r.offset] == 't' || r.bytes[r.offset] == 'f' : r.bytes[r.offset];
547+
return r.bytes[r.offset] == 't';
548+
}
549+
480550
@Override
481551
void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
482552
if (r.length != 1) {
@@ -499,6 +569,11 @@ byte[] encode(XContentParser parser) throws IOException {
499569
return bytes;
500570
}
501571

572+
@Override
573+
Object decode(BytesRef r) {
574+
return null;
575+
}
576+
502577
@Override
503578
void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
504579
b.nullValue();
@@ -517,6 +592,11 @@ byte[] encode(XContentParser parser) throws IOException {
517592
return bytes;
518593
}
519594

595+
@Override
596+
Object decode(BytesRef r) {
597+
return new BytesRef(r.bytes, r.offset + 1, r.length - 1);
598+
}
599+
520600
@Override
521601
void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
522602
b.value(r.bytes, r.offset + 1, r.length - 1);
@@ -538,6 +618,11 @@ byte[] encode(XContentParser parser) throws IOException {
538618
}
539619
}
540620

621+
@Override
622+
Object decode(BytesRef r) {
623+
throw new UnsupportedOperationException();
624+
}
625+
541626
@Override
542627
void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
543628
switch ((char) r.bytes[r.offset]) {
@@ -562,6 +647,11 @@ byte[] encode(XContentParser parser) {
562647
return bytes;
563648
}
564649

650+
@Override
651+
Object decode(BytesRef r) {
652+
throw new UnsupportedOperationException();
653+
}
654+
565655
@Override
566656
void decodeAndWrite(XContentBuilder b, BytesRef r) {
567657
// NOOP
@@ -591,6 +681,8 @@ void assertValidEncoding(byte[] encodedValue) {
591681

592682
abstract byte[] encode(XContentParser parser) throws IOException;
593683

684+
abstract Object decode(BytesRef r);
685+
594686
abstract void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException;
595687

596688
static byte[] encode(BigInteger n, Byte encoding) throws IOException {

0 commit comments

Comments
 (0)