Skip to content

Commit db3eb3a

Browse files
lktslkts
committed
wip
1 parent fc4d8d6 commit db3eb3a

File tree

37 files changed

+567
-43
lines changed

37 files changed

+567
-43
lines changed

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -343,7 +343,7 @@ public Builder builder(BlockFactory factory, int expectedCount) {
343343
private FallbackSyntheticSourceBlockLoader.Reader<?> fallbackSyntheticSourceBlockLoaderReader() {
344344
var nullValueAdjusted = nullValue != null ? adjustSourceValue(nullValue, scalingFactor) : null;
345345

346-
return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<Double>(nullValue) {
346+
return new FallbackSyntheticSourceBlockLoader.SingleValueReader<Double>(nullValue) {
347347
@Override
348348
public void convertValue(Object value, List<Double> accumulator) {
349349
if (coerce && value.equals("")) {

‎server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java‎

Lines changed: 72 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
package org.elasticsearch.index.mapper;
1010

1111
import org.apache.lucene.search.Query;
12+
import org.apache.lucene.util.BytesRef;
1213
import org.elasticsearch.common.CheckedBiConsumer;
1314
import org.elasticsearch.common.Explicit;
1415
import org.elasticsearch.common.geo.GeometryFormatterFactory;
@@ -67,12 +68,16 @@ public abstract void parse(XContentParser parser, CheckedConsumer<T, IOException
6768

6869
private void fetchFromSource(Object sourceMap, Consumer<T> consumer) {
6970
try (XContentParser parser = wrapObject(sourceMap)) {
70-
parse(parser, v -> consumer.accept(normalizeFromSource(v)), NoopMalformedValueHandler.INSTANCE);
71+
parseFromSource(parser, consumer);
7172
} catch (IOException e) {
7273
throw new UncheckedIOException(e);
7374
}
7475
}
7576

77+
private void parseFromSource(XContentParser parser, Consumer<T> consumer) throws IOException {
78+
parse(parser, v -> consumer.accept(normalizeFromSource(v)), NoopMalformedValueHandler.INSTANCE);
79+
}
80+
7681
/**
7782
* Normalize a geometry when reading from source. When reading from source we can skip
7883
* some expensive steps as the geometry has already been indexed.
@@ -187,6 +192,72 @@ protected BlockLoader blockLoaderFromSource(BlockLoaderContext blContext) {
187192
}
188193

189194
protected abstract Object nullValueAsSource(T nullValue);
195+
196+
protected BlockLoader blockLoaderFromFallbackSyntheticSource(BlockLoaderContext blContext) {
197+
Function<List<T>, List<Object>> formatter = getFormatter(GeometryFormatterFactory.WKB);
198+
199+
return new FallbackSyntheticSourceBlockLoader(new GeometriesFallbackSyntheticSourceReader(geometryParser, formatter), name()) {
200+
@Override
201+
public Builder builder(BlockFactory factory, int expectedCount) {
202+
return factory.bytesRefs(expectedCount);
203+
}
204+
};
205+
}
206+
207+
private class GeometriesFallbackSyntheticSourceReader implements FallbackSyntheticSourceBlockLoader.Reader<BytesRef> {
208+
private final Parser<T> geometryParser;
209+
private final Function<List<T>, List<Object>> formatter;
210+
211+
private GeometriesFallbackSyntheticSourceReader(Parser<T> geometryParser, Function<List<T>, List<Object>> formatter) {
212+
this.geometryParser = geometryParser;
213+
this.formatter = formatter;
214+
}
215+
216+
@Override
217+
public void convertValue(Object value, List<BytesRef> accumulator) {
218+
final List<T> values = new ArrayList<>();
219+
220+
geometryParser.fetchFromSource(value, values::add);
221+
var formatted = formatter.apply(values);
222+
223+
for (var formattedValue : formatted) {
224+
if (formattedValue instanceof byte[] wkb) {
225+
accumulator.add(new BytesRef(wkb));
226+
} else {
227+
throw new IllegalArgumentException(
228+
"Unsupported source type for spatial geometry: " + formattedValue.getClass().getSimpleName()
229+
);
230+
}
231+
}
232+
}
233+
234+
@Override
235+
public void parse(XContentParser parser, List<BytesRef> accumulator) throws IOException {
236+
final List<T> values = new ArrayList<>();
237+
238+
geometryParser.parseFromSource(parser, values::add);
239+
var formatted = formatter.apply(values);
240+
241+
for (var formattedValue : formatted) {
242+
if (formattedValue instanceof byte[] wkb) {
243+
accumulator.add(new BytesRef(wkb));
244+
} else {
245+
throw new IllegalArgumentException(
246+
"Unsupported source type for spatial geometry: " + formattedValue.getClass().getSimpleName()
247+
);
248+
}
249+
}
250+
}
251+
252+
@Override
253+
public void writeToBlock(List<BytesRef> values, BlockLoader.Builder blockBuilder) {
254+
var bytesRefBuilder = (BlockLoader.BytesRefBuilder) blockBuilder;
255+
256+
for (var value : values) {
257+
bytesRefBuilder.appendBytesRef(value);
258+
}
259+
}
260+
}
190261
}
191262

192263
private final Explicit<Boolean> ignoreMalformed;

‎server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -331,7 +331,7 @@ public Builder builder(BlockFactory factory, int expectedCount) {
331331
}
332332

333333
private FallbackSyntheticSourceBlockLoader.Reader<?> fallbackSyntheticSourceBlockLoaderReader() {
334-
return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<Boolean>(nullValue) {
334+
return new FallbackSyntheticSourceBlockLoader.SingleValueReader<Boolean>(nullValue) {
335335
@Override
336336
public void convertValue(Object value, List<Boolean> accumulator) {
337337
try {
@@ -360,10 +360,10 @@ protected void parseNonNullValue(XContentParser parser, List<Boolean> accumulato
360360

361361
@Override
362362
public void writeToBlock(List<Boolean> values, BlockLoader.Builder blockBuilder) {
363-
var longBuilder = (BlockLoader.BooleanBuilder) blockBuilder;
363+
var booleanBuilder = (BlockLoader.BooleanBuilder) blockBuilder;
364364

365365
for (var value : values) {
366-
longBuilder.appendBoolean(value);
366+
booleanBuilder.appendBoolean(value);
367367
}
368368
}
369369
};

‎server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -963,7 +963,7 @@ public Builder builder(BlockFactory factory, int expectedCount) {
963963
private FallbackSyntheticSourceBlockLoader.Reader<?> fallbackSyntheticSourceBlockLoaderReader() {
964964
Function<String, Long> dateParser = this::parse;
965965

966-
return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<Long>(nullValue) {
966+
return new FallbackSyntheticSourceBlockLoader.SingleValueReader<Long>(nullValue) {
967967
@Override
968968
public void convertValue(Object value, List<Long> accumulator) {
969969
try {

‎server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -235,13 +235,6 @@ private void parseFieldFromParent(IgnoredSourceFieldMapper.NameValue nameValue,
235235
}
236236

237237
private void parseWithReader(XContentParser parser, List<T> blockValues) throws IOException {
238-
if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
239-
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
240-
reader.parse(parser, blockValues);
241-
}
242-
return;
243-
}
244-
245238
reader.parse(parser, blockValues);
246239
}
247240

@@ -274,10 +267,15 @@ public interface Reader<T> {
274267
void writeToBlock(List<T> values, Builder blockBuilder);
275268
}
276269

277-
public abstract static class ReaderWithNullValueSupport<T> implements Reader<T> {
270+
/**
271+
* Reader for field types that don't parse arrays (arrays are always treated as multiple values)
272+
* as opposed to field types that treat arrays as special cases (for example point).
273+
* @param <T>
274+
*/
275+
public abstract static class SingleValueReader<T> implements Reader<T> {
278276
private final Object nullValue;
279277

280-
public ReaderWithNullValueSupport(Object nullValue) {
278+
public SingleValueReader(Object nullValue) {
281279
this.nullValue = nullValue;
282280
}
283281

@@ -289,6 +287,18 @@ public void parse(XContentParser parser, List<T> accumulator) throws IOException
289287
}
290288
return;
291289
}
290+
if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
291+
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
292+
if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
293+
if (nullValue != null) {
294+
convertValue(nullValue, accumulator);
295+
}
296+
} else {
297+
parseNonNullValue(parser, accumulator);
298+
}
299+
}
300+
return;
301+
}
292302

293303
parseNonNullValue(parser, accumulator);
294304
}

‎server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -770,7 +770,7 @@ public Builder builder(BlockFactory factory, int expectedCount) {
770770

771771
private FallbackSyntheticSourceBlockLoader.Reader<?> fallbackSyntheticSourceBlockLoaderReader() {
772772
var nullValueBytes = nullValue != null ? new BytesRef(nullValue) : null;
773-
return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<BytesRef>(nullValueBytes) {
773+
return new FallbackSyntheticSourceBlockLoader.SingleValueReader<BytesRef>(nullValueBytes) {
774774
@Override
775775
public void convertValue(Object value, List<BytesRef> accumulator) {
776776
String stringValue = ((BytesRef) value).utf8ToString();

‎server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1729,8 +1729,7 @@ public Builder builder(BlockFactory factory, int expectedCount) {
17291729
};
17301730
}
17311731

1732-
abstract static class NumberFallbackSyntheticSourceReader extends FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<
1733-
Number> {
1732+
abstract static class NumberFallbackSyntheticSourceReader extends FallbackSyntheticSourceBlockLoader.SingleValueReader<Number> {
17341733
private final NumberType type;
17351734
private final Number nullValue;
17361735
private final boolean coerce;

‎test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
import org.elasticsearch.index.fieldvisitor.StoredFieldLoader;
2121
import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification;
2222
import org.elasticsearch.logsdb.datageneration.DocumentGenerator;
23-
import org.elasticsearch.logsdb.datageneration.FieldType;
2423
import org.elasticsearch.logsdb.datageneration.Mapping;
2524
import org.elasticsearch.logsdb.datageneration.MappingGenerator;
2625
import org.elasticsearch.logsdb.datageneration.Template;
@@ -35,6 +34,7 @@
3534

3635
import java.io.IOException;
3736
import java.util.ArrayList;
37+
import java.util.Collection;
3838
import java.util.HashMap;
3939
import java.util.List;
4040
import java.util.Map;
@@ -59,14 +59,18 @@ public static List<Object[]> args() {
5959

6060
public record Params(boolean syntheticSource, MappedFieldType.FieldExtractPreference preference) {}
6161

62-
private final FieldType fieldType;
62+
private final String fieldType;
6363
protected final Params params;
6464

6565
private final String fieldName;
6666
private final MappingGenerator mappingGenerator;
6767
private final DocumentGenerator documentGenerator;
6868

69-
protected BlockLoaderTestCase(FieldType fieldType, Params params) {
69+
protected BlockLoaderTestCase(String fieldType, Params params) {
70+
this(fieldType, List.of(), params);
71+
}
72+
73+
protected BlockLoaderTestCase(String fieldType, Collection<DataSourceHandler> customHandlers, Params params) {
7074
this.fieldType = fieldType;
7175
this.params = params;
7276
this.fieldName = randomAlphaOfLengthBetween(5, 10);
@@ -87,6 +91,7 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle(
8791
return new DataSourceResponse.ObjectMappingParametersGenerator(HashMap::new); // just defaults
8892
}
8993
}))
94+
.withDataSourceHandlers(customHandlers)
9095
.build();
9196

9297
this.mappingGenerator = new MappingGenerator(specification);

‎test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldBlockLoaderTestCase.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
public abstract class NumberFieldBlockLoaderTestCase<T extends Number> extends BlockLoaderTestCase {
1919
public NumberFieldBlockLoaderTestCase(FieldType fieldType, Params params) {
20-
super(fieldType, params);
20+
super(fieldType.toString(), params);
2121
}
2222

2323
@Override

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DocumentGenerator.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,14 @@ private void generateFields(Map<String, Object> document, Map<String, Template.E
6161
// Unsigned long does not play well when dynamically mapped because
6262
// it gets mapped as just long and large values fail to index.
6363
// Just skip it.
64-
if (leaf.type() == FieldType.UNSIGNED_LONG && fieldMapping == null) {
64+
// TODO we can actually handle this in UnsignedLongFieldDataGenerator
65+
if (leaf.type().equals(FieldType.UNSIGNED_LONG.toString()) && fieldMapping == null) {
6566
continue;
6667
}
6768

68-
var generator = leaf.type().generator(fieldName, specification.dataSource());
69-
69+
var generator = specification.dataSource()
70+
.get(new DataSourceRequest.FieldDataGenerator(fieldName, leaf.type(), specification.dataSource()))
71+
.generator();
7072
document.put(fieldName, generator.generateValue(fieldMapping));
7173
} else if (templateEntry instanceof Template.Object object) {
7274
Optional<Integer> arrayLength = objectArrayGenerator.lengthGenerator().get();

0 commit comments

Comments
 (0)