From c0436037b466e86bfa3069447ed3e908809aed23 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Fri, 28 Mar 2025 12:26:29 -0700 Subject: [PATCH] Store arrays offsets for scaled float fields natively with synthetic source (#125793) This patch builds on the work in #113757, #122999, #124594, #125529, and #125709 to natively store array offsets for scaled float fields instead of falling back to ignored source when synthetic_source_keep: arrays. (cherry picked from commit 71e74bdd6689b76a9a84f75179809ae205e96240) # Conflicts: # server/src/main/java/org/elasticsearch/index/IndexVersions.java --- .../mapper/extras/ScaledFloatFieldMapper.java | 126 +++++++++++++++--- .../extras/ScaledFloatFieldMapperTests.java | 19 +++ .../extras/ScaledFloatFieldTypeTests.java | 4 +- ...ScaledFloatOffsetDocValuesLoaderTests.java | 49 +++++++ ...eticSourceNativeArrayIntegrationTests.java | 51 +++++++ .../NativeArrayIntegrationTestCase.java | 77 +++++------ .../mapper/OffsetDocValuesLoaderTestCase.java | 124 +++++------------ 7 files changed, 297 insertions(+), 153 deletions(-) create mode 100644 modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatOffsetDocValuesLoaderTests.java create mode 100644 modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatSyntheticSourceNativeArrayIntegrationTests.java diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index 8c38c5abca13f..ebbf59a754c9f 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -19,6 +19,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -32,6 +34,7 @@ import org.elasticsearch.index.mapper.BlockDocValuesReader; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.BlockSourceReader; +import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader; import org.elasticsearch.index.mapper.FieldMapper; @@ -40,6 +43,8 @@ import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.SimpleMappedFieldType; import org.elasticsearch.index.mapper.SortedNumericDocValuesSyntheticFieldLoader; +import org.elasticsearch.index.mapper.SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer; +import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.TimeSeriesParams; @@ -67,6 +72,8 @@ import java.util.Objects; import java.util.Set; +import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName; + /** A {@link FieldMapper} for scaled floats. Values are internally multiplied * by a scaling factor and rounded to the closest long. */ public class ScaledFloatFieldMapper extends FieldMapper { @@ -125,12 +132,34 @@ public static class Builder extends FieldMapper.Builder { private final Parameter metric; private final IndexMode indexMode; + private final IndexVersion indexCreatedVersion; + private final SourceKeepMode indexSourceKeepMode; - public Builder(String name, Settings settings, IndexMode indexMode) { - this(name, IGNORE_MALFORMED_SETTING.get(settings), COERCE_SETTING.get(settings), indexMode); + public Builder( + String name, + Settings settings, + IndexMode indexMode, + IndexVersion indexCreatedVersion, + SourceKeepMode indexSourceKeepMode + ) { + this( + name, + IGNORE_MALFORMED_SETTING.get(settings), + COERCE_SETTING.get(settings), + indexMode, + indexCreatedVersion, + indexSourceKeepMode + ); } - public Builder(String name, boolean ignoreMalformedByDefault, boolean coerceByDefault, IndexMode indexMode) { + public Builder( + String name, + boolean ignoreMalformedByDefault, + boolean coerceByDefault, + IndexMode indexMode, + IndexVersion indexCreatedVersion, + SourceKeepMode indexSourceKeepMode + ) { super(name); this.ignoreMalformed = Parameter.explicitBoolParam( "ignore_malformed", @@ -159,6 +188,8 @@ public Builder(String name, boolean ignoreMalformedByDefault, boolean coerceByDe ); } }); + this.indexCreatedVersion = indexCreatedVersion; + this.indexSourceKeepMode = indexSourceKeepMode; } Builder scalingFactor(double scalingFactor) { @@ -200,11 +231,35 @@ public ScaledFloatFieldMapper build(MapperBuilderContext context) { coerce.getValue().value(), context.isSourceSynthetic() ); - return new ScaledFloatFieldMapper(leafName(), type, builderParams(this, context), context.isSourceSynthetic(), this); + String offsetsFieldName = getOffsetsFieldName( + context, + indexSourceKeepMode, + hasDocValues.getValue(), + stored.getValue(), + this, + indexCreatedVersion, + IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY + ); + return new ScaledFloatFieldMapper( + leafName(), + type, + builderParams(this, context), + context.isSourceSynthetic(), + this, + offsetsFieldName + ); } } - public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getSettings(), c.getIndexSettings().getMode())); + public static final TypeParser PARSER = new TypeParser( + (n, c) -> new Builder( + n, + c.getSettings(), + c.getIndexSettings().getMode(), + c.indexVersionCreated(), + c.getIndexSettings().sourceKeepMode() + ) + ); public static final class ScaledFloatFieldType extends SimpleMappedFieldType { @@ -532,12 +587,17 @@ public String toString() { private final TimeSeriesParams.MetricType metricType; private final IndexMode indexMode; + private final IndexVersion indexCreatedVersion; + private final String offsetsFieldName; + private final SourceKeepMode indexSourceKeepMode; + private ScaledFloatFieldMapper( String simpleName, ScaledFloatFieldType mappedFieldType, BuilderParams builderParams, boolean isSourceSynthetic, - Builder builder + Builder builder, + String offsetsFieldName ) { super(simpleName, mappedFieldType, builderParams); this.isSourceSynthetic = isSourceSynthetic; @@ -552,6 +612,9 @@ private ScaledFloatFieldMapper( this.coerceByDefault = builder.coerce.getDefaultValue().value(); this.metricType = builder.metric.getValue(); this.indexMode = builder.indexMode; + this.indexCreatedVersion = builder.indexCreatedVersion; + this.offsetsFieldName = offsetsFieldName; + this.indexSourceKeepMode = builder.indexSourceKeepMode; } boolean coerce() { @@ -563,6 +626,11 @@ public boolean ignoreMalformed() { return ignoreMalformed.value(); } + @Override + public String getOffsetFieldName() { + return offsetsFieldName; + } + @Override public ScaledFloatFieldType fieldType() { return (ScaledFloatFieldType) super.fieldType(); @@ -575,7 +643,9 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), ignoreMalformedByDefault, coerceByDefault, indexMode).metric(metricType).init(this); + return new Builder(leafName(), ignoreMalformedByDefault, coerceByDefault, indexMode, indexCreatedVersion, indexSourceKeepMode) + .metric(metricType) + .init(this); } @Override @@ -605,11 +675,16 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio value = numericValue; } + boolean shouldStoreOffsets = offsetsFieldName != null && context.isImmediateParentAnArray() && context.canAddIgnoredField(); + if (value == null) { value = nullValue; } if (value == null) { + if (shouldStoreOffsets) { + context.getOffSetContext().recordNull(offsetsFieldName); + } return; } @@ -635,6 +710,10 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio NumberFieldMapper.NumberType.LONG.addFields(context.doc(), fieldType().name(), scaledValue, indexed, hasDocValues, stored); + if (shouldStoreOffsets) { + context.getOffSetContext().recordOffset(offsetsFieldName, scaledValue); + } + if (hasDocValues == false && (indexed || stored)) { context.addToFieldNames(fieldType().name()); } @@ -776,17 +855,34 @@ public int docValueCount() { } } + private SourceLoader.SyntheticFieldLoader docValuesSyntheticFieldLoader() { + if (offsetsFieldName != null) { + var layers = new ArrayList(2); + layers.add( + new SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer( + fullPath(), + offsetsFieldName, + (b, value) -> b.value(decodeForSyntheticSource(value, scalingFactor)) + ) + ); + if (ignoreMalformed.value()) { + layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath())); + } + return new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers); + } else { + return new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) { + @Override + protected void writeValue(XContentBuilder b, long value) throws IOException { + b.value(decodeForSyntheticSource(value, scalingFactor)); + } + }; + } + } + @Override protected SyntheticSourceSupport syntheticSourceSupport() { if (hasDocValues) { - return new SyntheticSourceSupport.Native( - () -> new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) { - @Override - protected void writeValue(XContentBuilder b, long value) throws IOException { - b.value(decodeForSyntheticSource(value, scalingFactor)); - } - } - ); + return new SyntheticSourceSupport.Native(this::docValuesSyntheticFieldLoader); } return super.syntheticSourceSupport(); diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java index 1f1430d70c690..15be6b08faddc 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapperTests; @@ -365,6 +366,24 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) return new ScaledFloatSyntheticSourceSupport(ignoreMalformed); } + @Override + protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode sourceKeepMode) { + return new ScaledFloatSyntheticSourceSupport(ignoreMalformed) { + @Override + public SyntheticSourceExample example(int maxVals) { + var example = super.example(maxVals); + // Need the expectedForSyntheticSource as inputValue since MapperTestCase#testSyntheticSourceKeepArrays + // uses the inputValue as both the input and expected. + return new SyntheticSourceExample( + example.expectedForSyntheticSource(), + example.expectedForSyntheticSource(), + example.expectedForBlockLoader(), + example.mapping() + ); + } + }; + } + private static class ScaledFloatSyntheticSourceSupport implements SyntheticSourceSupport { private final boolean ignoreMalformedEnabled; private final double scalingFactor = randomDoubleBetween(0, Double.MAX_VALUE, false); diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java index aaf1abc735c3d..f2c6f81c3f742 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java @@ -222,14 +222,14 @@ public void testFieldData() throws IOException { } public void testFetchSourceValue() throws IOException { - MappedFieldType mapper = new ScaledFloatFieldMapper.Builder("field", false, false, null).scalingFactor(100) + MappedFieldType mapper = new ScaledFloatFieldMapper.Builder("field", false, false, null, null, null).scalingFactor(100) .build(MapperBuilderContext.root(false, false)) .fieldType(); assertEquals(List.of(3.14), fetchSourceValue(mapper, 3.1415926)); assertEquals(List.of(3.14), fetchSourceValue(mapper, "3.1415")); assertEquals(List.of(), fetchSourceValue(mapper, "")); - MappedFieldType nullValueMapper = new ScaledFloatFieldMapper.Builder("field", false, false, null).scalingFactor(100) + MappedFieldType nullValueMapper = new ScaledFloatFieldMapper.Builder("field", false, false, null, null, null).scalingFactor(100) .nullValue(2.71) .build(MapperBuilderContext.root(false, false)) .fieldType(); diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatOffsetDocValuesLoaderTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatOffsetDocValuesLoaderTests.java new file mode 100644 index 0000000000000..6ab3996f788c4 --- /dev/null +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatOffsetDocValuesLoaderTests.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.extras; + +import org.elasticsearch.index.mapper.OffsetDocValuesLoaderTestCase; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collection; + +import static java.util.Collections.singletonList; + +public class ScaledFloatOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase { + private static final double TEST_SCALING_FACTOR = 10.0; + + @Override + protected Collection getPlugins() { + return singletonList(new MapperExtrasPlugin()); + } + + @Override + protected void minimalMapping(XContentBuilder b) throws IOException { + b.field("type", "scaled_float").field("scaling_factor", TEST_SCALING_FACTOR); + } + + public void testOffsetArray() throws Exception { + verifyOffsets("{\"field\":[1.0,10.0,100.0,0.1,10.0,1.0,0.1,100.0]}"); + verifyOffsets("{\"field\":[10.0,null,1.0,null,5.0,null,null,6.3,1.5]}"); + } + + @Override + protected String getFieldTypeName() { + fail("Should not be called because minimalMapping is overridden"); + return null; + } + + @Override + protected Double randomValue() { + return randomLong() / TEST_SCALING_FACTOR; + } +} diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatSyntheticSourceNativeArrayIntegrationTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatSyntheticSourceNativeArrayIntegrationTests.java new file mode 100644 index 0000000000000..9aab22d3e1f07 --- /dev/null +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatSyntheticSourceNativeArrayIntegrationTests.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.extras; + +import com.carrotsearch.randomizedtesting.generators.RandomStrings; + +import org.elasticsearch.index.mapper.NativeArrayIntegrationTestCase; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collection; + +import static java.util.Collections.singletonList; + +public class ScaledFloatSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase { + private static final double TEST_SCALING_FACTOR = 10.0; + + @Override + protected Collection> getPlugins() { + return singletonList(MapperExtrasPlugin.class); + } + + @Override + protected void minimalMapping(XContentBuilder b) throws IOException { + b.field("type", "scaled_float").field("scaling_factor", TEST_SCALING_FACTOR); + } + + @Override + protected String getFieldTypeName() { + fail("Should not be called because minimalMapping is overridden"); + return null; + } + + @Override + protected Object getRandomValue() { + return randomLong() / TEST_SCALING_FACTOR; + } + + @Override + protected Object getMalformedValue() { + return randomBoolean() ? RandomStrings.randomAsciiOfLength(random(), 8) : Double.POSITIVE_INFINITY; + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java index b02ceb31c96db..4b54a09135e7c 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java @@ -35,6 +35,7 @@ import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public abstract class NativeArrayIntegrationTestCase extends ESSingleNodeTestCase { @@ -123,14 +124,9 @@ public void testSynthesizeArrayRandomIgnoresMalformed() throws Exception { inputDocuments.add(inputDocument); } - var mapping = jsonBuilder().startObject() - .startObject("properties") - .startObject("field") - .field("type", getFieldTypeName()) - .field("ignore_malformed", true) - .endObject() - .endObject() - .endObject(); + var mapping = jsonBuilder().startObject().startObject("properties").startObject("field"); + minimalMapping(mapping); + mapping.field("ignore_malformed", true).endObject().endObject().endObject(); var indexService = createIndex( "test-index", Settings.builder().put("index.mapping.source.mode", "synthetic").put("index.mapping.synthetic_source_keep", "arrays").build(), @@ -177,13 +173,9 @@ public void testSynthesizeRandomArrayInNestedContext() throws Exception { .startObject("parent") .field("type", "nested") .startObject("properties") - .startObject("field") - .field("type", getFieldTypeName()) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject(); + .startObject("field"); + minimalMapping(mapping); + mapping.endObject().endObject().endObject().endObject().endObject(); var indexService = createIndex( "test-index", @@ -241,6 +233,12 @@ public void testSynthesizeRandomArrayInNestedContext() throws Exception { } } + protected void minimalMapping(XContentBuilder b) throws IOException { + String fieldTypeName = getFieldTypeName(); + assertThat(fieldTypeName, notNullValue()); + b.field("type", fieldTypeName); + } + protected abstract String getFieldTypeName(); protected abstract Object getRandomValue(); @@ -248,13 +246,9 @@ public void testSynthesizeRandomArrayInNestedContext() throws Exception { protected abstract Object getMalformedValue(); protected void verifySyntheticArray(Object[][] arrays) throws IOException { - var mapping = jsonBuilder().startObject() - .startObject("properties") - .startObject("field") - .field("type", getFieldTypeName()) - .endObject() - .endObject() - .endObject(); + var mapping = jsonBuilder().startObject().startObject("properties").startObject("field"); + minimalMapping(mapping); + mapping.endObject().endObject().endObject(); verifySyntheticArray(arrays, mapping, "_id"); } @@ -325,20 +319,17 @@ protected void verifySyntheticArray( } protected void verifySyntheticObjectArray(List> documents) throws IOException { + var mapping = jsonBuilder().startObject() + .startObject("properties") + .startObject("object") + .startObject("properties") + .startObject("field"); + minimalMapping(mapping); + mapping.endObject().endObject().endObject().endObject().endObject(); var indexService = createIndex( "test-index", Settings.builder().put("index.mapping.source.mode", "synthetic").put("index.mapping.synthetic_source_keep", "arrays").build(), - jsonBuilder().startObject() - .startObject("properties") - .startObject("object") - .startObject("properties") - .startObject("field") - .field("type", getFieldTypeName()) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() + mapping ); for (int i = 0; i < documents.size(); i++) { var document = documents.get(i); @@ -393,20 +384,18 @@ protected void verifySyntheticObjectArray(List> documents) throws } protected void verifySyntheticArrayInObject(List documents) throws IOException { + var mapping = jsonBuilder().startObject() + .startObject("properties") + .startObject("object") + .startObject("properties") + .startObject("field"); + minimalMapping(mapping); + mapping.endObject().endObject().endObject().endObject().endObject(); + var indexService = createIndex( "test-index", Settings.builder().put("index.mapping.source.mode", "synthetic").put("index.mapping.synthetic_source_keep", "arrays").build(), - jsonBuilder().startObject() - .startObject("properties") - .startObject("object") - .startObject("properties") - .startObject("field") - .field("type", getFieldTypeName()) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() + mapping ); for (int i = 0; i < documents.size(); i++) { var arrayValue = documents.get(i); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/OffsetDocValuesLoaderTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/OffsetDocValuesLoaderTestCase.java index c18f85b8e6c0d..227248cf7d5d2 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/OffsetDocValuesLoaderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/OffsetDocValuesLoaderTestCase.java @@ -20,6 +20,7 @@ import java.util.HashSet; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public abstract class OffsetDocValuesLoaderTestCase extends MapperServiceTestCase { @@ -33,18 +34,10 @@ protected Settings getIndexSettings() { } public void testOffsetArrayNoDocValues() throws Exception { - String mapping = """ - { - "_doc": { - "properties": { - "field": { - "type": "{{type}}", - "doc_values": false - } - } - } - } - """.replace("{{type}}", getFieldTypeName()); + XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties").startObject("field"); + minimalMapping(mapping); + mapping.field("doc_values", false); + mapping.endObject().endObject().endObject().endObject(); try (var mapperService = createMapperService(mapping)) { var fieldMapper = mapperService.mappingLookup().getMapper("field"); assertThat(fieldMapper.getOffsetFieldName(), nullValue()); @@ -52,19 +45,10 @@ public void testOffsetArrayNoDocValues() throws Exception { } public void testOffsetArrayStored() throws Exception { - String mapping = """ - { - "_doc": { - "properties": { - "field": { - "type": "{{type}}", - "store": true - } - } - } - } - """.replace("{{type}}", getFieldTypeName()); - ; + XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties").startObject("field"); + minimalMapping(mapping); + mapping.field("store", true); + mapping.endObject().endObject().endObject().endObject(); try (var mapperService = createMapperService(mapping)) { var fieldMapper = mapperService.mappingLookup().getMapper("field"); assertThat(fieldMapper.getOffsetFieldName(), nullValue()); @@ -72,22 +56,10 @@ public void testOffsetArrayStored() throws Exception { } public void testOffsetMultiFields() throws Exception { - String mapping = """ - { - "_doc": { - "properties": { - "field": { - "type": "{{type}}", - "fields": { - "sub": { - "type": "text" - } - } - } - } - } - } - """.replace("{{type}}", getFieldTypeName()); + XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties").startObject("field"); + minimalMapping(mapping); + mapping.startObject("fields").startObject("sub").field("type", "text").endObject().endObject(); + mapping.endObject().endObject().endObject().endObject(); try (var mapperService = createMapperService(mapping)) { var fieldMapper = mapperService.mappingLookup().getMapper("field"); assertThat(fieldMapper.getOffsetFieldName(), nullValue()); @@ -95,17 +67,9 @@ public void testOffsetMultiFields() throws Exception { } public void testOffsetArrayNoSyntheticSource() throws Exception { - String mapping = """ - { - "_doc": { - "properties": { - "field": { - "type": "{{type}}" - } - } - } - } - """.replace("{{type}}", getFieldTypeName()); + XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties").startObject("field"); + minimalMapping(mapping); + mapping.endObject().endObject().endObject().endObject(); try (var mapperService = createMapperService(Settings.EMPTY, mapping)) { var fieldMapper = mapperService.mappingLookup().getMapper("field"); assertThat(fieldMapper.getOffsetFieldName(), nullValue()); @@ -114,36 +78,14 @@ public void testOffsetArrayNoSyntheticSource() throws Exception { public void testOffsetArrayNoSourceArrayKeep() throws Exception { var settingsBuilder = Settings.builder().put("index.mapping.source.mode", "synthetic"); - String mapping; + XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties").startObject("field"); + minimalMapping(mapping); if (randomBoolean()) { - mapping = """ - { - "_doc": { - "properties": { - "field": { - "type": "{{type}}", - "synthetic_source_keep": "{{synthetic_source_keep}}" - } - } - } - } - """.replace("{{synthetic_source_keep}}", randomBoolean() ? "none" : "all").replace("{{type}}", getFieldTypeName()); - } else { - mapping = """ - { - "_doc": { - "properties": { - "field": { - "type": "{{type}}" - } - } - } - } - """.replace("{{type}}", getFieldTypeName()); - if (randomBoolean()) { - settingsBuilder.put("index.mapping.synthetic_source_keep", "none"); - } + mapping.field("synthetic_source_keep", randomBoolean() ? "none" : "all"); + } else if (randomBoolean()) { + settingsBuilder.put("index.mapping.synthetic_source_keep", "none"); } + mapping.endObject().endObject().endObject().endObject(); try (var mapperService = createMapperService(settingsBuilder.build(), mapping)) { var fieldMapper = mapperService.mappingLookup().getMapper("field"); assertThat(fieldMapper.getOffsetFieldName(), nullValue()); @@ -183,6 +125,12 @@ public void testOffsetArrayRandom() throws Exception { verifyOffsets("{\"field\":" + values + "}"); } + protected void minimalMapping(XContentBuilder b) throws IOException { + String fieldTypeName = getFieldTypeName(); + assertThat(fieldTypeName, notNullValue()); + b.field("type", fieldTypeName); + } + protected abstract String getFieldTypeName(); protected abstract Object randomValue(); @@ -192,21 +140,13 @@ protected void verifyOffsets(String source) throws IOException { } protected void verifyOffsets(String source, String expectedSource) throws IOException { - String mapping = """ - { - "_doc": { - "properties": { - "field": { - "type": "{{type}}" - } - } - } - } - """.replace("{{type}}", getFieldTypeName()); + XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties").startObject("field"); + minimalMapping(mapping); + mapping.endObject().endObject().endObject().endObject(); verifyOffsets(mapping, source, expectedSource); } - private void verifyOffsets(String mapping, String source, String expectedSource) throws IOException { + private void verifyOffsets(XContentBuilder mapping, String source, String expectedSource) throws IOException { try (var mapperService = createMapperService(mapping)) { var mapper = mapperService.documentMapper();