From 700a15b640ce38451365ff8a4df6885b7de0c9db Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Thu, 27 Mar 2025 09:13:54 -0700 Subject: [PATCH 1/6] Initial implementation of offset encoding for scaled float fields --- .../mapper/extras/ScaledFloatFieldMapper.java | 126 +++++++++++++++--- .../extras/ScaledFloatFieldTypeTests.java | 4 +- .../elasticsearch/index/IndexVersions.java | 1 + 3 files changed, 114 insertions(+), 17 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index 70622f72d1167..9e7940201c4a3 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -19,6 +19,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -32,6 +34,7 @@ import org.elasticsearch.index.mapper.BlockDocValuesReader; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.BlockSourceReader; +import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader; import org.elasticsearch.index.mapper.FieldMapper; @@ -40,6 +43,8 @@ import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.SimpleMappedFieldType; import org.elasticsearch.index.mapper.SortedNumericDocValuesSyntheticFieldLoader; +import org.elasticsearch.index.mapper.SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer; +import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.TimeSeriesParams; @@ -67,6 +72,8 @@ import java.util.Objects; import java.util.Set; +import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName; + /** A {@link FieldMapper} for scaled floats. Values are internally multiplied * by a scaling factor and rounded to the closest long. */ public class ScaledFloatFieldMapper extends FieldMapper { @@ -125,12 +132,34 @@ public static class Builder extends FieldMapper.Builder { private final Parameter metric; private final IndexMode indexMode; + private final IndexVersion indexCreatedVersion; + private final SourceKeepMode indexSourceKeepMode; - public Builder(String name, Settings settings, IndexMode indexMode) { - this(name, IGNORE_MALFORMED_SETTING.get(settings), COERCE_SETTING.get(settings), indexMode); + public Builder( + String name, + Settings settings, + IndexMode indexMode, + IndexVersion indexCreatedVersion, + SourceKeepMode indexSourceKeepMode + ) { + this( + name, + IGNORE_MALFORMED_SETTING.get(settings), + COERCE_SETTING.get(settings), + indexMode, + indexCreatedVersion, + indexSourceKeepMode + ); } - public Builder(String name, boolean ignoreMalformedByDefault, boolean coerceByDefault, IndexMode indexMode) { + public Builder( + String name, + boolean ignoreMalformedByDefault, + boolean coerceByDefault, + IndexMode indexMode, + IndexVersion indexCreatedVersion, + SourceKeepMode indexSourceKeepMode + ) { super(name); this.ignoreMalformed = Parameter.explicitBoolParam( "ignore_malformed", @@ -159,6 +188,8 @@ public Builder(String name, boolean ignoreMalformedByDefault, boolean coerceByDe ); } }); + this.indexCreatedVersion = indexCreatedVersion; + this.indexSourceKeepMode = indexSourceKeepMode; } Builder scalingFactor(double scalingFactor) { @@ -200,11 +231,35 @@ public ScaledFloatFieldMapper build(MapperBuilderContext context) { coerce.getValue().value(), context.isSourceSynthetic() ); - return new ScaledFloatFieldMapper(leafName(), type, builderParams(this, context), context.isSourceSynthetic(), this); + String offsetsFieldName = getOffsetsFieldName( + context, + indexSourceKeepMode, + hasDocValues.getValue(), + stored.getValue(), + this, + indexCreatedVersion, + IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_SCALED_FLOAT + ); + return new ScaledFloatFieldMapper( + leafName(), + type, + builderParams(this, context), + context.isSourceSynthetic(), + this, + offsetsFieldName + ); } } - public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getSettings(), c.getIndexSettings().getMode())); + public static final TypeParser PARSER = new TypeParser( + (n, c) -> new Builder( + n, + c.getSettings(), + c.getIndexSettings().getMode(), + c.indexVersionCreated(), + c.getIndexSettings().sourceKeepMode() + ) + ); public static final class ScaledFloatFieldType extends SimpleMappedFieldType { @@ -533,12 +588,17 @@ public String toString() { private final TimeSeriesParams.MetricType metricType; private final IndexMode indexMode; + private final IndexVersion indexCreatedVersion; + private final String offsetsFieldName; + private final SourceKeepMode indexSourceKeepMode; + private ScaledFloatFieldMapper( String simpleName, ScaledFloatFieldType mappedFieldType, BuilderParams builderParams, boolean isSourceSynthetic, - Builder builder + Builder builder, + String offsetsFieldName ) { super(simpleName, mappedFieldType, builderParams); this.isSourceSynthetic = isSourceSynthetic; @@ -553,6 +613,9 @@ private ScaledFloatFieldMapper( this.coerceByDefault = builder.coerce.getDefaultValue().value(); this.metricType = builder.metric.getValue(); this.indexMode = builder.indexMode; + this.indexCreatedVersion = builder.indexCreatedVersion; + this.offsetsFieldName = offsetsFieldName; + this.indexSourceKeepMode = builder.indexSourceKeepMode; } boolean coerce() { @@ -564,6 +627,11 @@ public boolean ignoreMalformed() { return ignoreMalformed.value(); } + @Override + public String getOffsetFieldName() { + return offsetsFieldName; + } + @Override public ScaledFloatFieldType fieldType() { return (ScaledFloatFieldType) super.fieldType(); @@ -576,7 +644,9 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), ignoreMalformedByDefault, coerceByDefault, indexMode).metric(metricType).init(this); + return new Builder(leafName(), ignoreMalformedByDefault, coerceByDefault, indexMode, indexCreatedVersion, indexSourceKeepMode) + .metric(metricType) + .init(this); } @Override @@ -606,11 +676,16 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio value = numericValue; } + boolean shouldStoreOffsets = offsetsFieldName != null && context.isImmediateParentAnArray() && context.canAddIgnoredField(); + if (value == null) { value = nullValue; } if (value == null) { + if (shouldStoreOffsets) { + context.getOffSetContext().recordNull(offsetsFieldName); + } return; } @@ -636,6 +711,10 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio NumberFieldMapper.NumberType.LONG.addFields(context.doc(), fieldType().name(), scaledValue, indexed, hasDocValues, stored); + if (shouldStoreOffsets) { + context.getOffSetContext().recordOffset(offsetsFieldName, scaledValue); + } + if (hasDocValues == false && (indexed || stored)) { context.addToFieldNames(fieldType().name()); } @@ -777,17 +856,34 @@ public int docValueCount() { } } + private SourceLoader.SyntheticFieldLoader docValuesSyntheticFieldLoader() { + if (offsetsFieldName != null) { + var layers = new ArrayList(2); + layers.add( + new SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer( + fullPath(), + offsetsFieldName, + (b, value) -> b.value(decodeForSyntheticSource(value, scalingFactor)) + ) + ); + if (ignoreMalformed.value()) { + layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath())); + } + return new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers); + } else { + return new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) { + @Override + protected void writeValue(XContentBuilder b, long value) throws IOException { + b.value(decodeForSyntheticSource(value, scalingFactor)); + } + }; + } + } + @Override protected SyntheticSourceSupport syntheticSourceSupport() { if (hasDocValues) { - return new SyntheticSourceSupport.Native( - () -> new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) { - @Override - protected void writeValue(XContentBuilder b, long value) throws IOException { - b.value(decodeForSyntheticSource(value, scalingFactor)); - } - } - ); + return new SyntheticSourceSupport.Native(this::docValuesSyntheticFieldLoader); } return super.syntheticSourceSupport(); diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java index aaf1abc735c3d..f2c6f81c3f742 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java @@ -222,14 +222,14 @@ public void testFieldData() throws IOException { } public void testFetchSourceValue() throws IOException { - MappedFieldType mapper = new ScaledFloatFieldMapper.Builder("field", false, false, null).scalingFactor(100) + MappedFieldType mapper = new ScaledFloatFieldMapper.Builder("field", false, false, null, null, null).scalingFactor(100) .build(MapperBuilderContext.root(false, false)) .fieldType(); assertEquals(List.of(3.14), fetchSourceValue(mapper, 3.1415926)); assertEquals(List.of(3.14), fetchSourceValue(mapper, "3.1415")); assertEquals(List.of(), fetchSourceValue(mapper, "")); - MappedFieldType nullValueMapper = new ScaledFloatFieldMapper.Builder("field", false, false, null).scalingFactor(100) + MappedFieldType nullValueMapper = new ScaledFloatFieldMapper.Builder("field", false, false, null, null, null).scalingFactor(100) .nullValue(2.71) .build(MapperBuilderContext.root(false, false)) .fieldType(); diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 77c231f633c5d..8c2d3bccf60ba 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -157,6 +157,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_BOOLEAN = def(9_017_0_00, Version.LUCENE_10_1_0); public static final IndexVersion RESCORE_PARAMS_ALLOW_ZERO_TO_QUANTIZED_VECTORS = def(9_018_0_00, Version.LUCENE_10_1_0); public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_UNSIGNED_LONG = def(9_019_0_00, Version.LUCENE_10_1_0); + public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_SCALED_FLOAT = def(9_020_0_00, Version.LUCENE_10_1_0); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ From a917387c93e2a8549001aeee4dab97c11d45c380 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Thu, 27 Mar 2025 08:29:35 -0700 Subject: [PATCH 2/6] Add mechanism to include other mapping parameters in offset tests --- .../NativeArrayIntegrationTestCase.java | 77 +++++------ .../mapper/OffsetDocValuesLoaderTestCase.java | 124 +++++------------- 2 files changed, 65 insertions(+), 136 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java index b02ceb31c96db..4b54a09135e7c 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java @@ -35,6 +35,7 @@ import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public abstract class NativeArrayIntegrationTestCase extends ESSingleNodeTestCase { @@ -123,14 +124,9 @@ public void testSynthesizeArrayRandomIgnoresMalformed() throws Exception { inputDocuments.add(inputDocument); } - var mapping = jsonBuilder().startObject() - .startObject("properties") - .startObject("field") - .field("type", getFieldTypeName()) - .field("ignore_malformed", true) - .endObject() - .endObject() - .endObject(); + var mapping = jsonBuilder().startObject().startObject("properties").startObject("field"); + minimalMapping(mapping); + mapping.field("ignore_malformed", true).endObject().endObject().endObject(); var indexService = createIndex( "test-index", Settings.builder().put("index.mapping.source.mode", "synthetic").put("index.mapping.synthetic_source_keep", "arrays").build(), @@ -177,13 +173,9 @@ public void testSynthesizeRandomArrayInNestedContext() throws Exception { .startObject("parent") .field("type", "nested") .startObject("properties") - .startObject("field") - .field("type", getFieldTypeName()) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject(); + .startObject("field"); + minimalMapping(mapping); + mapping.endObject().endObject().endObject().endObject().endObject(); var indexService = createIndex( "test-index", @@ -241,6 +233,12 @@ public void testSynthesizeRandomArrayInNestedContext() throws Exception { } } + protected void minimalMapping(XContentBuilder b) throws IOException { + String fieldTypeName = getFieldTypeName(); + assertThat(fieldTypeName, notNullValue()); + b.field("type", fieldTypeName); + } + protected abstract String getFieldTypeName(); protected abstract Object getRandomValue(); @@ -248,13 +246,9 @@ public void testSynthesizeRandomArrayInNestedContext() throws Exception { protected abstract Object getMalformedValue(); protected void verifySyntheticArray(Object[][] arrays) throws IOException { - var mapping = jsonBuilder().startObject() - .startObject("properties") - .startObject("field") - .field("type", getFieldTypeName()) - .endObject() - .endObject() - .endObject(); + var mapping = jsonBuilder().startObject().startObject("properties").startObject("field"); + minimalMapping(mapping); + mapping.endObject().endObject().endObject(); verifySyntheticArray(arrays, mapping, "_id"); } @@ -325,20 +319,17 @@ protected void verifySyntheticArray( } protected void verifySyntheticObjectArray(List> documents) throws IOException { + var mapping = jsonBuilder().startObject() + .startObject("properties") + .startObject("object") + .startObject("properties") + .startObject("field"); + minimalMapping(mapping); + mapping.endObject().endObject().endObject().endObject().endObject(); var indexService = createIndex( "test-index", Settings.builder().put("index.mapping.source.mode", "synthetic").put("index.mapping.synthetic_source_keep", "arrays").build(), - jsonBuilder().startObject() - .startObject("properties") - .startObject("object") - .startObject("properties") - .startObject("field") - .field("type", getFieldTypeName()) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() + mapping ); for (int i = 0; i < documents.size(); i++) { var document = documents.get(i); @@ -393,20 +384,18 @@ protected void verifySyntheticObjectArray(List> documents) throws } protected void verifySyntheticArrayInObject(List documents) throws IOException { + var mapping = jsonBuilder().startObject() + .startObject("properties") + .startObject("object") + .startObject("properties") + .startObject("field"); + minimalMapping(mapping); + mapping.endObject().endObject().endObject().endObject().endObject(); + var indexService = createIndex( "test-index", Settings.builder().put("index.mapping.source.mode", "synthetic").put("index.mapping.synthetic_source_keep", "arrays").build(), - jsonBuilder().startObject() - .startObject("properties") - .startObject("object") - .startObject("properties") - .startObject("field") - .field("type", getFieldTypeName()) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() + mapping ); for (int i = 0; i < documents.size(); i++) { var arrayValue = documents.get(i); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/OffsetDocValuesLoaderTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/OffsetDocValuesLoaderTestCase.java index ab53454fa9b56..9ed6c88c71a69 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/OffsetDocValuesLoaderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/OffsetDocValuesLoaderTestCase.java @@ -20,6 +20,7 @@ import java.util.HashSet; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public abstract class OffsetDocValuesLoaderTestCase extends MapperServiceTestCase { @@ -33,18 +34,10 @@ protected Settings getIndexSettings() { } public void testOffsetArrayNoDocValues() throws Exception { - String mapping = """ - { - "_doc": { - "properties": { - "field": { - "type": "{{type}}", - "doc_values": false - } - } - } - } - """.replace("{{type}}", getFieldTypeName()); + XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties").startObject("field"); + minimalMapping(mapping); + mapping.field("doc_values", false); + mapping.endObject().endObject().endObject().endObject(); try (var mapperService = createMapperService(mapping)) { var fieldMapper = mapperService.mappingLookup().getMapper("field"); assertThat(fieldMapper.getOffsetFieldName(), nullValue()); @@ -52,19 +45,10 @@ public void testOffsetArrayNoDocValues() throws Exception { } public void testOffsetArrayStored() throws Exception { - String mapping = """ - { - "_doc": { - "properties": { - "field": { - "type": "{{type}}", - "store": true - } - } - } - } - """.replace("{{type}}", getFieldTypeName()); - ; + XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties").startObject("field"); + minimalMapping(mapping); + mapping.field("store", true); + mapping.endObject().endObject().endObject().endObject(); try (var mapperService = createMapperService(mapping)) { var fieldMapper = mapperService.mappingLookup().getMapper("field"); assertThat(fieldMapper.getOffsetFieldName(), nullValue()); @@ -72,22 +56,10 @@ public void testOffsetArrayStored() throws Exception { } public void testOffsetMultiFields() throws Exception { - String mapping = """ - { - "_doc": { - "properties": { - "field": { - "type": "{{type}}", - "fields": { - "sub": { - "type": "text" - } - } - } - } - } - } - """.replace("{{type}}", getFieldTypeName()); + XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties").startObject("field"); + minimalMapping(mapping); + mapping.startObject("fields").startObject("sub").field("type", "text").endObject().endObject(); + mapping.endObject().endObject().endObject().endObject(); try (var mapperService = createMapperService(mapping)) { var fieldMapper = mapperService.mappingLookup().getMapper("field"); assertThat(fieldMapper.getOffsetFieldName(), nullValue()); @@ -95,17 +67,9 @@ public void testOffsetMultiFields() throws Exception { } public void testOffsetArrayNoSyntheticSource() throws Exception { - String mapping = """ - { - "_doc": { - "properties": { - "field": { - "type": "{{type}}" - } - } - } - } - """.replace("{{type}}", getFieldTypeName()); + XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties").startObject("field"); + minimalMapping(mapping); + mapping.endObject().endObject().endObject().endObject(); try (var mapperService = createMapperService(Settings.EMPTY, mapping)) { var fieldMapper = mapperService.mappingLookup().getMapper("field"); assertThat(fieldMapper.getOffsetFieldName(), nullValue()); @@ -114,36 +78,14 @@ public void testOffsetArrayNoSyntheticSource() throws Exception { public void testOffsetArrayNoSourceArrayKeep() throws Exception { var settingsBuilder = Settings.builder().put("index.mapping.source.mode", "synthetic"); - String mapping; + XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties").startObject("field"); + minimalMapping(mapping); if (randomBoolean()) { - mapping = """ - { - "_doc": { - "properties": { - "field": { - "type": "{{type}}", - "synthetic_source_keep": "{{synthetic_source_keep}}" - } - } - } - } - """.replace("{{synthetic_source_keep}}", randomBoolean() ? "none" : "all").replace("{{type}}", getFieldTypeName()); - } else { - mapping = """ - { - "_doc": { - "properties": { - "field": { - "type": "{{type}}" - } - } - } - } - """.replace("{{type}}", getFieldTypeName()); - if (randomBoolean()) { - settingsBuilder.put("index.mapping.synthetic_source_keep", "none"); - } + mapping.field("synthetic_source_keep", randomBoolean() ? "none" : "all"); + } else if (randomBoolean()) { + settingsBuilder.put("index.mapping.synthetic_source_keep", "none"); } + mapping.endObject().endObject().endObject().endObject(); try (var mapperService = createMapperService(settingsBuilder.build(), mapping)) { var fieldMapper = mapperService.mappingLookup().getMapper("field"); assertThat(fieldMapper.getOffsetFieldName(), nullValue()); @@ -183,6 +125,12 @@ public void testOffsetArrayRandom() throws Exception { verifyOffsets("{\"field\":" + values + "}"); } + protected void minimalMapping(XContentBuilder b) throws IOException { + String fieldTypeName = getFieldTypeName(); + assertThat(fieldTypeName, notNullValue()); + b.field("type", fieldTypeName); + } + protected abstract String getFieldTypeName(); protected abstract Object randomValue(); @@ -192,21 +140,13 @@ protected void verifyOffsets(String source) throws IOException { } protected void verifyOffsets(String source, String expectedSource) throws IOException { - String mapping = """ - { - "_doc": { - "properties": { - "field": { - "type": "{{type}}" - } - } - } - } - """.replace("{{type}}", getFieldTypeName()); + XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties").startObject("field"); + minimalMapping(mapping); + mapping.endObject().endObject().endObject().endObject(); verifyOffsets(mapping, source, expectedSource); } - private void verifyOffsets(String mapping, String source, String expectedSource) throws IOException { + private void verifyOffsets(XContentBuilder mapping, String source, String expectedSource) throws IOException { try (var mapperService = createMapperService(mapping)) { var mapper = mapperService.documentMapper(); From 11ed0ea18737991cbe5b39105f9fd0d8c9eb2e9d Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Thu, 27 Mar 2025 08:30:26 -0700 Subject: [PATCH 3/6] Add offset tests for scaled float --- ...ScaledFloatOffsetDocValuesLoaderTests.java | 47 ++++++++++++++++++ ...eticSourceNativeArrayIntegrationTests.java | 48 +++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatOffsetDocValuesLoaderTests.java create mode 100644 modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatSyntheticSourceNativeArrayIntegrationTests.java diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatOffsetDocValuesLoaderTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatOffsetDocValuesLoaderTests.java new file mode 100644 index 0000000000000..bcbb5c2ec334c --- /dev/null +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatOffsetDocValuesLoaderTests.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.extras; + +import org.elasticsearch.index.mapper.OffsetDocValuesLoaderTestCase; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collection; + +import static java.util.Collections.singletonList; + +public class ScaledFloatOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase { + + @Override + protected Collection getPlugins() { + return singletonList(new MapperExtrasPlugin()); + } + + @Override + protected void minimalMapping(XContentBuilder b) throws IOException { + b.field("type", "scaled_float").field("scaling_factor", 10.0); + } + + public void testOffsetArray() throws Exception { + verifyOffsets("{\"field\":[1.0,10.0,100.0,0.1,10.0,1.0,0.1,100.0]}"); + verifyOffsets("{\"field\":[10.0,null,1.0,null,5.0,null,null,6.3,1.5]}"); + } + + @Override + protected String getFieldTypeName() { + return null; + } + + @Override + protected Object randomValue() { + return Math.round(randomDoubleBetween(Long.MIN_VALUE / 10.0, Long.MAX_VALUE / 10.0, true) * 10) / 10.0; + } +} diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatSyntheticSourceNativeArrayIntegrationTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatSyntheticSourceNativeArrayIntegrationTests.java new file mode 100644 index 0000000000000..9056afc860801 --- /dev/null +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatSyntheticSourceNativeArrayIntegrationTests.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.extras; + +import com.carrotsearch.randomizedtesting.generators.RandomStrings; + +import org.elasticsearch.index.mapper.NativeArrayIntegrationTestCase; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collection; + +import static java.util.Collections.singletonList; + +public class ScaledFloatSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase { + @Override + protected Collection> getPlugins() { + return singletonList(MapperExtrasPlugin.class); + } + + @Override + protected void minimalMapping(XContentBuilder b) throws IOException { + b.field("type", "scaled_float").field("scaling_factor", 10.0); + } + + @Override + protected String getFieldTypeName() { + return null; + } + + @Override + protected Object getRandomValue() { + return Math.round(randomDoubleBetween(Long.MIN_VALUE / 10.0, Long.MAX_VALUE / 10.0, true) * 10) / 10.0; + } + + @Override + protected Object getMalformedValue() { + return randomBoolean() ? RandomStrings.randomAsciiOfLength(random(), 8) : Double.POSITIVE_INFINITY; + } +} From b98a8292db862757cfb5928540a155d2a9fcdab2 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Thu, 27 Mar 2025 08:31:11 -0700 Subject: [PATCH 4/6] Fix ScaledFloatFieldMapperTests#testSyntheticSourceKeepArrays --- .../extras/ScaledFloatFieldMapperTests.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java index 1f1430d70c690..15be6b08faddc 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapperTests; @@ -365,6 +366,24 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) return new ScaledFloatSyntheticSourceSupport(ignoreMalformed); } + @Override + protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode sourceKeepMode) { + return new ScaledFloatSyntheticSourceSupport(ignoreMalformed) { + @Override + public SyntheticSourceExample example(int maxVals) { + var example = super.example(maxVals); + // Need the expectedForSyntheticSource as inputValue since MapperTestCase#testSyntheticSourceKeepArrays + // uses the inputValue as both the input and expected. + return new SyntheticSourceExample( + example.expectedForSyntheticSource(), + example.expectedForSyntheticSource(), + example.expectedForBlockLoader(), + example.mapping() + ); + } + }; + } + private static class ScaledFloatSyntheticSourceSupport implements SyntheticSourceSupport { private final boolean ignoreMalformedEnabled; private final double scalingFactor = randomDoubleBetween(0, Double.MAX_VALUE, false); From 0546fc92c436b4a64e5fc6996f5ee7783d340ab1 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Thu, 27 Mar 2025 09:19:25 -0700 Subject: [PATCH 5/6] Simplify scaled float getRandomValue in offset encoding tests --- .../mapper/extras/ScaledFloatOffsetDocValuesLoaderTests.java | 4 ++-- ...ScaledFloatSyntheticSourceNativeArrayIntegrationTests.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatOffsetDocValuesLoaderTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatOffsetDocValuesLoaderTests.java index bcbb5c2ec334c..7b8a6640145cf 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatOffsetDocValuesLoaderTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatOffsetDocValuesLoaderTests.java @@ -41,7 +41,7 @@ protected String getFieldTypeName() { } @Override - protected Object randomValue() { - return Math.round(randomDoubleBetween(Long.MIN_VALUE / 10.0, Long.MAX_VALUE / 10.0, true) * 10) / 10.0; + protected Double randomValue() { + return randomLong() / 10.0; } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatSyntheticSourceNativeArrayIntegrationTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatSyntheticSourceNativeArrayIntegrationTests.java index 9056afc860801..ad5772946bc12 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatSyntheticSourceNativeArrayIntegrationTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatSyntheticSourceNativeArrayIntegrationTests.java @@ -38,7 +38,7 @@ protected String getFieldTypeName() { @Override protected Object getRandomValue() { - return Math.round(randomDoubleBetween(Long.MIN_VALUE / 10.0, Long.MAX_VALUE / 10.0, true) * 10) / 10.0; + return randomLong() / 10.0; } @Override From 5f25c3b30f7ed878819103f9774ec22eb58b3c29 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Fri, 28 Mar 2025 11:12:55 -0700 Subject: [PATCH 6/6] Address review comments --- .../extras/ScaledFloatOffsetDocValuesLoaderTests.java | 6 ++++-- ...ledFloatSyntheticSourceNativeArrayIntegrationTests.java | 7 +++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatOffsetDocValuesLoaderTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatOffsetDocValuesLoaderTests.java index 7b8a6640145cf..6ab3996f788c4 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatOffsetDocValuesLoaderTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatOffsetDocValuesLoaderTests.java @@ -19,6 +19,7 @@ import static java.util.Collections.singletonList; public class ScaledFloatOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase { + private static final double TEST_SCALING_FACTOR = 10.0; @Override protected Collection getPlugins() { @@ -27,7 +28,7 @@ protected Collection getPlugins() { @Override protected void minimalMapping(XContentBuilder b) throws IOException { - b.field("type", "scaled_float").field("scaling_factor", 10.0); + b.field("type", "scaled_float").field("scaling_factor", TEST_SCALING_FACTOR); } public void testOffsetArray() throws Exception { @@ -37,11 +38,12 @@ public void testOffsetArray() throws Exception { @Override protected String getFieldTypeName() { + fail("Should not be called because minimalMapping is overridden"); return null; } @Override protected Double randomValue() { - return randomLong() / 10.0; + return randomLong() / TEST_SCALING_FACTOR; } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatSyntheticSourceNativeArrayIntegrationTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatSyntheticSourceNativeArrayIntegrationTests.java index ad5772946bc12..9aab22d3e1f07 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatSyntheticSourceNativeArrayIntegrationTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatSyntheticSourceNativeArrayIntegrationTests.java @@ -21,6 +21,8 @@ import static java.util.Collections.singletonList; public class ScaledFloatSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase { + private static final double TEST_SCALING_FACTOR = 10.0; + @Override protected Collection> getPlugins() { return singletonList(MapperExtrasPlugin.class); @@ -28,17 +30,18 @@ protected Collection> getPlugins() { @Override protected void minimalMapping(XContentBuilder b) throws IOException { - b.field("type", "scaled_float").field("scaling_factor", 10.0); + b.field("type", "scaled_float").field("scaling_factor", TEST_SCALING_FACTOR); } @Override protected String getFieldTypeName() { + fail("Should not be called because minimalMapping is overridden"); return null; } @Override protected Object getRandomValue() { - return randomLong() / 10.0; + return randomLong() / TEST_SCALING_FACTOR; } @Override