[8.x] Natively store synthetic source array offsets for numeric fields (#124594) | Fix ignores malformed testcase (#125337) | Fix offsets not recording duplicate values (#125354)

docs/changelog/124594.yaml

@@ -0,0 +1,5 @@
+pr: 124594
+summary: Store arrays offsets for numeric fields natively with synthetic source
+area: Mapping
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -126,6 +126,7 @@ private static IndexVersion def(int id, Version luceneVersion) {
     public static final IndexVersion LOGSB_OPTIONAL_SORTING_ON_HOST_NAME_BACKPORT = def(8_525_0_00, Version.LUCENE_9_12_1);
     public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT_BACKPORT = def(8_526_0_00, Version.LUCENE_9_12_1);
     public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD = def(8_527_0_00, Version.LUCENE_9_12_1);
+    public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_NUMBER = def(8_528_0_00, Version.LUCENE_9_12_1);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java

@@ -352,7 +352,8 @@ public boolean newDynamicLongField(DocumentParserContext context, String name) t
                     ScriptCompiler.NONE,
                     context.indexSettings().getSettings(),
                     context.indexSettings().getIndexVersionCreated(),
-                    context.indexSettings().getMode()
+                    context.indexSettings().getMode(),
+                    context.indexSettings().sourceKeepMode()
                 ),
                 context
             );
@@ -370,7 +371,8 @@ public boolean newDynamicDoubleField(DocumentParserContext context, String name)
                     ScriptCompiler.NONE,
                     context.indexSettings().getSettings(),
                     context.indexSettings().getIndexVersionCreated(),
-                    context.indexSettings().getMode()
+                    context.indexSettings().getMode(),
+                    context.indexSettings().sourceKeepMode()
                 ),
                 context
             );

server/src/main/java/org/elasticsearch/index/mapper/FieldArrayContext.java

@@ -14,7 +14,6 @@
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.index.IndexVersion;
-import org.elasticsearch.index.IndexVersions;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -89,16 +88,18 @@ static String getOffsetsFieldName(
         boolean hasDocValues,
         boolean isStored,
         FieldMapper.Builder fieldMapperBuilder,
-        IndexVersion indexCreatedVersion
+        IndexVersion indexCreatedVersion,
+        IndexVersion minSupportedVersion
     ) {
         var sourceKeepMode = fieldMapperBuilder.sourceKeepMode.orElse(indexSourceKeepMode);
         if (context.isSourceSynthetic()
             && sourceKeepMode == Mapper.SourceKeepMode.ARRAYS
             && hasDocValues
             && isStored == false
+            && context.isInNestedContext() == false
             && fieldMapperBuilder.copyTo.copyToFields().isEmpty()
             && fieldMapperBuilder.multiFieldsBuilder.hasMultiFields() == false
-            && indexCreatedVersion.onOrAfter(IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD)) {
+            && indexCreatedVersion.onOrAfter(minSupportedVersion)) {
             // Skip stored, we will be synthesizing from stored fields, no point to keep track of the offsets
             // Skip copy_to and multi fields, supporting that requires more work. However, copy_to usage is rare in metrics and
             // logging use cases

server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java

@@ -200,7 +200,8 @@ public IpFieldMapper build(MapperBuilderContext context) {
                 hasDocValues.getValue(),
                 stored.getValue(),
                 this,
-                indexCreatedVersion
+                indexCreatedVersion,
+                IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD
             );
             return new IpFieldMapper(
                 leafName(),

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -41,6 +41,7 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.FieldData;
@@ -389,7 +390,8 @@ public KeywordFieldMapper build(MapperBuilderContext context) {
                 hasDocValues.getValue(),
                 stored.getValue(),
                 this,
-                indexCreatedVersion
+                indexCreatedVersion,
+                IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD
             );
             return new KeywordFieldMapper(
                 leafName(),