[8.x] Natively store synthetic source array offsets for numeric fields (#124594) | Fix ignores malformed testcase (#125337) | Fix offsets not recording duplicate values (#125354) (#125440)

* Natively store synthetic source array offsets for numeric fields (#124594)

This patch builds on the work in #122999 and #113757 to natively store
array offsets for numeric fields instead of falling back to ignored source
when `source_keep_mode: arrays`.

(cherry picked from commit 376abfe)

# Conflicts:
#	server/src/main/java/org/elasticsearch/index/IndexVersions.java
#	server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java

* Fix ignores malformed testcase (#125337)

Fix and unmute testSynthesizeArrayRandomIgnoresMalformed

(cherry picked from commit 2ff03ac)

# Conflicts:
#	muted-tests.yml

* Fix offsets not recording duplicate values (#125354)

Previously, when calculating the offsets, we just compared the values as-is
without any loss of precision. However, when the values were saved into doc
values and loaded in the doc values loader, they could have lost precision.
This meant that values that were not duplicates when calculating the
offsets could now be duplicates in the doc values loader. This interfered
with the de-duplication logic, causing incorrect values to be returned.

My solution is to apply the precision loss before calculating the offsets,
so that both the offsets calculation and the SortedNumericDocValues
de-duplication see the same values as duplicates.

(cherry picked from commit db73175)

Author: jordan-powers

docs/changelog/124594.yaml

@@ -0,0 +1,5 @@
+pr: 124594
+summary: Store arrays offsets for numeric fields natively with synthetic source
+area: Mapping
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -126,6 +126,7 @@ private static IndexVersion def(int id, Version luceneVersion) {
     public static final IndexVersion LOGSB_OPTIONAL_SORTING_ON_HOST_NAME_BACKPORT = def(8_525_0_00, Version.LUCENE_9_12_1);
     public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT_BACKPORT = def(8_526_0_00, Version.LUCENE_9_12_1);
     public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD = def(8_527_0_00, Version.LUCENE_9_12_1);
+    public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_NUMBER = def(8_528_0_00, Version.LUCENE_9_12_1);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java

@@ -352,7 +352,8 @@ public boolean newDynamicLongField(DocumentParserContext context, String name) t
                     ScriptCompiler.NONE,
                     context.indexSettings().getSettings(),
                     context.indexSettings().getIndexVersionCreated(),
-                    context.indexSettings().getMode()
+                    context.indexSettings().getMode(),
+                    context.indexSettings().sourceKeepMode()
                 ),
                 context
             );
@@ -370,7 +371,8 @@ public boolean newDynamicDoubleField(DocumentParserContext context, String name)
                     ScriptCompiler.NONE,
                     context.indexSettings().getSettings(),
                     context.indexSettings().getIndexVersionCreated(),
-                    context.indexSettings().getMode()
+                    context.indexSettings().getMode(),
+                    context.indexSettings().sourceKeepMode()
                 ),
                 context
             );

server/src/main/java/org/elasticsearch/index/mapper/FieldArrayContext.java

@@ -14,7 +14,6 @@
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.index.IndexVersion;
-import org.elasticsearch.index.IndexVersions;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -89,16 +88,18 @@ static String getOffsetsFieldName(
         boolean hasDocValues,
         boolean isStored,
         FieldMapper.Builder fieldMapperBuilder,
-        IndexVersion indexCreatedVersion
+        IndexVersion indexCreatedVersion,
+        IndexVersion minSupportedVersion
     ) {
         var sourceKeepMode = fieldMapperBuilder.sourceKeepMode.orElse(indexSourceKeepMode);
         if (context.isSourceSynthetic()
             && sourceKeepMode == Mapper.SourceKeepMode.ARRAYS
             && hasDocValues
             && isStored == false
+            && context.isInNestedContext() == false
             && fieldMapperBuilder.copyTo.copyToFields().isEmpty()
             && fieldMapperBuilder.multiFieldsBuilder.hasMultiFields() == false
-            && indexCreatedVersion.onOrAfter(IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD)) {
+            && indexCreatedVersion.onOrAfter(minSupportedVersion)) {
             // Skip stored, we will be synthesizing from stored fields, no point to keep track of the offsets
             // Skip copy_to and multi fields, supporting that requires more work. However, copy_to usage is rare in metrics and
             // logging use cases

server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java

@@ -200,7 +200,8 @@ public IpFieldMapper build(MapperBuilderContext context) {
                 hasDocValues.getValue(),
                 stored.getValue(),
                 this,
-                indexCreatedVersion
+                indexCreatedVersion,
+                IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD
             );
             return new IpFieldMapper(
                 leafName(),

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -41,6 +41,7 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.FieldData;
@@ -389,7 +390,8 @@ public KeywordFieldMapper build(MapperBuilderContext context) {
                 hasDocValues.getValue(),
                 stored.getValue(),
                 this,
-                indexCreatedVersion
+                indexCreatedVersion,
+                IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD
             );
             return new KeywordFieldMapper(
                 leafName(),