Merge branch 'main' into exclusive2

Author: jdconrad

docs/changelog/123403.yaml

@@ -0,0 +1,5 @@
+pr: 123403
+summary: Use ordered maps for `PipelineConfiguration` xcontent deserialization
+area: Ingest Node
+type: bug
+issues: []

muted-tests.yml

@@ -340,6 +340,15 @@ tests:
 - class: org.elasticsearch.action.admin.indices.diskusage.IndexDiskUsageAnalyzerTests
   method: testCompletionField
   issue: https://github.com/elastic/elasticsearch/issues/123269
+- class: org.elasticsearch.index.mapper.IPSyntheticSourceNativeArrayIntegrationTests
+  method: testSynthesizeArray
+  issue: https://github.com/elastic/elasticsearch/issues/123417
+- class: org.elasticsearch.index.mapper.IPSyntheticSourceNativeArrayIntegrationTests
+  method: testSynthesizeArrayRandom
+  issue: https://github.com/elastic/elasticsearch/issues/123418
+- class: org.elasticsearch.index.mapper.IPSyntheticSourceNativeArrayIntegrationTests
+  method: testSynthesizeArrayIgnoreMalformed
+  issue: https://github.com/elastic/elasticsearch/issues/123419
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java

@@ -19,7 +19,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
@@ -373,8 +372,6 @@ public static class DataStreamTemplate implements Writeable, ToXContentObject {
 
         private static final ParseField HIDDEN = new ParseField("hidden");
         private static final ParseField ALLOW_CUSTOM_ROUTING = new ParseField("allow_custom_routing");
-        // Remove this after this PR gets backported
-        @UpdateForV9(owner = UpdateForV9.Owner.DATA_MANAGEMENT)
         private static final ParseField FAILURE_STORE = new ParseField("failure_store");
 
         public static final ConstructingObjectParser<DataStreamTemplate, Void> PARSER = new ConstructingObjectParser<>(

server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java

@@ -46,7 +46,7 @@ public final class PipelineConfiguration implements SimpleDiffable<PipelineConfi
     static {
         PARSER.declareString(Builder::setId, new ParseField("id"));
         PARSER.declareField(
-            (parser, builder, aVoid) -> builder.setConfig(parser.map()),
+            (parser, builder, aVoid) -> builder.setConfig(parser.mapOrdered()),
             new ParseField("config"),
             ObjectParser.ValueType.OBJECT
         );

server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java

@@ -28,9 +28,11 @@
 import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Predicate;
 
 import static org.hamcrest.Matchers.anEmptyMap;
+import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.sameInstance;
@@ -143,6 +145,41 @@ public void testGetVersion() {
         }
     }
 
+    @SuppressWarnings("unchecked")
+    public void testMapKeyOrderingRoundTrip() throws IOException {
+        // make up two random keys
+        String key1 = randomAlphaOfLength(10);
+        String key2 = randomValueOtherThan(key1, () -> randomAlphaOfLength(10));
+        // stick them as mappings onto themselves in the _meta of a pipeline configuration
+        // this happens to use the _meta as a convenient map to test that the ordering of the key sets is the same
+        String configJson = Strings.format("""
+            {"description": "blah", "_meta" : {"foo": "bar", "%s": "%s", "%s": "%s"}}""", key1, key1, key2, key2);
+        PipelineConfiguration configuration = new PipelineConfiguration(
+            "1",
+            new BytesArray(configJson.getBytes(StandardCharsets.UTF_8)),
+            XContentType.JSON
+        );
+
+        // serialize it to bytes
+        XContentType xContentType = randomFrom(XContentType.values());
+        final BytesReference bytes;
+        try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
+            configuration.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            bytes = BytesReference.bytes(builder);
+        }
+
+        // deserialize it back
+        ContextParser<Void, PipelineConfiguration> parser = PipelineConfiguration.getParser();
+        XContentParser xContentParser = xContentType.xContent()
+            .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, bytes.streamInput());
+        PipelineConfiguration parsed = parser.parse(xContentParser, null);
+
+        // make sure the _meta key sets are in the same order
+        Set<String> keys1 = ((Map<String, Object>) configuration.getConfig().get("_meta")).keySet();
+        Set<String> keys2 = ((Map<String, Object>) parsed.getConfig().get("_meta")).keySet();
+        assertThat(keys1, contains(keys2.toArray(new String[0])));
+    }
+
     @Override
     protected PipelineConfiguration createTestInstance() {
         BytesArray config;

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -64,7 +64,7 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.core.UpdateForV9;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.health.node.selection.HealthNode;
 import org.elasticsearch.index.IndexSettings;
@@ -1993,11 +1993,12 @@ protected static boolean indexExists(RestClient client, String index) throws IOE
     }
 
     /**
-     * Deprecation message emitted since 7.12.0 for the rest of the 7.x series. Can be removed in v9 since it is not
-     * emitted in v8. Note that this message is also permitted in certain YAML test cases, it can be removed there too.
-     * See https://github.com/elastic/elasticsearch/issues/66419 for more details.
+     * Deprecation message emitted since 7.12.0 for the rest of the 7.x series. Can be removed in v10 since it is not
+     * emitted in v8 or v9 and N-2 versions are now supported.
+     * Note that this message is also permitted in certain YAML test cases, it can be removed there too.
+     * See https://github.com/elastic/elasticsearch/issues/66419 and https://github.com/elastic/elasticsearch/pull/119594 for more details.
      */
-    @UpdateForV9(owner = UpdateForV9.Owner.DATA_MANAGEMENT)
+    @UpdateForV10(owner = UpdateForV10.Owner.DISTRIBUTED_COORDINATION)
     private static final String WAIT_FOR_ACTIVE_SHARDS_DEFAULT_DEPRECATION_MESSAGE = "the default value for the ?wait_for_active_shards "
         + "parameter will change from '0' to 'index-setting' in version 8; specify '?wait_for_active_shards=index-setting' "
         + "to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour";