Remove obsolete Metadata BWC for repositories (elastic#129685)

ywangd · web-flow · commit 0e83b19de786 · 2025-06-20T11:34:48.000+10:00
When migrating RepositoriesMetadata from cluster custom to project custom (elastic#125398), we needed temporary BWC handling for clusters running on a version that is before this change but after the initial MP change. Such a cluster can only exist in the serverless environment which has progressed way past any applicable versions. Therefore we no longer need the BWC handling and this PR removes it. Relates: elastic#125398
diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -244,7 +244,6 @@ static TransportVersion def(int id) {
     public static final TransportVersion RESCORE_VECTOR_ALLOW_ZERO = def(9_039_0_00);
     public static final TransportVersion PROJECT_ID_IN_SNAPSHOT = def(9_040_0_00);
     public static final TransportVersion INDEX_STATS_AND_METADATA_INCLUDE_PEAK_WRITE_LOAD = def(9_041_0_00);
-    public static final TransportVersion REPOSITORIES_METADATA_AS_PROJECT_CUSTOM = def(9_042_0_00);
     public static final TransportVersion BATCHED_QUERY_PHASE_VERSION = def(9_043_0_00);
     public static final TransportVersion REMOTE_EXCEPTION = def(9_044_0_00);
     public static final TransportVersion ESQL_REMOVE_AGGREGATE_TYPE = def(9_045_0_00);
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java
@@ -11,7 +11,6 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.Diff;
@@ -53,7 +52,6 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -997,12 +995,11 @@ private MetadataDiff(StreamInput in) throws IOException {
                 multiProject = null;
             } else {
                 fromNodeBeforeMultiProjectsSupport = false;
-                // Repositories metadata is sent as Metadata#customs diff from old node. We need to
-                // 1. Split it from the Metadata#customs diff
-                // 2. Merge it into the default project's ProjectMetadataDiff
-                final var bwcCustoms = maybeReadBwcCustoms(in);
-                clusterCustoms = bwcCustoms.v1();
-                final var defaultProjectCustoms = bwcCustoms.v2();
+                clusterCustoms = DiffableUtils.readImmutableOpenMapDiff(
+                    in,
+                    DiffableUtils.getStringKeySerializer(),
+                    CLUSTER_CUSTOM_VALUE_SERIALIZER
+                );
 
                 reservedStateMetadata = DiffableUtils.readImmutableOpenMapDiff(
                     in,
@@ -1011,61 +1008,15 @@ private MetadataDiff(StreamInput in) throws IOException {
                 );
 
                 singleProject = null;
-                multiProject = readMultiProjectDiffs(in, defaultProjectCustoms);
-            }
-        }
-
-        private static
-            Tuple<
-                MapDiff<String, ClusterCustom, ImmutableOpenMap<String, ClusterCustom>>,
-                MapDiff<String, ProjectCustom, ImmutableOpenMap<String, ProjectCustom>>>
-            maybeReadBwcCustoms(StreamInput in) throws IOException {
-            if (in.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM)) {
-                return readBwcCustoms(in);
-            } else {
-                return new Tuple<>(
-                    DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), CLUSTER_CUSTOM_VALUE_SERIALIZER),
-                    null
+                multiProject = DiffableUtils.readJdkMapDiff(
+                    in,
+                    PROJECT_ID_SERIALIZER,
+                    ProjectMetadata::readFrom,
+                    ProjectMetadata.ProjectMetadataDiff::new
                 );
             }
         }
 
-        @SuppressWarnings("unchecked")
-        private static MapDiff<ProjectId, ProjectMetadata, Map<ProjectId, ProjectMetadata>> readMultiProjectDiffs(
-            StreamInput in,
-            MapDiff<String, ProjectCustom, ImmutableOpenMap<String, ProjectCustom>> defaultProjectCustoms
-        ) throws IOException {
-            final var multiProject = DiffableUtils.readJdkMapDiff(
-                in,
-                PROJECT_ID_SERIALIZER,
-                ProjectMetadata::readFrom,
-                ProjectMetadata.ProjectMetadataDiff::new
-            );
-
-            // If the defaultProjectCustoms has content, the diff is read from an old node. We need to merge it into the
-            // default project's ProjectMetadataDiff
-            if (defaultProjectCustoms != null && defaultProjectCustoms.isEmpty() == false) {
-                return DiffableUtils.updateDiffsAndUpserts(multiProject, ProjectId.DEFAULT::equals, (k, v) -> {
-                    assert ProjectId.DEFAULT.equals(k) : k;
-                    assert v instanceof ProjectMetadata.ProjectMetadataDiff : v;
-                    final var projectMetadataDiff = (ProjectMetadata.ProjectMetadataDiff) v;
-                    return projectMetadataDiff.withCustoms(
-                        DiffableUtils.merge(
-                            projectMetadataDiff.customs(),
-                            defaultProjectCustoms,
-                            DiffableUtils.getStringKeySerializer(),
-                            BWC_CUSTOM_VALUE_SERIALIZER
-                        )
-                    );
-                }, (k, v) -> {
-                    assert ProjectId.DEFAULT.equals(k) : k;
-                    return ProjectMetadata.builder(v).clearCustoms().customs(defaultProjectCustoms.apply(v.customs())).build();
-                });
-            } else {
-                return multiProject;
-            }
-        }
-
         @SuppressWarnings("unchecked")
         private static
             Tuple<
@@ -1109,105 +1060,16 @@ public void writeTo(StreamOutput out) throws IOException {
                 buildUnifiedCustomDiff().writeTo(out);
                 buildUnifiedReservedStateMetadataDiff().writeTo(out);
             } else {
-                final var multiProjectToWrite = multiProject != null
-                    ? multiProject
-                    : DiffableUtils.singleEntryDiff(DEFAULT_PROJECT_ID, singleProject, PROJECT_ID_SERIALIZER);
-
-                if (out.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM)) {
-                    writeDiffWithRepositoriesMetadataAsClusterCustom(out, clusterCustoms, multiProjectToWrite, reservedStateMetadata);
+                clusterCustoms.writeTo(out);
+                reservedStateMetadata.writeTo(out);
+                if (multiProject != null) {
+                    multiProject.writeTo(out);
                 } else {
-                    clusterCustoms.writeTo(out);
-                    reservedStateMetadata.writeTo(out);
-                    multiProjectToWrite.writeTo(out);
+                    DiffableUtils.singleEntryDiff(DEFAULT_PROJECT_ID, singleProject, PROJECT_ID_SERIALIZER).writeTo(out);
                 }
             }
         }
 
-        @SuppressWarnings({ "rawtypes", "unchecked" })
-        private static void writeDiffWithRepositoriesMetadataAsClusterCustom(
-            StreamOutput out,
-            MapDiff<String, ClusterCustom, ImmutableOpenMap<String, ClusterCustom>> clusterCustoms,
-            MapDiff<ProjectId, ProjectMetadata, Map<ProjectId, ProjectMetadata>> multiProject,
-            MapDiff<String, ReservedStateMetadata, ImmutableOpenMap<String, ReservedStateMetadata>> reservedStateMetadata
-        ) throws IOException {
-            assert out.getTransportVersion().onOrAfter(TransportVersions.MULTI_PROJECT)
-                && out.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM) : out.getTransportVersion();
-
-            // For old nodes, RepositoriesMetadata needs to be sent as a cluster custom. This is possible when (a) the repositories
-            // are defined only for the default project or (b) no repositories at all. What we need to do are:
-            // 1. Iterate through the multi-project's MapDiff to extract the RepositoriesMetadata of the default project
-            // 2. Throws if any repositories are found for non-default projects
-            // 3. Merge default project's RepositoriesMetadata into Metadata#customs
-            final var combineClustersCustoms = new SetOnce<MapDiff<String, MetadataCustom, Map<String, MetadataCustom>>>();
-            final var updatedMultiProject = DiffableUtils.updateDiffsAndUpserts(multiProject, ignore -> true, (k, v) -> {
-                assert v instanceof ProjectMetadata.ProjectMetadataDiff : v;
-                final var projectMetadataDiff = (ProjectMetadata.ProjectMetadataDiff) v;
-                final var bwcCustoms = DiffableUtils.split(
-                    projectMetadataDiff.customs(),
-                    RepositoriesMetadata.TYPE::equals,
-                    PROJECT_CUSTOM_VALUE_SERIALIZER,
-                    type -> RepositoriesMetadata.TYPE.equals(type) == false,
-                    PROJECT_CUSTOM_VALUE_SERIALIZER
-                );
-                // Simply return if RepositoriesMetadata is not found
-                if (bwcCustoms.v1().isEmpty()) {
-                    return projectMetadataDiff;
-                }
-                // RepositoriesMetadata can only be defined for the default project. Otherwise throw exception.
-                if (ProjectId.DEFAULT.equals(k) == false) {
-                    throwForVersionBeforeRepositoriesMetadataMigration(out);
-                }
-                // RepositoriesMetadata is found for the default project as a diff, merge it into the Metadata#customs
-                combineClustersCustoms.set(
-                    DiffableUtils.<String, MetadataCustom, ClusterCustom, ProjectCustom, Map<String, MetadataCustom>>merge(
-                        clusterCustoms,
-                        bwcCustoms.v1(),
-                        DiffableUtils.getStringKeySerializer()
-                    )
-                );
-                return projectMetadataDiff.withCustoms(bwcCustoms.v2());
-            }, (k, v) -> {
-                final ProjectCustom projectCustom = v.customs().get(RepositoriesMetadata.TYPE);
-                // Simply return if RepositoriesMetadata is not found
-                if (projectCustom == null) {
-                    return v;
-                }
-                // RepositoriesMetadata can only be defined for the default project. Otherwise throw exception.
-                if (ProjectId.DEFAULT.equals(k) == false) {
-                    throwForVersionBeforeRepositoriesMetadataMigration(out);
-                }
-                // RepositoriesMetadata found for the default project as an upsert, package it as MapDiff and merge into Metadata#customs
-                combineClustersCustoms.set(
-                    DiffableUtils.<String, MetadataCustom, ClusterCustom, ProjectCustom, Map<String, MetadataCustom>>merge(
-                        clusterCustoms,
-                        DiffableUtils.singleUpsertDiff(RepositoriesMetadata.TYPE, projectCustom, DiffableUtils.getStringKeySerializer()),
-                        DiffableUtils.getStringKeySerializer()
-                    )
-                );
-                return ProjectMetadata.builder(v).removeCustom(RepositoriesMetadata.TYPE).build();
-            });
-
-            if (combineClustersCustoms.get() != null) {
-                combineClustersCustoms.get().writeTo(out);
-            } else {
-                clusterCustoms.writeTo(out);
-            }
-
-            reservedStateMetadata.writeTo(out);
-            updatedMultiProject.writeTo(out);
-        }
-
-        private static void throwForVersionBeforeRepositoriesMetadataMigration(StreamOutput out) {
-            assert out.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM) : out.getTransportVersion();
-            throw new UnsupportedOperationException(
-                "Serialize a diff with repositories defined for multiple projects requires version on or after ["
-                    + TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM
-                    + "], but got ["
-                    + out.getTransportVersion()
-                    + "]"
-            );
-        }
-
         @SuppressWarnings("unchecked")
         private Diff<ImmutableOpenMap<String, ?>> buildUnifiedCustomDiff() {
             assert multiProject == null : "should only be used for single project metadata";
@@ -1361,34 +1223,19 @@ public static Metadata readFrom(StreamInput in) throws IOException {
                 builder.put(ReservedStateMetadata.readFrom(in));
             }
         } else {
-            List<ProjectCustom> defaultProjectCustoms = List.of();
-            if (in.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM)) {
-                // Extract the default project's repositories metadata from the Metadata#customs from an old node
-                defaultProjectCustoms = new ArrayList<>();
-                readBwcCustoms(in, builder, defaultProjectCustoms::add);
-                assert defaultProjectCustoms.size() <= 1
-                    : "expect only a single default project custom for repository metadata, but got "
-                        + defaultProjectCustoms.stream().map(ProjectCustom::getWriteableName).toList();
-            } else {
-                readClusterCustoms(in, builder);
-            }
+            readClusterCustoms(in, builder);
 
             int reservedStateSize = in.readVInt();
             for (int i = 0; i < reservedStateSize; i++) {
                 builder.put(ReservedStateMetadata.readFrom(in));
             }
 
             builder.projectMetadata(in.readMap(ProjectId::readFrom, ProjectMetadata::readFrom));
-            defaultProjectCustoms.forEach(c -> builder.getProject(ProjectId.DEFAULT).putCustom(c.getWriteableName(), c));
         }
         return builder.build();
     }
 
     private static void readBwcCustoms(StreamInput in, Builder builder) throws IOException {
-        readBwcCustoms(in, builder, projectCustom -> builder.putProjectCustom(projectCustom.getWriteableName(), projectCustom));
-    }
-
-    private static void readBwcCustoms(StreamInput in, Builder builder, Consumer<ProjectCustom> projectCustomConsumer) throws IOException {
         final Set<String> clusterScopedNames = in.namedWriteableRegistry().getReaders(ClusterCustom.class).keySet();
         final Set<String> projectScopedNames = in.namedWriteableRegistry().getReaders(ProjectCustom.class).keySet();
         final int count = in.readVInt();
@@ -1404,9 +1251,9 @@ private static void readBwcCustoms(StreamInput in, Builder builder, Consumer<Pro
                 if (custom instanceof PersistentTasksCustomMetadata persistentTasksCustomMetadata) {
                     final var tuple = persistentTasksCustomMetadata.split();
                     builder.putCustom(tuple.v1().getWriteableName(), tuple.v1());
-                    projectCustomConsumer.accept(tuple.v2());
+                    builder.putProjectCustom(tuple.v2().getWriteableName(), tuple.v2());
                 } else {
-                    projectCustomConsumer.accept(custom);
+                    builder.putProjectCustom(custom.getWriteableName(), custom);
                 }
             } else {
                 throw new IllegalArgumentException("Unknown custom name [" + name + "]");
@@ -1473,42 +1320,13 @@ public void writeTo(StreamOutput out) throws IOException {
             combinedMetadata.addAll(singleProject.reservedStateMetadata().values());
             out.writeCollection(combinedMetadata);
         } else {
-            if (out.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM)) {
-                if (isSingleProject() || hasNoNonDefaultProjectRepositories(projects().values())) {
-                    // Repositories metadata must be sent as Metadata#customs for old nodes
-                    final List<VersionedNamedWriteable> combinedCustoms = new ArrayList<>(customs.size() + 1);
-                    combinedCustoms.addAll(customs.values());
-                    final ProjectCustom custom = getProject(ProjectId.DEFAULT).custom(RepositoriesMetadata.TYPE);
-                    if (custom != null) {
-                        combinedCustoms.add(custom);
-                    }
-                    VersionedNamedWriteable.writeVersionedWriteables(out, combinedCustoms);
-                } else {
-                    throw new UnsupportedOperationException(
-                        "Serialize metadata with repositories defined for multiple projects requires version on or after ["
-                            + TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM
-                            + "], but got ["
-                            + out.getTransportVersion()
-                            + "]"
-                    );
-                }
-            } else {
-                VersionedNamedWriteable.writeVersionedWriteables(out, customs.values());
-            }
+            VersionedNamedWriteable.writeVersionedWriteables(out, customs.values());
 
             out.writeCollection(reservedStateMetadata.values());
             out.writeMap(projectMetadata, StreamOutput::writeWriteable, StreamOutput::writeWriteable);
         }
     }
 
-    /**
-     * @return {@code true} iff no repositories are defined for non-default-projects.
-     */
-    private static boolean hasNoNonDefaultProjectRepositories(Collection<ProjectMetadata> projects) {
-        return projects.stream()
-            .allMatch(project -> ProjectId.DEFAULT.equals(project.id()) || project.custom(RepositoriesMetadata.TYPE) == null);
-    }
-
     public static Builder builder() {
         return new Builder();
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java
@@ -1530,11 +1530,6 @@ public Builder removeCustomIf(BiPredicate<String, ? super Metadata.ProjectCustom
             return this;
         }
 
-        public Builder clearCustoms() {
-            customs.clear();
-            return this;
-        }
-
         public Builder customs(Map<String, Metadata.ProjectCustom> customs) {
             customs.forEach((key, value) -> Objects.requireNonNull(value, key));
             this.customs.putAllFromMap(customs);
@@ -2266,17 +2261,7 @@ public void writeTo(StreamOutput out) throws IOException {
             indexMetadata.writeTo(out, true);
         }
         out.writeCollection(templates.values());
-        Collection<Metadata.ProjectCustom> filteredCustoms = customs.values();
-        if (out.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM)) {
-            // RepositoriesMetadata is sent as part of Metadata#customs for version before RepositoriesMetadata migration
-            // So we exclude it from the project level customs
-            if (custom(RepositoriesMetadata.TYPE) != null) {
-                assert ProjectId.DEFAULT.equals(id)
-                    : "Only default project can have repositories metadata. Otherwise the code should have thrown before it reaches here";
-                filteredCustoms = filteredCustoms.stream().filter(custom -> custom instanceof RepositoriesMetadata == false).toList();
-            }
-        }
-        VersionedNamedWriteable.writeVersionedWriteables(out, filteredCustoms);
+        VersionedNamedWriteable.writeVersionedWriteables(out, customs.values());
         out.writeCollection(reservedStateMetadata.values());
 
         if (out.getTransportVersion().onOrAfter(TransportVersions.PROJECT_METADATA_SETTINGS)) {
@@ -2409,12 +2394,6 @@ public ProjectMetadata apply(ProjectMetadata part) {
             builder.settings = settingsDiff.apply(part.settings);
             return builder.build(true);
         }
-
-        ProjectMetadataDiff withCustoms(
-            DiffableUtils.MapDiff<String, Metadata.ProjectCustom, ImmutableOpenMap<String, Metadata.ProjectCustom>> customs
-        ) {
-            return new ProjectMetadataDiff(indices, templates, customs, reservedStateMetadata, settingsDiff);
-        }
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataRepositoriesMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataRepositoriesMetadataTests.java
