Remove obsolete Metadata.FORMAT field and usages

The only production usage is for cleaning up all global state files. It
is replaced by directly calling the relevant method without creating the
FORAMT instance. Test only usages are either replaced by equivalent
method calls or dropped.

Relates: #114698

Author: ywangd

server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java

@@ -40,7 +40,6 @@
 import org.elasticsearch.core.FixForMultiProject;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.gateway.MetadataStateFormat;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexVersion;
@@ -50,7 +49,6 @@
 import org.elasticsearch.xcontent.NamedObjectNotFoundException;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -2051,30 +2049,6 @@ static <C extends MetadataCustom<C>> void parseCustomObject(
         }
     }
 
-    private static final ToXContent.Params FORMAT_PARAMS;
-    static {
-        Map<String, String> params = Maps.newMapWithExpectedSize(2);
-        params.put("binary", "true");
-        params.put(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY);
-        FORMAT_PARAMS = new ToXContent.MapParams(params);
-    }
-
-    /**
-     * State format for {@link Metadata} to write to and load from disk
-     */
-    public static final MetadataStateFormat<Metadata> FORMAT = new MetadataStateFormat<>(GLOBAL_STATE_FILE_PREFIX) {
-
-        @Override
-        public void toXContent(XContentBuilder builder, Metadata state) throws IOException {
-            ChunkedToXContent.wrapAsToXContent(state).toXContent(builder, FORMAT_PARAMS);
-        }
-
-        @Override
-        public Metadata fromXContent(XContentParser parser) throws IOException {
-            return Builder.fromXContent(parser);
-        }
-    };
-
     private volatile Metadata.ProjectLookup projectLookup = null;
 
     /**

server/src/main/java/org/elasticsearch/gateway/MetaStateService.java

@@ -11,6 +11,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.store.NIOFSDirectory;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Manifest;
@@ -25,6 +26,8 @@
 import java.util.List;
 import java.util.function.Predicate;
 
+import static org.elasticsearch.cluster.metadata.Metadata.GLOBAL_STATE_FILE_PREFIX;
+
 /**
  * Handles writing and loading {@link Manifest}, {@link Metadata} and {@link IndexMetadata} as used for cluster state persistence in
  * versions prior to {@link Version#V_7_6_0}, used to read this older format during an upgrade from these versions.
@@ -75,20 +78,13 @@ List<IndexMetadata> loadIndicesStates(Predicate<String> excludeIndexPathIdsPredi
         return indexMetadataList;
     }
 
-    /**
-     * Loads the global state, *without* index state
-     */
-    Metadata loadGlobalState() throws IOException {
-        return Metadata.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths());
-    }
-
     /**
      * Creates empty cluster state file on disk, deleting global metadata and unreferencing all index metadata
      * (only used for dangling indices at that point).
      */
     public void unreferenceAll() throws IOException {
         Manifest.FORMAT.writeAndCleanup(Manifest.empty(), nodeEnv.nodeDataPaths()); // write empty file so that indices become unreferenced
-        Metadata.FORMAT.cleanupOldFiles(Long.MAX_VALUE, nodeEnv.nodeDataPaths());
+        MetadataStateFormat.cleanupOldFiles(GLOBAL_STATE_FILE_PREFIX, Long.MAX_VALUE, nodeEnv.nodeDataPaths(), NIOFSDirectory::new);
     }
 
     /**

server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.common.lucene.store.IndexOutputOutputStream;
 import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Tuple;
@@ -349,12 +350,29 @@ protected Directory newDirectory(Path dir) throws IOException {
      * @param currentGeneration state generation to keep.
      * @param locations         state paths.
      */
-    public void cleanupOldFiles(final long currentGeneration, Path[] locations) {
-        final String fileNameToKeep = getStateFileName(currentGeneration);
+    public void cleanupOldFiles(final long currentGeneration, final Path[] locations) {
+        cleanupOldFiles(prefix, currentGeneration, locations, this::newDirectory);
+    }
+
+    /**
+     * Clean ups all state files not matching passed generation.
+     *
+     * @param prefix filename prefix for filtering state files.
+     * @param currentGeneration state generation to keep.
+     * @param locations         state paths.
+     * @param directoryFunc function to create a {@link Directory} for the given path.
+     */
+    public static void cleanupOldFiles(
+        final String prefix,
+        final long currentGeneration,
+        final Path[] locations,
+        final CheckedFunction<Path, Directory, IOException> directoryFunc
+    ) {
+        final String fileNameToKeep = getStateFileName(prefix, currentGeneration);
         for (Path location : locations) {
             logger.trace("cleanupOldFiles: cleaning up {}", location);
             Path stateLocation = location.resolve(STATE_DIR_NAME);
-            try (Directory stateDir = newDirectory(stateLocation)) {
+            try (Directory stateDir = directoryFunc.apply(stateLocation)) {
                 for (String file : stateDir.listAll()) {
                     if (file.startsWith(prefix) && file.equals(fileNameToKeep) == false) {
                         deleteFileIgnoreExceptions(stateLocation, stateDir, file);
@@ -412,6 +430,10 @@ List<Path> findStateFilesByGeneration(final long generation, Path... locations)
     }
 
     public String getStateFileName(long generation) {
+        return getStateFileName(prefix, generation);
+    }
+
+    public static String getStateFileName(String prefix, long generation) {
         return prefix + generation + STATE_FILE_EXTENSION;
     }
 

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java

@@ -629,7 +629,7 @@ public void testXContentWithIndexGraveyard() throws IOException {
         final Metadata originalMeta = Metadata.builder().put(ProjectMetadata.builder(projectId).indexGraveyard(graveyard)).build();
         final XContentBuilder builder = JsonXContent.contentBuilder();
         builder.startObject();
-        Metadata.FORMAT.toXContent(builder, originalMeta);
+        ChunkedToXContent.wrapAsToXContent(originalMeta).toXContent(builder, formatParams());
         builder.endObject();
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
             final Metadata fromXContentMeta = Metadata.fromXContent(parser);
@@ -647,7 +647,7 @@ public void testXContentClusterUUID() throws IOException {
             .build();
         final XContentBuilder builder = JsonXContent.contentBuilder();
         builder.startObject();
-        Metadata.FORMAT.toXContent(builder, originalMeta);
+        ChunkedToXContent.wrapAsToXContent(originalMeta).toXContent(builder, formatParams());
         builder.endObject();
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
             final Metadata fromXContentMeta = Metadata.fromXContent(parser);
@@ -732,7 +732,7 @@ public void testXContentWithCoordinationMetadata() throws IOException {
 
         final XContentBuilder builder = JsonXContent.contentBuilder();
         builder.startObject();
-        Metadata.FORMAT.toXContent(builder, metadata);
+        ChunkedToXContent.wrapAsToXContent(metadata).toXContent(builder, formatParams());
         builder.endObject();
 
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
@@ -3345,6 +3345,10 @@ private static CreateIndexResult createIndices(int numIndices, int numBackingInd
         return new CreateIndexResult(indices, backingIndices, b.build());
     }
 
+    private static ToXContent.Params formatParams() {
+        return new ToXContent.MapParams(Map.of("binary", "true", Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY));
+    }
+
     private static class CreateIndexResult {
         final List<Index> indices;
         final List<Index> backingIndices;

server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java

@@ -133,7 +133,11 @@ public void testSimpleJsonFromAndTo() throws IOException {
 
         XContentBuilder builder = JsonXContent.contentBuilder();
         builder.startObject();
-        Metadata.FORMAT.toXContent(builder, metadata);
+        ChunkedToXContent.wrapAsToXContent(metadata)
+            .toXContent(
+                builder,
+                new ToXContent.MapParams(Map.of("binary", "true", Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY))
+            );
         builder.endObject();
 
         Metadata parsedMetadata;

server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java

@@ -9,9 +9,7 @@
 package org.elasticsearch.gateway;
 
 import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexVersion;
@@ -53,20 +51,4 @@ public void testWriteLoadIndex() throws Exception {
     public void testLoadMissingIndex() throws Exception {
         assertThat(metaStateService.loadIndexState(new Index("test1", "test1UUID")), nullValue());
     }
-
-    public void testWriteLoadGlobal() throws Exception {
-        Metadata metadata = Metadata.builder().persistentSettings(Settings.builder().put("test1", "value1").build()).build();
-        MetaStateWriterUtils.writeGlobalState(env, "test_write", metadata);
-        assertThat(metaStateService.loadGlobalState().persistentSettings(), equalTo(metadata.persistentSettings()));
-    }
-
-    public void testWriteGlobalStateWithIndexAndNoIndexIsLoaded() throws Exception {
-        Metadata metadata = Metadata.builder().persistentSettings(Settings.builder().put("test1", "value1").build()).build();
-        IndexMetadata index = indexMetadata("test1");
-        Metadata metadataWithIndex = Metadata.builder(metadata).put(index, true).build();
-
-        MetaStateWriterUtils.writeGlobalState(env, "test_write", metadataWithIndex);
-        assertThat(metaStateService.loadGlobalState().persistentSettings(), equalTo(metadata.persistentSettings()));
-        assertThat(metaStateService.loadGlobalState().getProject().hasIndex("test1"), equalTo(false));
-    }
 }

test/framework/src/main/java/org/elasticsearch/gateway/MetaStateWriterUtils.java

@@ -13,8 +13,6 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.cluster.metadata.Manifest;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 
@@ -29,22 +27,6 @@ private MetaStateWriterUtils() {
         throw new AssertionError("static methods only");
     }
 
-    /**
-     * Writes manifest file (represented by {@link Manifest}) to disk and performs cleanup of old manifest state file if
-     * the write succeeds or newly created manifest state if the write fails.
-     *
-     * @throws WriteStateException if exception when writing state occurs. See also {@link WriteStateException#isDirty()}
-     */
-    public static void writeManifestAndCleanup(NodeEnvironment nodeEnv, String reason, Manifest manifest) throws WriteStateException {
-        logger.trace("[_meta] writing state, reason [{}]", reason);
-        try {
-            long generation = Manifest.FORMAT.writeAndCleanup(manifest, nodeEnv.nodeDataPaths());
-            logger.trace("[_meta] state written (generation: {})", generation);
-        } catch (WriteStateException ex) {
-            throw new WriteStateException(ex.isDirty(), "[_meta]: failed to write meta state", ex);
-        }
-    }
-
     /**
      * Writes the index state.
      * <p>
@@ -65,21 +47,4 @@ public static long writeIndex(NodeEnvironment nodeEnv, String reason, IndexMetad
         }
     }
 
-    /**
-     * Writes the global state, *without* the indices states.
-     *
-     * @throws WriteStateException if exception when writing state occurs. {@link WriteStateException#isDirty()} will always return
-     *                             false, because new global state file is not yet referenced by manifest file.
-     */
-    static long writeGlobalState(NodeEnvironment nodeEnv, String reason, Metadata metadata) throws WriteStateException {
-        logger.trace("[_global] writing state, reason [{}]", reason);
-        try {
-            long generation = Metadata.FORMAT.write(metadata, nodeEnv.nodeDataPaths());
-            logger.trace("[_global] state written");
-            return generation;
-        } catch (WriteStateException ex) {
-            throw new WriteStateException(false, "[_global]: failed to write global state", ex);
-        }
-    }
-
 }