Merge branch 'main' into add-parseboolean-to-forbidden-apis

Author: kvanerum

server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java

@@ -40,7 +40,6 @@
 import org.elasticsearch.core.FixForMultiProject;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.gateway.MetadataStateFormat;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexVersion;
@@ -50,7 +49,6 @@
 import org.elasticsearch.xcontent.NamedObjectNotFoundException;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -2101,30 +2099,6 @@ static <C extends MetadataCustom<C>> void parseCustomObject(
         }
     }
 
-    private static final ToXContent.Params FORMAT_PARAMS;
-    static {
-        Map<String, String> params = Maps.newMapWithExpectedSize(2);
-        params.put("binary", "true");
-        params.put(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY);
-        FORMAT_PARAMS = new ToXContent.MapParams(params);
-    }
-
-    /**
-     * State format for {@link Metadata} to write to and load from disk
-     */
-    public static final MetadataStateFormat<Metadata> FORMAT = new MetadataStateFormat<>(GLOBAL_STATE_FILE_PREFIX) {
-
-        @Override
-        public void toXContent(XContentBuilder builder, Metadata state) throws IOException {
-            ChunkedToXContent.wrapAsToXContent(state).toXContent(builder, FORMAT_PARAMS);
-        }
-
-        @Override
-        public Metadata fromXContent(XContentParser parser) throws IOException {
-            return Builder.fromXContent(parser);
-        }
-    };
-
     private volatile Metadata.ProjectLookup projectLookup = null;
 
     /**

server/src/main/java/org/elasticsearch/gateway/MetaStateService.java

@@ -21,10 +21,14 @@
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Predicate;
 
+import static org.elasticsearch.cluster.metadata.Metadata.GLOBAL_STATE_FILE_PREFIX;
+
 /**
  * Handles writing and loading {@link Manifest}, {@link Metadata} and {@link IndexMetadata} as used for cluster state persistence in
  * versions prior to {@link Version#V_7_6_0}, used to read this older format during an upgrade from these versions.
@@ -75,20 +79,29 @@ List<IndexMetadata> loadIndicesStates(Predicate<String> excludeIndexPathIdsPredi
         return indexMetadataList;
     }
 
-    /**
-     * Loads the global state, *without* index state
-     */
-    Metadata loadGlobalState() throws IOException {
-        return Metadata.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths());
-    }
-
     /**
      * Creates empty cluster state file on disk, deleting global metadata and unreferencing all index metadata
      * (only used for dangling indices at that point).
      */
     public void unreferenceAll() throws IOException {
         Manifest.FORMAT.writeAndCleanup(Manifest.empty(), nodeEnv.nodeDataPaths()); // write empty file so that indices become unreferenced
-        Metadata.FORMAT.cleanupOldFiles(Long.MAX_VALUE, nodeEnv.nodeDataPaths());
+        cleanUpGlobalStateFiles();
+    }
+
+    private void cleanUpGlobalStateFiles() throws IOException {
+        for (Path location : nodeEnv.nodeDataPaths()) {
+            logger.trace("cleanupOldFiles: cleaning up {} for global state files", location);
+            final Path stateLocation = location.resolve(MetadataStateFormat.STATE_DIR_NAME);
+            try (var paths = Files.list(stateLocation)) {
+                paths.filter(file -> file.getFileName().toString().startsWith(GLOBAL_STATE_FILE_PREFIX)).forEach(file -> {
+                    try {
+                        Files.deleteIfExists(file);
+                    } catch (IOException e) {
+                        logger.trace("failed to delete global state file: {}", file);
+                    }
+                });
+            }
+        }
     }
 
     /**

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java

@@ -629,7 +629,7 @@ public void testXContentWithIndexGraveyard() throws IOException {
         final Metadata originalMeta = Metadata.builder().put(ProjectMetadata.builder(projectId).indexGraveyard(graveyard)).build();
         final XContentBuilder builder = JsonXContent.contentBuilder();
         builder.startObject();
-        Metadata.FORMAT.toXContent(builder, originalMeta);
+        ChunkedToXContent.wrapAsToXContent(originalMeta).toXContent(builder, formatParams());
         builder.endObject();
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
             final Metadata fromXContentMeta = Metadata.fromXContent(parser);
@@ -647,7 +647,7 @@ public void testXContentClusterUUID() throws IOException {
             .build();
         final XContentBuilder builder = JsonXContent.contentBuilder();
         builder.startObject();
-        Metadata.FORMAT.toXContent(builder, originalMeta);
+        ChunkedToXContent.wrapAsToXContent(originalMeta).toXContent(builder, formatParams());
         builder.endObject();
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
             final Metadata fromXContentMeta = Metadata.fromXContent(parser);
@@ -732,7 +732,7 @@ public void testXContentWithCoordinationMetadata() throws IOException {
 
         final XContentBuilder builder = JsonXContent.contentBuilder();
         builder.startObject();
-        Metadata.FORMAT.toXContent(builder, metadata);
+        ChunkedToXContent.wrapAsToXContent(metadata).toXContent(builder, formatParams());
         builder.endObject();
 
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
@@ -3345,6 +3345,10 @@ private static CreateIndexResult createIndices(int numIndices, int numBackingInd
         return new CreateIndexResult(indices, backingIndices, b.build());
     }
 
+    private static ToXContent.Params formatParams() {
+        return new ToXContent.MapParams(Map.of("binary", "true", Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY));
+    }
+
     private static class CreateIndexResult {
         final List<Index> indices;
         final List<Index> backingIndices;

server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java

@@ -133,7 +133,11 @@ public void testSimpleJsonFromAndTo() throws IOException {
 
         XContentBuilder builder = JsonXContent.contentBuilder();
         builder.startObject();
-        Metadata.FORMAT.toXContent(builder, metadata);
+        ChunkedToXContent.wrapAsToXContent(metadata)
+            .toXContent(
+                builder,
+                new ToXContent.MapParams(Map.of("binary", "true", Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY))
+            );
         builder.endObject();
 
         Metadata parsedMetadata;

server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java

@@ -9,9 +9,7 @@
 package org.elasticsearch.gateway;
 
 import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexVersion;
@@ -53,20 +51,4 @@ public void testWriteLoadIndex() throws Exception {
     public void testLoadMissingIndex() throws Exception {
         assertThat(metaStateService.loadIndexState(new Index("test1", "test1UUID")), nullValue());
     }
-
-    public void testWriteLoadGlobal() throws Exception {
-        Metadata metadata = Metadata.builder().persistentSettings(Settings.builder().put("test1", "value1").build()).build();
-        MetaStateWriterUtils.writeGlobalState(env, "test_write", metadata);
-        assertThat(metaStateService.loadGlobalState().persistentSettings(), equalTo(metadata.persistentSettings()));
-    }
-
-    public void testWriteGlobalStateWithIndexAndNoIndexIsLoaded() throws Exception {
-        Metadata metadata = Metadata.builder().persistentSettings(Settings.builder().put("test1", "value1").build()).build();
-        IndexMetadata index = indexMetadata("test1");
-        Metadata metadataWithIndex = Metadata.builder(metadata).put(index, true).build();
-
-        MetaStateWriterUtils.writeGlobalState(env, "test_write", metadataWithIndex);
-        assertThat(metaStateService.loadGlobalState().persistentSettings(), equalTo(metadata.persistentSettings()));
-        assertThat(metaStateService.loadGlobalState().getProject().hasIndex("test1"), equalTo(false));
-    }
 }

test/framework/src/main/java/org/elasticsearch/gateway/MetaStateWriterUtils.java

@@ -13,8 +13,6 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.cluster.metadata.Manifest;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 
@@ -29,22 +27,6 @@ private MetaStateWriterUtils() {
         throw new AssertionError("static methods only");
     }
 
-    /**
-     * Writes manifest file (represented by {@link Manifest}) to disk and performs cleanup of old manifest state file if
-     * the write succeeds or newly created manifest state if the write fails.
-     *
-     * @throws WriteStateException if exception when writing state occurs. See also {@link WriteStateException#isDirty()}
-     */
-    public static void writeManifestAndCleanup(NodeEnvironment nodeEnv, String reason, Manifest manifest) throws WriteStateException {
-        logger.trace("[_meta] writing state, reason [{}]", reason);
-        try {
-            long generation = Manifest.FORMAT.writeAndCleanup(manifest, nodeEnv.nodeDataPaths());
-            logger.trace("[_meta] state written (generation: {})", generation);
-        } catch (WriteStateException ex) {
-            throw new WriteStateException(ex.isDirty(), "[_meta]: failed to write meta state", ex);
-        }
-    }
-
     /**
      * Writes the index state.
      * <p>
@@ -65,21 +47,4 @@ public static long writeIndex(NodeEnvironment nodeEnv, String reason, IndexMetad
         }
     }
 
-    /**
-     * Writes the global state, *without* the indices states.
-     *
-     * @throws WriteStateException if exception when writing state occurs. {@link WriteStateException#isDirty()} will always return
-     *                             false, because new global state file is not yet referenced by manifest file.
-     */
-    static long writeGlobalState(NodeEnvironment nodeEnv, String reason, Metadata metadata) throws WriteStateException {
-        logger.trace("[_global] writing state, reason [{}]", reason);
-        try {
-            long generation = Metadata.FORMAT.write(metadata, nodeEnv.nodeDataPaths());
-            logger.trace("[_global] state written");
-            return generation;
-        } catch (WriteStateException ex) {
-            throw new WriteStateException(false, "[_global]: failed to write global state", ex);
-        }
-    }
-
 }