Make GCP Objectstore delete requests retryable (#1768)

Make GCP Objectstore delete requests retryable

Author: Yavor16

multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java

@@ -52,7 +52,7 @@ public GcpObjectStoreFileStorage(Map<String, Object> credentials) {
     protected Storage createObjectStoreStorage(Map<String, Object> credentials) {
         return StorageOptions.http()
                              .setCredentials(getGcpCredentialsSupplier(credentials))
-                             .setStorageRetryStrategy(StorageRetryStrategy.getDefaultStorageRetryStrategy())
+                             .setStorageRetryStrategy(StorageRetryStrategy.getUniformStorageRetryStrategy())
                              .setRetrySettings(
                                  RetrySettings.newBuilder()
                                               .setMaxAttempts(OBJECT_STORE_MAX_ATTEMPTS_CONFIG)
@@ -110,7 +110,19 @@ public List<FileEntry> getFileEntriesWithoutContent(List<FileEntry> fileEntries)
 
     @Override
     public void deleteFile(String id, String space) {
-        storage.delete(bucketName, id);
+        deleteFileWithGeneration(id);
+    }
+
+    private boolean deleteFileWithGeneration(String id) {
+        Blob blob = storage.get(bucketName, id);
+        if (blob == null) {
+            return false;
+        }
+        if (blob.getGeneration() == null) {
+            return storage.delete(bucketName, id);
+        }
+        //Without generationMatch the delete requests are not retried because a retry can "accidentally delete a newer object version"
+        return storage.delete(bucketName, id, Storage.BlobSourceOption.generationMatch(blob.getGeneration()));
     }
 
     @Override
@@ -212,8 +224,10 @@ private int removeBlobsByFilter(Predicate<? super Blob> filter) {
         return deletedBlobsResults.size();
     }
 
-    protected List<Boolean> deleteBlobs(List<BlobId> blobIds) {
-        return storage.delete(blobIds);
+    private List<Boolean> deleteBlobs(List<BlobId> blobIds) {
+        return blobIds.stream()
+                      .map(blobId -> deleteFileWithGeneration(blobId.getName()))
+                      .toList();
     }
 
     protected Set<String> getEntryNames(Predicate<? super Blob> filter) {

multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java

@@ -3,10 +3,9 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Map;
 import java.util.UUID;
+
 import com.google.cloud.storage.Blob;
 import com.google.cloud.storage.BlobId;
 import com.google.cloud.storage.BlobInfo;
@@ -16,6 +15,7 @@
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.springframework.http.MediaType;
+
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNull;
 
@@ -34,15 +34,6 @@ public void setUp() {
             protected Storage createObjectStoreStorage(Map<String, Object> credentials) {
                 return storage;
             }
-
-            @Override
-            protected List<Boolean> deleteBlobs(List<BlobId> blobIds) {
-                List<Boolean> deletedBlobsResults = new ArrayList<>();
-                for (BlobId blobId : blobIds) {
-                    deletedBlobsResults.add(storage.delete(blobId));
-                }
-                return List.copyOf(deletedBlobsResults);
-            }
         };
         spaceId = UUID.randomUUID()
                       .toString();