UID2-1113 Do not create .bak files when writing files in S3 (#73)

thomasm-ttd · web-flow · commit 7c8100a6672e · 2023-05-16T13:52:23.000+10:00
* Removed code to create .bak files in S3

* Updated shared version

* Tag salts with either obsolete or current tags
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM openjdk:11
+FROM eclipse-temurin@sha256:d19c17f59e768549cd3d26f577be73b5e26e652dd66210d91a6738a355aa1dfe
 
 WORKDIR /app
 EXPOSE 8089
diff --git a/conf/local-config.json b/conf/local-config.json
@@ -9,7 +9,7 @@
   "keys_metadata_path": "keys/metadata.json",
   "keys_acl_metadata_path": "keys_acl/metadata.json",
   "salts_metadata_path": "salts/metadata.json",
-  "salt_snapshot_location_prefix": "salts/salts.txt",
+  "salt_snapshot_location_prefix": "salts/salts.txt.",
   "operators_metadata_path": "operators/metadata.json",
   "enclaves_metadata_path": "enclaves/metadata.json",
   "partners_metadata_path": "partners/metadata.json",
diff --git a/conf/local-e2e-docker-config.json b/conf/local-e2e-docker-config.json
@@ -9,7 +9,7 @@
   "keys_metadata_path": "keys/metadata.json",
   "keys_acl_metadata_path": "keys_acl/metadata.json",
   "salts_metadata_path": "salts/metadata.json",
-  "salt_snapshot_location_prefix": "salts/salts.txt",
+  "salt_snapshot_location_prefix": "salts/salts.txt.",
   "operators_metadata_path": "operators/metadata.json",
   "enclaves_metadata_path": "enclaves/metadata.json",
   "partners_metadata_path": "partners/metadata.json",
diff --git a/pom.xml b/pom.xml
@@ -16,7 +16,7 @@
         <!-- check micrometer.version vertx-micrometer-metrics consumes before bumping up -->
         <micrometer.version>1.1.0</micrometer.version>
         <junit-jupiter.version>5.7.0</junit-jupiter.version>
-        <uid2-shared.version>2.1.0</uid2-shared.version>
+        <uid2-shared.version>2.5.0-6429c70acc</uid2-shared.version>
         <image.version>${project.version}</image.version>
     </properties>
 
@@ -61,11 +61,6 @@
             <artifactId>micrometer-registry-prometheus</artifactId>
             <version>${micrometer.version}</version>
         </dependency>
-        <dependency>
-            <groupId>com.sendgrid</groupId>
-            <artifactId>sendgrid-java</artifactId>
-            <version>4.6.6</version>
-        </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.datatype</groupId>
             <artifactId>jackson-datatype-jsr310</artifactId>
diff --git a/src/main/java/com/uid2/admin/Main.java b/src/main/java/com/uid2/admin/Main.java
@@ -26,7 +26,7 @@
 import com.uid2.shared.auth.EnclaveIdentifierProvider;
 import com.uid2.shared.auth.RotatingOperatorKeyProvider;
 import com.uid2.shared.cloud.CloudUtils;
-import com.uid2.shared.cloud.ICloudStorage;
+import com.uid2.shared.cloud.TaggableCloudStorage;
 import com.uid2.shared.jmx.AdminApi;
 import com.uid2.shared.middleware.AuthMiddleware;
 import com.uid2.shared.store.CloudPath;
@@ -75,7 +75,7 @@ public Main(Vertx vertx, JsonObject config) {
     public void run() {
         try {
             AuthFactory authFactory = new GithubAuthFactory(config);
-            ICloudStorage cloudStorage = CloudUtils.createStorage(config.getString(Const.Config.CoreS3BucketProp), config);
+            TaggableCloudStorage cloudStorage = CloudUtils.createStorage(config.getString(Const.Config.CoreS3BucketProp), config);
             FileStorage fileStorage = new TmpFileStorage();
             ObjectWriter jsonWriter = JsonUtil.createJsonWriter();
             FileManager fileManager = new FileManager(cloudStorage, fileStorage);
diff --git a/src/main/java/com/uid2/admin/store/FileManager.java b/src/main/java/com/uid2/admin/store/FileManager.java
@@ -7,7 +7,6 @@
 import io.vertx.core.json.JsonObject;
 
 import java.io.IOException;
-import java.io.InputStream;
 import java.util.List;
 
 public class FileManager {
@@ -24,13 +23,6 @@ public void uploadFile(CloudPath location, FileName fileName, String content) th
         cloudStorage.upload(newFile, location.toString());
     }
 
-    public void backupFile(CloudPath path, FileName fileName, long timestamp) throws IOException, CloudStorageException {
-        InputStream download = cloudStorage.download(path.toString());
-        String localTemp = fileStorage.create(fileName, download);
-        cloudStorage.upload(localTemp, path + ".bak");
-        cloudStorage.upload(localTemp, path + "." + timestamp + ".bak");
-    }
-
     public void uploadMetadata(JsonObject metadata, String name, CloudPath location) throws Exception {
         FileName fileName = new FileName(name + "-metadata", ".json");
         String content = Json.encodePrettily(metadata);
diff --git a/src/main/java/com/uid2/admin/store/writer/AdminUserStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/AdminUserStoreWriter.java
@@ -28,7 +28,6 @@ public AdminUserStoreWriter(AdminUserProvider provider, FileManager fileManager,
 
     public void upload(Collection<AdminUser> data) throws Exception {
         long generated = Instant.now().getEpochSecond();
-        FileName backupFile = new FileName("admins-old", ".json");
         FileName dataFile = new FileName("admins", ".json");
 
         JsonObject metadata = provider.getMetadata();
@@ -39,8 +38,6 @@ public void upload(Collection<AdminUser> data) throws Exception {
         // get location to upload
         CloudPath location = new CloudPath(metadata.getJsonObject("admins").getString("location"));
 
-        fileManager.backupFile(location, backupFile, generated);
-
         // generate new admins
         String content = jsonWriter.writeValueAsString(data);
         fileManager.uploadFile(location, dataFile, content);
diff --git a/src/main/java/com/uid2/admin/store/writer/ClientKeyStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/ClientKeyStoreWriter.java
@@ -19,9 +19,8 @@ public class ClientKeyStoreWriter implements StoreWriter<Collection<ClientKey>>
     public ClientKeyStoreWriter(RotatingClientKeyProvider provider, FileManager fileManager, ObjectWriter jsonWriter, VersionGenerator versionGenerator, Clock clock, StoreScope scope) {
         this.jsonWriter = jsonWriter;
         FileName dataFile = new FileName("clients", ".json");
-        FileName backupFile = new FileName("clients-old", ".json");
         String dataType = "client_keys";
-        writer = new ScopedStoreWriter(provider, fileManager, versionGenerator, clock, scope, dataFile, backupFile, dataType);
+        writer = new ScopedStoreWriter(provider, fileManager, versionGenerator, clock, scope, dataFile, dataType);
     }
 
     @Override
diff --git a/src/main/java/com/uid2/admin/store/writer/EnclaveStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/EnclaveStoreWriter.java
@@ -27,16 +27,13 @@ public EnclaveStoreWriter(EnclaveIdentifierProvider provider, FileManager fileMa
 
     public void upload(Collection<EnclaveIdentifier> data) throws Exception {
         long generated = Instant.now().getEpochSecond();
-        FileName backupFile = new FileName("enclaves-old", ".json");
         FileName dataFile = new FileName("enclaves", ".json");
         JsonObject metadata = provider.getMetadata();
 
         metadata.put("version", versionGenerator.getVersion());
         metadata.put("generated", generated);
         CloudPath location = new CloudPath(metadata.getJsonObject("enclaves").getString("location"));
 
-        fileManager.backupFile(location, backupFile, generated);
-
         // generate new clients
         String content = jsonWriter.writeValueAsString(data);
         fileManager.uploadFile(location, dataFile, content);
diff --git a/src/main/java/com/uid2/admin/store/writer/EncryptionKeyStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/EncryptionKeyStoreWriter.java
@@ -18,9 +18,8 @@ public class EncryptionKeyStoreWriter implements StoreWriter<Collection<Encrypti
 
     public EncryptionKeyStoreWriter(RotatingKeyStore provider, FileManager fileManager, VersionGenerator versionGenerator, Clock clock, StoreScope scope) {
         FileName dataFile = new FileName("keys", ".json");
-        FileName backupFile = new FileName("keys-old", ".json");
         String dataType = "keys";
-        writer = new ScopedStoreWriter(provider, fileManager, versionGenerator, clock, scope, dataFile, backupFile, dataType);
+        writer = new ScopedStoreWriter(provider, fileManager, versionGenerator, clock, scope, dataFile, dataType);
     }
 
     @Override
diff --git a/src/main/java/com/uid2/admin/store/writer/KeyAclStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/KeyAclStoreWriter.java
@@ -19,9 +19,8 @@ public class KeyAclStoreWriter implements StoreWriter<Map<Integer, EncryptionKey
 
     public KeyAclStoreWriter(StoreReader<Map<Integer, EncryptionKeyAcl>> provider, FileManager fileManager, ObjectWriter jsonWriter, VersionGenerator versionGenerator, Clock clock, StoreScope scope) {
         FileName dataFile = new FileName("keys_acl", ".json");
-        FileName backupFile = new FileName("keys_acl-old", ".json");
         String dataType = "keys_acl";
-        writer = new ScopedStoreWriter(provider, fileManager, versionGenerator, clock, scope, dataFile, backupFile, dataType);
+        writer = new ScopedStoreWriter(provider, fileManager, versionGenerator, clock, scope, dataFile, dataType);
     }
 
     @Override
diff --git a/src/main/java/com/uid2/admin/store/writer/OperatorKeyStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/OperatorKeyStoreWriter.java
@@ -28,7 +28,6 @@ public OperatorKeyStoreWriter(RotatingOperatorKeyProvider provider, FileManager
     public void upload(Collection<OperatorKey> data) throws Exception {
         long generated = Instant.now().getEpochSecond();
         FileName dataFile = new FileName("operators", ".json");
-        FileName backupFile = new FileName("operators-old", ".json");
 
         JsonObject metadata = provider.getMetadata();
         // bump up metadata version
@@ -38,8 +37,6 @@ public void upload(Collection<OperatorKey> data) throws Exception {
         // get location to upload
         CloudPath location = new CloudPath(metadata.getJsonObject("operators").getString("location"));
 
-        fileManager.backupFile(location, backupFile, generated);
-
         // generate new operators
         String content = jsonWriter.writeValueAsString(data);
         fileManager.uploadFile(location, dataFile, content);
diff --git a/src/main/java/com/uid2/admin/store/writer/PartnerStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/PartnerStoreWriter.java
@@ -24,7 +24,6 @@ public PartnerStoreWriter(RotatingPartnerStore provider, FileManager fileManager
 
     public void upload(JsonArray partners) throws Exception {
         long generated = Instant.now().getEpochSecond();
-        FileName backupFile = new FileName("partners-old", ".json");
         FileName dataFile = new FileName("partners", ".json");
 
         JsonObject metadata = provider.getMetadata();
@@ -35,8 +34,6 @@ public void upload(JsonArray partners) throws Exception {
         // get location to upload
         CloudPath location = new CloudPath(metadata.getJsonObject("partners").getString("location"));
 
-        fileManager.backupFile(location, backupFile, generated);
-
         // generate new partners
         String content = partners.encodePrettily();
         fileManager.uploadFile(location, dataFile, content);
diff --git a/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java
@@ -2,10 +2,10 @@
 
 import com.uid2.admin.store.FileManager;
 import com.uid2.admin.store.version.VersionGenerator;
-import com.uid2.shared.cloud.ICloudStorage;
+import com.uid2.shared.cloud.CloudStorageException;
+import com.uid2.shared.cloud.TaggableCloudStorage;
 import com.uid2.shared.model.SaltEntry;
 import com.uid2.shared.store.CloudPath;
-import com.uid2.admin.store.FileName;
 import com.uid2.shared.store.RotatingSaltProvider;
 import io.vertx.core.json.JsonArray;
 import io.vertx.core.json.JsonObject;
@@ -18,6 +18,7 @@
 import java.time.Instant;
 import java.util.Comparator;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -28,23 +29,23 @@ public class SaltStoreWriter {
     private final String saltSnapshotLocationPrefix;
     private final VersionGenerator versionGenerator;
 
-    private final ICloudStorage cloudStorage;
+    private final TaggableCloudStorage cloudStorage;
 
-    public SaltStoreWriter(JsonObject config, RotatingSaltProvider provider, FileManager fileManager, ICloudStorage cloudStorage, VersionGenerator versionGenerator) {
+    private final Map<String, String> currentTags = Map.of("status", "current");
+    private final Map<String, String> obsoleteTags = Map.of("status", "obsolete");
+
+    public SaltStoreWriter(JsonObject config, RotatingSaltProvider provider, FileManager fileManager, TaggableCloudStorage cloudStorage, VersionGenerator versionGenerator) {
         this.provider = provider;
         this.fileManager = fileManager;
         this.cloudStorage = cloudStorage;
         this.saltSnapshotLocationPrefix = config.getString("salt_snapshot_location_prefix");
         this.versionGenerator = versionGenerator;
     }
+
     public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
         final Instant now = Instant.now();
         final long generated = now.getEpochSecond();
 
-        FileName backupFile = new FileName("salts-old", ".json");
-
-        fileManager.backupFile(new CloudPath(provider.getMetadataPath()), backupFile, generated);
-
         final JsonObject metadata = provider.getMetadata();
         // bump up metadata version
         metadata.put("version", versionGenerator.getVersion());
@@ -91,13 +92,36 @@ public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
         provider.loadContent();
     }
 
+    /**
+     * reads the metadata file, and marks each referenced file as ready for archiving
+     */
+    public void archiveSaltLocations() throws Exception {
+        final JsonObject metadata = provider.getMetadata();
+
+        metadata.getJsonArray("salts").forEach(instance -> {
+            try {
+                JsonObject salt = (JsonObject) instance;
+                String location = salt.getString("location", "");
+                if (!location.isBlank()) {
+                    this.setStatusTagToObsolete(location);
+                }
+            } catch (Exception ex) {
+                LOGGER.error("Error marking object as ready for archiving", ex);
+            }
+        });
+    }
+
     private String getSaltSnapshotLocation(RotatingSaltProvider.SaltSnapshot snapshot) {
         return saltSnapshotLocationPrefix + snapshot.getEffective().toEpochMilli();
     }
 
     private void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, String location) throws Exception {
         // do not overwrite existing files
-        if (!cloudStorage.list(location).isEmpty()) return;
+        if (!cloudStorage.list(location).isEmpty()) {
+            // update the tags on the file to ensure it is still marked as current
+            this.setStatusTagToCurrent(location);
+            return;
+        }
 
         final Path newSaltsFile = Files.createTempFile("operators", ".txt");
         try (BufferedWriter w = Files.newBufferedWriter(newSaltsFile)) {
@@ -106,6 +130,14 @@ private void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, Str
             }
         }
 
-        cloudStorage.upload(newSaltsFile.toString(), location);
+        cloudStorage.upload(newSaltsFile.toString(), location, this.currentTags);
+    }
+
+    private void setStatusTagToCurrent(String location) throws CloudStorageException {
+        this.cloudStorage.setTags(location, this.currentTags);
+    }
+
+    private void setStatusTagToObsolete(String location) throws CloudStorageException {
+        this.cloudStorage.setTags(location, this.obsoleteTags);
     }
 }
diff --git a/src/main/java/com/uid2/admin/store/writer/ScopedStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/ScopedStoreWriter.java
@@ -16,7 +16,6 @@ public class ScopedStoreWriter {
     private final Clock clock;
     private final StoreScope scope;
     private final FileName dataFile;
-    private final FileName backupFile;
     private final String dataType;
 
     public ScopedStoreWriter(
@@ -26,7 +25,6 @@ public ScopedStoreWriter(
             Clock clock,
             StoreScope scope,
             FileName dataFile,
-            FileName backupFile,
             String dataType
     ) {
         this.provider = provider;
@@ -35,7 +33,6 @@ public ScopedStoreWriter(
         this.clock = clock;
         this.scope = scope;
         this.dataFile = dataFile;
-        this.backupFile = backupFile;
         this.dataType = dataType;
     }
 
@@ -59,10 +56,6 @@ public void upload(String data, JsonObject extraMeta) throws Exception {
             metadata.addExtra(extraMeta);
         }
 
-        if (!isFirstWrite) {
-            fileManager.backupFile(location, backupFile, generated);
-        }
-
         fileManager.uploadFile(location, dataFile, data);
         fileManager.uploadMetadata(metadata.getJson(), dataType, scope.getMetadataPath());
 
diff --git a/src/main/java/com/uid2/admin/store/writer/SiteStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/SiteStoreWriter.java
@@ -19,9 +19,8 @@ public class SiteStoreWriter implements StoreWriter<Collection<Site>> {
     public SiteStoreWriter(IMetadataVersionedStore reader, FileManager fileManager, ObjectWriter jsonWriter, VersionGenerator versionGenerator, Clock clock, StoreScope scope) {
         this.jsonWriter = jsonWriter;
         FileName dataFile = new FileName("sites", ".json");
-        FileName backupFile = new FileName("sites-old", ".json");
         String dataType = "sites";
-        writer = new ScopedStoreWriter(reader, fileManager, versionGenerator, clock, scope, dataFile, backupFile, dataType);
+        writer = new ScopedStoreWriter(reader, fileManager, versionGenerator, clock, scope, dataFile, dataType);
     }
 
     @Override
diff --git a/src/main/java/com/uid2/admin/vertx/service/SaltService.java b/src/main/java/com/uid2/admin/vertx/service/SaltService.java
@@ -80,6 +80,9 @@ private void handleSaltRotate(RoutingContext rc) {
             // force refresh
             this.saltProvider.loadContent();
 
+            // mark all the referenced files as ready to archive
+            storageManager.archiveSaltLocations();
+
             final List<RotatingSaltProvider.SaltSnapshot> snapshots = this.saltProvider.getSnapshots();
             final RotatingSaltProvider.SaltSnapshot lastSnapshot = snapshots.get(snapshots.size() - 1);
             final ISaltRotation.Result result = saltRotation.rotateSalts(
diff --git a/src/test/java/com/uid2/admin/store/writer/ClientKeyStoreTest.java b/src/test/java/com/uid2/admin/store/writer/ClientKeyStoreTest.java
@@ -53,7 +53,7 @@ void overridesWithNewDataOnSubsequentUploads() throws Exception {
         }
 
         @Test
-        void backsUpOldData() throws Exception {
+        void doesNotBackUpOldData() throws Exception {
             Long now = 1L; // seconds since epoch
             when(clock.getEpochSecond()).thenReturn(now);
 
@@ -65,7 +65,11 @@ void backsUpOldData() throws Exception {
             List<String> files = cloudStorage.list(rootDir);
             String datedBackup = "this-test-data-type/clients.json." + now + ".bak";
             String latestBackup = "this-test-data-type/clients.json.bak";
-            assertThat(files).contains(datedBackup, latestBackup);
+            assertThat(files).doesNotContain(datedBackup, latestBackup);
+
+            String metaDataFile = "this-test-data-type/test-metadata.json";
+            String clientFile = "this-test-data-type/clients.json";
+            assertThat(files).contains(metaDataFile, clientFile);
         }
 
         @Test
diff --git a/src/test/java/com/uid2/admin/store/writer/ScopedStoreWriterTest.java b/src/test/java/com/uid2/admin/store/writer/ScopedStoreWriterTest.java
diff --git a/src/test/java/com/uid2/admin/vertx/SaltServiceTest.java b/src/test/java/com/uid2/admin/vertx/SaltServiceTest.java

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-FROM openjdk:11`
	`1`	`+FROM eclipse-temurin@sha256:d19c17f59e768549cd3d26f577be73b5e26e652dd66210d91a6738a355aa1dfe`
`2`	`2`
`3`	`3`	`WORKDIR /app`
`4`	`4`	`EXPOSE 8089`
Original file line number	Diff line number	Diff line change
`@@ -18,9 +18,8 @@ public class EncryptionKeyStoreWriter implements StoreWriter<Collection<Encrypti`
`18`	`18`
`19`	`19`	`public EncryptionKeyStoreWriter(RotatingKeyStore provider, FileManager fileManager, VersionGenerator versionGenerator, Clock clock, StoreScope scope) {`
`20`	`20`	`FileName dataFile = new FileName("keys", ".json");`
`21`		`- FileName backupFile = new FileName("keys-old", ".json");`
`22`	`21`	`String dataType = "keys";`
`23`		`- writer = new ScopedStoreWriter(provider, fileManager, versionGenerator, clock, scope, dataFile, backupFile, dataType);`
	`22`	`+ writer = new ScopedStoreWriter(provider, fileManager, versionGenerator, clock, scope, dataFile, dataType);`
`24`	`23`	`}`
`25`	`24`
`26`	`25`	`@Override`