diff --git a/pom.xml b/pom.xml index 8708acb4..08e0b3c1 100644 --- a/pom.xml +++ b/pom.xml @@ -10,13 +10,13 @@ UTF-8 - 4.5.11 + 4.5.13 1.0.22 com.uid2.admin.vertx.AdminVerticle 1.12.2 5.11.2 - 8.0.32 + 8.1.22 0.5.10 ${project.version} diff --git a/src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java index 775b29b8..9ce37db4 100644 --- a/src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java +++ b/src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java @@ -18,14 +18,18 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Base64; import java.util.Collection; +import java.util.List; public class EncryptedSaltStoreWriter extends SaltStoreWriter implements StoreWriter { private StoreScope scope; private RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider; private Integer siteId; + private final List previousSeenSnapshots = new ArrayList<>(); + private static final Logger LOGGER = LoggerFactory.getLogger(EncryptedSaltStoreWriter.class); public EncryptedSaltStoreWriter(JsonObject config, RotatingSaltProvider provider, FileManager fileManager, TaggableCloudStorage cloudStorage, VersionGenerator versionGenerator, StoreScope scope, @@ -91,6 +95,16 @@ protected void refreshProvider() { // we do not need to refresh the provider on encrypted writers } + @Override + protected List getSnapshots(RotatingSaltProvider.SaltSnapshot data){ + /* + Since metadata.json is overwritten during the process, we maintain a history of all snapshots seen so far. + On the final write, we append this history to metadata.json to ensure no snapshots are lost. + */ + this.previousSeenSnapshots.add(data); + return this.previousSeenSnapshots; + } + @Override public void upload(Object data, JsonObject extraMeta) throws Exception { for(RotatingSaltProvider.SaltSnapshot saltSnapshot: (Collection) data) { diff --git a/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java index f8bfc2a5..738ee64c 100644 --- a/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java +++ b/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java @@ -43,6 +43,30 @@ public SaltStoreWriter(JsonObject config, RotatingSaltProvider provider, FileMan this.versionGenerator = versionGenerator; } + protected List getSnapshots(RotatingSaltProvider.SaltSnapshot data){ + if (provider.getSnapshots() == null) { + throw new IllegalStateException("Snapshots cannot be null"); + } + final Instant now = Instant.now(); + List currentSnapshots = Stream.concat(provider.getSnapshots().stream(), Stream.of(data)) + .sorted(Comparator.comparing(RotatingSaltProvider.SaltSnapshot::getEffective)) + .collect(Collectors.toList()); + RotatingSaltProvider.SaltSnapshot newestEffectiveSnapshot = currentSnapshots.stream() + .filter(snapshot -> snapshot.isEffective(now)) + .reduce((a, b) -> b).orElse(null); + return Stream.concat(provider.getSnapshots().stream(), Stream.of(data)) + .filter(snapshot -> { + boolean isValid = newestEffectiveSnapshot == null || snapshot == newestEffectiveSnapshot; + if (!isValid) { + LOGGER.info("Skipping effective snapshot, effective=" + snapshot.getEffective() + ", expires=" + snapshot.getExpires() + + " in favour of newer snapshot, effective=" + newestEffectiveSnapshot.getEffective() + ", expires=" + newestEffectiveSnapshot.getExpires()); + } + return isValid; + }) + .sorted(Comparator.comparing(RotatingSaltProvider.SaltSnapshot::getEffective)) + .collect(Collectors.toList()); + } + public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception { final Instant now = Instant.now(); final long generated = now.getEpochSecond(); @@ -64,33 +88,14 @@ public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception { final JsonArray snapshotsMetadata = new JsonArray(); metadata.put("salts", snapshotsMetadata); - List currentSnapshots = provider.getSnapshots(); - List snapshots = null; + List snapshots = this.getSnapshots(data); - if (currentSnapshots != null) { - snapshots = Stream.concat(currentSnapshots.stream(), Stream.of(data)) - .sorted(Comparator.comparing(RotatingSaltProvider.SaltSnapshot::getEffective)) - .collect(Collectors.toList()); - } else { - snapshots = List.of(data); - } - // of the currently effective snapshots keep only the most recent one - RotatingSaltProvider.SaltSnapshot newestEffectiveSnapshot = snapshots.stream() - .filter(snapshot -> snapshot.isEffective(now)) - .reduce((a, b) -> b).orElse(null); for (RotatingSaltProvider.SaltSnapshot snapshot : snapshots) { if (!now.isBefore(snapshot.getExpires())) { LOGGER.info("Skipping expired snapshot, effective=" + snapshot.getEffective() + ", expires=" + snapshot.getExpires()); continue; } - if (newestEffectiveSnapshot != null && snapshot != newestEffectiveSnapshot) { - LOGGER.info("Skipping effective snapshot, effective=" + snapshot.getEffective() + ", expires=" + snapshot.getExpires() - + " in favour of newer snapshot, effective=" + newestEffectiveSnapshot.getEffective() + ", expires=" + newestEffectiveSnapshot.getExpires()); - continue; - } - newestEffectiveSnapshot = null; - final String location = getSaltSnapshotLocation(snapshot); final JsonObject snapshotMetadata = new JsonObject(); diff --git a/src/main/resources/localstack/s3/core/salts/metadata.json b/src/main/resources/localstack/s3/core/salts/metadata.json index b0ccfeda..8b7bf3a4 100644 --- a/src/main/resources/localstack/s3/core/salts/metadata.json +++ b/src/main/resources/localstack/s3/core/salts/metadata.json @@ -4,10 +4,17 @@ "first_level" : "fOGY/aRE44peL23i+cE9MkJrzmEeNZZziNZBfq7qqk8=", "id_prefix" : "b", "id_secret" : "HF6Qz42HBbVHINxhh191dB09BCuTWyBkNtrNicO4ZCw=", - "salts" : [{ + "salts" : [ + { "effective" : 1670796729291, "expires" : 1766125493000, "location" : "salts/salts.txt.1670796729291", "size" : 2 - }] + },{ + "effective" : 1766125493000, + "expires" : 1766720293000, + "location" : "salts/salts.txt.1766125493000", + "size" : 4 + } + ] } \ No newline at end of file diff --git a/src/main/resources/localstack/s3/core/salts/salts.txt.1766125493000 b/src/main/resources/localstack/s3/core/salts/salts.txt.1766125493000 new file mode 100644 index 00000000..83dfa69a --- /dev/null +++ b/src/main/resources/localstack/s3/core/salts/salts.txt.1766125493000 @@ -0,0 +1,4 @@ +1000000,1614556800000,vgv1BwiNRCW7F3VcNXHlZh+7oHJ4G4gCshbGcVOLnss= +1000001,1643235130717,vgv1BwiNRCW7F3VcNXHlZh+7oHJ4G4gCshbGcVOLnss= +1000002,1614556000010,vgv1BwiNRCW7F3VcNXHlZh+7oHJ4G4gCshbGcVOLnss= +1000003,1643235230717,vgv1BwiNRCW7F3VcNXHlZh+7oHJ4G4gCshbGcVOLnss= \ No newline at end of file diff --git a/src/test/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriterTest.java b/src/test/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriterTest.java index f7555106..142b3903 100644 --- a/src/test/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriterTest.java +++ b/src/test/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriterTest.java @@ -10,6 +10,7 @@ import com.uid2.shared.store.RotatingSaltProvider; import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; import com.uid2.shared.store.scope.StoreScope; +import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -20,6 +21,7 @@ import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.time.Instant; @@ -107,21 +109,77 @@ private void verifyFile(String filelocation, RotatingSaltProvider.SaltSnapshot s @Test public void testUploadNew() throws Exception { - RotatingSaltProvider.SaltSnapshot snapshot = makeSnapshot(Instant.now(), Instant.ofEpochMilli(Instant.now().toEpochMilli() + 10000), 1000000); - + RotatingSaltProvider.SaltSnapshot snapshot = makeSnapshot(Instant.ofEpochMilli(1740607938167L), Instant.ofEpochMilli(Instant.now().toEpochMilli() + 90002), 100); + RotatingSaltProvider.SaltSnapshot snapshot2 = makeSnapshot(Instant.ofEpochMilli(1740694476392L), Instant.ofEpochMilli(Instant.now().toEpochMilli() + 130000), 10); when(rotatingSaltProvider.getMetadata()).thenThrow(new CloudStorageException("The specified key does not exist: AmazonS3Exception: test-core-bucket")); when(rotatingSaltProvider.getSnapshots()).thenReturn(null); when(taggableCloudStorage.list(anyString())).thenReturn(new ArrayList<>()); + ArgumentCaptor metadataCaptor = ArgumentCaptor.forClass(JsonObject.class); + ArgumentCaptor nameCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor locationCaptor = ArgumentCaptor.forClass(CloudPath.class); + EncryptedSaltStoreWriter encryptedSaltStoreWriter = new EncryptedSaltStoreWriter(config, rotatingSaltProvider, fileManager, taggableCloudStorage, versionGenerator, storeScope, rotatingCloudEncryptionKeyProvider, siteId); encryptedSaltStoreWriter.upload(snapshot); + verify(fileManager).uploadMetadata(metadataCaptor.capture(), nameCaptor.capture(), locationCaptor.capture()); + + // Capture the metadata + JsonObject capturedMetadata = metadataCaptor.getValue(); + assertEquals(1, capturedMetadata.getJsonArray("salts").size(), "The 'salts' array should contain exactly 1 item"); + encryptedSaltStoreWriter.upload(snapshot2); - verify(taggableCloudStorage).upload(pathCaptor.capture(), cloudPathCaptor.capture(), any()); - assertEquals(cloudPathCaptor.getValue(), "test/path"); + verify(fileManager,times(2)).uploadMetadata(metadataCaptor.capture(), nameCaptor.capture(), locationCaptor.capture()); + capturedMetadata = metadataCaptor.getValue(); + assertEquals(2, capturedMetadata.getJsonArray("salts").size(), "The 'salts' array should contain 2 items"); + + verify(taggableCloudStorage,times(3)).upload(pathCaptor.capture(), cloudPathCaptor.capture(), any()); verifyFile(pathCaptor.getValue(), snapshot); } + + @Test + public void testUnencryptedAndEncryptedBehavesTheSame() throws Exception { + RotatingSaltProvider.SaltSnapshot snapshot = makeSnapshot(Instant.ofEpochMilli(1740607938167L), Instant.ofEpochMilli(Instant.now().toEpochMilli() + 90000), 100); + RotatingSaltProvider.SaltSnapshot snapshot2 = makeSnapshot(Instant.ofEpochMilli(1740694476392L), Instant.ofEpochMilli(Instant.now().toEpochMilli() + 130000), 10); + List snapshots = List.of(snapshot, snapshot2); + + when(rotatingSaltProvider.getMetadata()).thenThrow(new CloudStorageException("The specified key does not exist: AmazonS3Exception: test-core-bucket")); + when(rotatingSaltProvider.getSnapshots()).thenReturn(snapshots); + when(taggableCloudStorage.list(anyString())).thenReturn(new ArrayList<>()); + + ArgumentCaptor metadataCaptor = ArgumentCaptor.forClass(JsonObject.class); + ArgumentCaptor nameCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor locationCaptor = ArgumentCaptor.forClass(CloudPath.class); + + SaltStoreWriter saltStoreWriter = new SaltStoreWriter(config, rotatingSaltProvider, + fileManager, taggableCloudStorage, versionGenerator); + + saltStoreWriter.upload(snapshot); + verify(fileManager).uploadMetadata(metadataCaptor.capture(), nameCaptor.capture(), locationCaptor.capture()); + + JsonObject capturedMetadata = metadataCaptor.getValue(); + JsonArray saltsArray = capturedMetadata.getJsonArray("salts"); + assertEquals(1, saltsArray.size(), "Salts array should have exactly one entry, as other is removed in newest-effective logic"); + JsonObject salt = saltsArray.getJsonObject(0); + assertEquals(1740694476392L, salt.getLong("effective"), "Effective timestamp should match second entry"); + assertEquals(10, salt.getInteger("size"), "Size should match second entries"); + + //Now sending snapshot2 to encrypted to verify that does the same. + EncryptedSaltStoreWriter encryptedSaltStoreWriter = new EncryptedSaltStoreWriter(config, rotatingSaltProvider, + fileManager, taggableCloudStorage, versionGenerator, storeScope, rotatingCloudEncryptionKeyProvider, siteId); + + encryptedSaltStoreWriter.upload(snapshot2); + + verify(fileManager,atLeastOnce()).uploadMetadata(metadataCaptor.capture(), nameCaptor.capture(), locationCaptor.capture()); + + capturedMetadata = metadataCaptor.getValue(); + saltsArray = capturedMetadata.getJsonArray("salts"); + salt = saltsArray.getJsonObject(0); + assertEquals(1740694476392L, salt.getLong("effective"), "Effective timestamp should match second entry"); + assertEquals(10, salt.getInteger("size"), "Size should match second entries"); + verify(taggableCloudStorage,atLeastOnce()).upload(pathCaptor.capture(), cloudPathCaptor.capture(), any()); + } }