diff --git a/src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java index aa2180ed..53f56223 100644 --- a/src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java +++ b/src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java @@ -6,7 +6,6 @@ import com.uid2.shared.cloud.TaggableCloudStorage; import com.uid2.shared.encryption.AesGcm; import com.uid2.shared.model.CloudEncryptionKey; -import com.uid2.shared.model.SaltEntry; import com.uid2.shared.store.CloudPath; import com.uid2.shared.store.salt.RotatingSaltProvider; import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; @@ -15,10 +14,7 @@ import org.slf4j.LoggerFactory; import io.vertx.core.json.JsonObject; -import java.io.BufferedWriter; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.*; public class EncryptedSaltStoreWriter extends SaltStoreWriter implements StoreWriter { @@ -37,6 +33,19 @@ public EncryptedSaltStoreWriter(JsonObject config, RotatingSaltProvider provider this.siteId = siteId; } + @Override + public void upload(Object data, JsonObject extraMeta) throws Exception { + this.unEncryptedMetadataData = extraMeta; + @SuppressWarnings("unchecked") + List snapshots = new ArrayList<>((Collection) data); + this.buildAndUploadMetadata(snapshots); + } + + @Override + public void rewriteMeta() throws Exception { + // No rewrite_metadata endpoint for salts + } + @Override protected java.lang.String getSaltSnapshotLocation(RotatingSaltProvider.SaltSnapshot snapshot) { return scope.resolve(new CloudPath("salts.txt." + snapshot.getEffective().toEpochMilli())).toString(); @@ -82,13 +91,16 @@ protected boolean tryUploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snaps return false; } } - StringBuilder stringBuilder = new StringBuilder(); - for (SaltEntry entry: snapshot.getAllRotatingSalts()) { - stringBuilder.append(entry.id()).append(",").append(entry.lastUpdated()).append(",").append(entry.currentSalt()).append("\n"); - } + var saltCsv = SaltSerializer.toCsv(snapshot.getAllRotatingSalts()); + var encryptedSaltCsv = addEncryptedEnvelope(encryptionKey, saltCsv); + uploadSaltsFile(location, encryptedSaltCsv); + + LOGGER.info("File encryption completed for site_id={} key_id={} store={}", siteId, encryptionKey.getId(), "salts"); + return true; + } - String data = stringBuilder.toString(); + private String addEncryptedEnvelope(CloudEncryptionKey encryptionKey, String data) { JsonObject encryptedJson = new JsonObject(); if (encryptionKey != null) { byte[] secret = Base64.getDecoder().decode(encryptionKey.getSecret()); @@ -100,13 +112,7 @@ protected boolean tryUploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snaps throw new IllegalStateException("No Cloud Encryption keys available for encryption for site ID: " + siteId); } - final Path newSaltsFile = Files.createTempFile("salts", ".txt"); - try (BufferedWriter w = Files.newBufferedWriter(newSaltsFile)) { - w.write(encryptedJson.encodePrettily()); - } - this.upload(newSaltsFile.toString(), location); - LOGGER.info("File encryption completed for site_id={} key_id={} store={}", siteId, encryptionKey.getId(), "salts"); - return true; + return encryptedJson.encodePrettily(); } @Override @@ -118,17 +124,4 @@ protected JsonObject getMetadata(){ protected Long getMetadataVersion() throws Exception { return this.unEncryptedMetadataData.getLong("version"); } - - @Override - public void upload(Object data, JsonObject extraMeta) throws Exception { - this.unEncryptedMetadataData = extraMeta; - @SuppressWarnings("unchecked") - List snapshots = new ArrayList<>((Collection) data); - this.buildAndUploadMetadata(snapshots); - } - - @Override - public void rewriteMeta() throws Exception { - - } } diff --git a/src/main/java/com/uid2/admin/store/writer/SaltSerializer.java b/src/main/java/com/uid2/admin/store/writer/SaltSerializer.java new file mode 100644 index 00000000..26b6f68e --- /dev/null +++ b/src/main/java/com/uid2/admin/store/writer/SaltSerializer.java @@ -0,0 +1,56 @@ +package com.uid2.admin.store.writer; + +import com.uid2.shared.model.SaltEntry; + +public final class SaltSerializer { + private SaltSerializer() {} + + public static String toCsv(SaltEntry[] entries) { + StringBuilder stringBuilder = new StringBuilder(); + + for (SaltEntry entry : entries) { + addLine(entry, stringBuilder); + } + + return stringBuilder.toString(); + } + + private static void addLine(SaltEntry entry, StringBuilder stringBuilder) { + stringBuilder + .append(entry.id()) + .append(",") + .append(entry.lastUpdated()) + .append(",") + .append(entry.currentSalt()); + + stringBuilder.append(","); + stringBuilder.append(serializeNullable(entry.refreshFrom())); + + stringBuilder.append(","); + stringBuilder.append(serializeNullable(entry.previousSalt())); + + appendKey(stringBuilder, entry.currentKey()); + appendKey(stringBuilder, entry.previousKey()); + + stringBuilder.append("\n"); + } + + private static void appendKey(StringBuilder stringBuilder, SaltEntry.KeyMaterial key) { + if (key != null) { + stringBuilder + .append(",") + .append(key.id()) + .append(",") + .append(serializeNullable(key.key())) + .append(",") + .append(serializeNullable(key.salt())); + } + else { + stringBuilder.append(",,,"); + } + } + + public static String serializeNullable(T obj) { + return obj == null ? "" : obj.toString(); + } +} diff --git a/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java index a3557830..f1bc34d8 100644 --- a/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java +++ b/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java @@ -2,7 +2,6 @@ import com.uid2.admin.store.FileManager; import com.uid2.admin.store.version.VersionGenerator; -import com.uid2.shared.Utils; import com.uid2.shared.cloud.CloudStorageException; import com.uid2.shared.cloud.TaggableCloudStorage; import com.uid2.shared.model.SaltEntry; @@ -14,7 +13,6 @@ import org.slf4j.LoggerFactory; import java.io.BufferedWriter; -import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; @@ -22,7 +20,6 @@ import java.util.Comparator; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import java.util.stream.Stream; public class SaltStoreWriter { @@ -45,6 +42,30 @@ public SaltStoreWriter(JsonObject config, RotatingSaltProvider provider, FileMan this.versionGenerator = versionGenerator; } + public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception { + this.buildAndUploadMetadata(this.getSnapshots(data)); + refreshProvider(); + } + + /** + * reads the metadata file, and marks each referenced file as ready for archiving + */ + public void archiveSaltLocations() throws Exception { + final JsonObject metadata = provider.getMetadata(); + + metadata.getJsonArray("salts").forEach(instance -> { + try { + JsonObject salt = (JsonObject) instance; + String location = salt.getString("location", ""); + if (!location.isBlank()) { + this.setStatusTagToObsolete(location); + } + } catch (Exception ex) { + LOGGER.error("Error marking object as ready for archiving", ex); + } + }); + } + private List getSnapshots(RotatingSaltProvider.SaltSnapshot data){ if (provider.getSnapshots() == null) { throw new IllegalStateException("Snapshots cannot be null"); @@ -126,6 +147,7 @@ protected void buildAndUploadMetadata(List sn protected JsonObject enrichMetadata(JsonObject metadata){ return metadata; } + /** * Builds snapshot metadata and uploads snapshots if they need to be updated. *

@@ -151,34 +173,10 @@ private JsonArray uploadAndGetSnapshotsMetadata(List { - try { - JsonObject salt = (JsonObject) instance; - String location = salt.getString("location", ""); - if (!location.isBlank()) { - this.setStatusTagToObsolete(location); - } - } catch (Exception ex) { - LOGGER.error("Error marking object as ready for archiving", ex); - } - }); - } - protected String getSaltSnapshotLocation(RotatingSaltProvider.SaltSnapshot snapshot) { return saltSnapshotLocationPrefix + snapshot.getEffective().toEpochMilli(); } @@ -200,14 +198,18 @@ protected boolean tryUploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snaps return false; } - final Path newSaltsFile = Files.createTempFile("operators", ".txt"); + var saltCsv = SaltSerializer.toCsv(snapshot.getAllRotatingSalts()); + uploadSaltsFile(location, saltCsv); + + return true; + } + + protected void uploadSaltsFile(String location, String data) throws Exception { + final Path newSaltsFile = Files.createTempFile("salts", ".txt"); try (BufferedWriter w = Files.newBufferedWriter(newSaltsFile)) { - for (SaltEntry entry : snapshot.getAllRotatingSalts()) { - w.write(entry.id() + "," + entry.lastUpdated() + "," + entry.currentSalt() + "\n"); - } + w.write(data); } this.upload(newSaltsFile.toString(), location); - return true; } protected void upload(String data, String location) throws Exception { diff --git a/src/test/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriterTest.java b/src/test/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriterTest.java index cc412389..18f1c4bf 100644 --- a/src/test/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriterTest.java +++ b/src/test/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriterTest.java @@ -6,6 +6,7 @@ import com.uid2.shared.cloud.TaggableCloudStorage; import com.uid2.shared.model.CloudEncryptionKey; import com.uid2.shared.model.SaltEntry; +import com.uid2.shared.model.SaltEntry.KeyMaterial; import com.uid2.shared.store.CloudPath; import com.uid2.shared.store.salt.RotatingSaltProvider; import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; @@ -26,11 +27,12 @@ import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Paths; +import java.time.Duration; import java.time.Instant; import java.util.*; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; +import static java.lang.Long.parseLong; +import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; import static com.uid2.shared.util.CloudEncryptionHelpers.decryptInputStream; @@ -38,6 +40,10 @@ @MockitoSettings(strictness = Strictness.LENIENT) public class EncryptedSaltStoreWriterTest { private static final Integer SITE_ID = 1; + private final Instant feb26 = Instant.parse("2025-02-26T22:12:18Z"); + private final Instant feb27 = Instant.parse("2025-02-27T22:14:36Z"); + private final Instant mar23 = Instant.parse("2025-03-23T14:52:08Z"); + private final Instant mar25 = Instant.parse("2025-03-25T14:52:08Z"); @Mock private FileManager fileManager; @@ -81,34 +87,24 @@ public void setup() throws Exception { when(rotatingCloudEncryptionKeyProvider.getEncryptionKeyForSite(SITE_ID)).thenReturn(encryptionKey); } - private RotatingSaltProvider.SaltSnapshot makeSnapshot(Instant effective, Instant expires, int nsalts) { - SaltEntry[] entries = new SaltEntry[nsalts]; - for (int i = 0; i < entries.length; ++i) { - entries[i] = new SaltEntry(i, "hashed_id", effective.toEpochMilli(), "salt", null, null, null, null); - } - return new RotatingSaltProvider.SaltSnapshot(effective, expires, entries, "test_first_level_salt"); - } - - private void verifyFile(String filelocation, RotatingSaltProvider.SaltSnapshot snapshot) throws IOException { - InputStream encoded = Files.newInputStream(Paths.get(filelocation)); - String contents = decryptInputStream(encoded, rotatingCloudEncryptionKeyProvider, "salts"); - SaltEntry[] entries = snapshot.getAllRotatingSalts(); - int idx = 0; - for (String line : contents.split("\n")) { - String[] entrySplit = line.split(","); - assertEquals(entries[idx].id(), Long.parseLong(entrySplit[0])); - assertEquals(entries[idx].currentSalt(), entrySplit[2]); - idx++; - } - } - @Test public void testUploadNew() throws Exception { - RotatingSaltProvider.SaltSnapshot snapshot = makeSnapshot(Instant.ofEpochMilli(1740607938167L), Instant.ofEpochMilli(Instant.now().toEpochMilli() + 90002), 100); - RotatingSaltProvider.SaltSnapshot snapshot2 = makeSnapshot(Instant.ofEpochMilli(1740694476392L), Instant.ofEpochMilli(Instant.now().toEpochMilli() + 130000), 10); + RotatingSaltProvider.SaltSnapshot olderSnapshot = makeSnapshot( + feb26, + Instant.now().plus(Duration.ofSeconds(90)), + 100 + ); + + RotatingSaltProvider.SaltSnapshot activeSnapshot = makeSnapshot( + feb27, + Instant.now().plus(Duration.ofMinutes(2)), + 10 + ); + + JsonObject metadata = new JsonObject() - .put("version", 1742770328863L) - .put("generated", 1742770328) + .put("version", mar23.toEpochMilli()) + .put("generated", mar23.getEpochSecond()) .put("first_level", "FIRST-LEVEL") .put("id_prefix", "a") .put("id_secret", "ID-SECRET"); @@ -124,7 +120,7 @@ public void testUploadNew() throws Exception { EncryptedSaltStoreWriter encryptedSaltStoreWriter = new EncryptedSaltStoreWriter(config, rotatingSaltProvider, fileManager, taggableCloudStorage, versionGenerator, storeScope, rotatingCloudEncryptionKeyProvider, SITE_ID); - encryptedSaltStoreWriter.upload(List.of(snapshot,snapshot2), metadata); + encryptedSaltStoreWriter.upload(List.of(olderSnapshot,activeSnapshot), metadata); verify(fileManager).uploadMetadata(metadataCaptor.capture(), nameCaptor.capture(), locationCaptor.capture()); // Capture the metadata @@ -135,13 +131,23 @@ public void testUploadNew() throws Exception { assertEquals(capturedMetadata.getString("id_prefix"), metadata.getValue("id_prefix")); verify(taggableCloudStorage,times(2)).upload(pathCaptor.capture(), cloudPathCaptor.capture(), any()); - verifyFile(pathCaptor.getValue(), snapshot); + assertWrittenFileEquals(pathCaptor.getValue(), activeSnapshot); } @Test public void testUnencryptedAndEncryptedBehavesTheSame() throws Exception { - RotatingSaltProvider.SaltSnapshot snapshot = makeSnapshot(Instant.ofEpochMilli(1740607938167L), Instant.ofEpochMilli(Instant.now().toEpochMilli() + 90000), 100); - RotatingSaltProvider.SaltSnapshot snapshot2 = makeSnapshot(Instant.ofEpochMilli(1740694476392L), Instant.ofEpochMilli(Instant.now().toEpochMilli() + 130000), 10); + RotatingSaltProvider.SaltSnapshot snapshot = makeSnapshot( + feb26, + Instant.now().plus(Duration.ofSeconds(90)), + 100 + ); + + RotatingSaltProvider.SaltSnapshot snapshot2 = makeSnapshot( + feb27, + Instant.now().plus(Duration.ofMinutes(2)), + 10 + ); + List snapshots = List.of(snapshot, snapshot2); when(rotatingSaltProvider.getMetadata()).thenThrow(new CloudStorageException("The specified key does not exist: AmazonS3Exception: test-core-bucket")); @@ -164,7 +170,11 @@ public void testUnencryptedAndEncryptedBehavesTheSame() throws Exception { JsonArray saltsArray = capturedMetadata.getJsonArray("salts"); assertEquals(1, saltsArray.size(), "Salts array should have exactly one entry, as other is removed in newest-effective logic"); JsonObject salt = saltsArray.getJsonObject(0); - assertEquals(1740694476392L, salt.getLong("effective"), "Effective timestamp should match second entry"); + assertEquals( + feb27.toEpochMilli(), + salt.getLong("effective"), + "Effective timestamp should match second entry" + ); assertEquals(10, salt.getInteger("size"), "Size should match second entries"); //Now sending snapshot2 to encrypted to verify that does the same. @@ -172,8 +182,8 @@ public void testUnencryptedAndEncryptedBehavesTheSame() throws Exception { fileManager, taggableCloudStorage, versionGenerator, storeScope, rotatingCloudEncryptionKeyProvider, SITE_ID); JsonObject metadata = new JsonObject() - .put("version", 1742770328863L) - .put("generated", 1742770328) + .put("version", mar25.toEpochMilli()) + .put("generated", mar25.getEpochSecond()) .put("first_level", "FIRST-LEVEL") .put("id_prefix", "a") .put("id_secret", "ID-SECRET"); @@ -187,8 +197,56 @@ public void testUnencryptedAndEncryptedBehavesTheSame() throws Exception { saltsArray = capturedMetadata.getJsonArray("salts"); salt = saltsArray.getJsonObject(0); assertEquals(1, key_id); - assertEquals(1740694476392L, salt.getLong("effective"), "Effective timestamp should match second entry"); + assertEquals( + feb27.toEpochMilli(), + salt.getLong("effective"), + "Effective timestamp should match second entry" + ); assertEquals(10, salt.getInteger("size"), "Size should match second entries"); verify(taggableCloudStorage,atLeastOnce()).upload(pathCaptor.capture(), cloudPathCaptor.capture(), any()); } + + private RotatingSaltProvider.SaltSnapshot makeSnapshot(Instant effective, Instant expires, int nsalts) { + SaltEntry[] entries = new SaltEntry[nsalts]; + + for (int i = 0; i < entries.length; ++i) { + entries[i] = new SaltEntry( + i, + "hashed_id", + effective.toEpochMilli(), + "salt", + 1000L, + "previous salt", + new KeyMaterial(1, "key 1", "key salt 1"), + new KeyMaterial(2, "key 2", "key salt 2") + ); + } + return new RotatingSaltProvider.SaltSnapshot(effective, expires, entries, "test_first_level_salt"); + } + + private void assertWrittenFileEquals(String fileLocation, RotatingSaltProvider.SaltSnapshot snapshot) throws IOException { + InputStream encoded = Files.newInputStream(Paths.get(fileLocation)); + String contents = decryptInputStream(encoded, rotatingCloudEncryptionKeyProvider, "salts"); + SaltEntry[] entries = snapshot.getAllRotatingSalts(); + var lines = contents.split("\n"); + for (int i = 0; i < lines.length; i++) { + var line = lines[i]; + var entry = entries[i]; + String[] fields = line.split(","); + + assertAll( + () -> assertEquals(entry.id(), parseLong(fields[0])), + () -> assertEquals(entry.lastUpdated(), parseLong(fields[1])), + () -> assertEquals(entry.currentSalt(), fields[2]), + () -> assertEquals(entry.refreshFrom(), parseLong(fields[3])), + () -> assertEquals(entry.previousSalt(), fields[4]), + () -> assertEquals(entry.currentKey().id(), parseLong(fields[5])), + () -> assertEquals(entry.currentKey().key(), fields[6]), + () -> assertEquals(entry.currentKey().salt(), fields[7]), + () -> assertEquals(entry.previousKey().id(), parseLong(fields[8])), + () -> assertEquals(entry.previousKey().key(), fields[9]), + () -> assertEquals(entry.previousKey().salt(), fields[10]) + ); + } + } } diff --git a/src/test/java/com/uid2/admin/store/writer/SaltSerializerTest.java b/src/test/java/com/uid2/admin/store/writer/SaltSerializerTest.java new file mode 100644 index 00000000..58e29148 --- /dev/null +++ b/src/test/java/com/uid2/admin/store/writer/SaltSerializerTest.java @@ -0,0 +1,121 @@ +package com.uid2.admin.store.writer; + +import com.uid2.shared.model.SaltEntry; +import com.uid2.shared.model.SaltEntry.KeyMaterial; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +class SaltSerializerTest { + @Test + void toCsv_serializesNoSalts() { + var expected = ""; + + var salts = new SaltEntry[]{}; + var actual = SaltSerializer.toCsv(salts); + + assertThat(actual).isEqualTo(expected); + } + + @Test + void toCsv_serializesSaltsWithNoOptionalFields() { + var expected = """ +1,100,salt1,,,,,,,, +2,200,salt2,,,,,,,, +"""; + + var salts = new SaltEntry[]{ + new SaltEntry(1, "hashedId1", 100, "salt1", null, null, null, null), + new SaltEntry(2, "hashedId2", 200, "salt2", null, null, null, null), + }; + var actual = SaltSerializer.toCsv(salts); + + assertThat(actual).isEqualTo(expected); + } + + @Test + void toCsv_serializesSaltsWithAllOptionalFields() { + var expected = """ +1,100,salt1,1000,previousSalt1,11,key1,keySalt1,111,key11,keySalt11 +2,200,salt2,2000,previousSalt2,22,key2,keySalt2,222,key22,keySalt22 +"""; + + var salts = new SaltEntry[]{ + new SaltEntry(1, + "hashedId1", + 100, + "salt1", + 1000L, + "previousSalt1", + new KeyMaterial(11, "key1", "keySalt1"), + new KeyMaterial(111, "key11", "keySalt11") + ), + new SaltEntry(2, + "hashedId2", + 200, + "salt2", + 2000L, + "previousSalt2", + new KeyMaterial(22, "key2", "keySalt2"), + new KeyMaterial(222, "key22", "keySalt22") + ), + }; + var actual = SaltSerializer.toCsv(salts); + + assertThat(actual).isEqualTo(expected); + } + + @Test + void toCsv_serializesSaltsWithV3IdentityMapFields() { + var expected = """ +1,100,salt1,1000,previousSalt1,,,,,, +2,200,salt2,2000,previousSalt2,,,,,, +"""; + + var salts = new SaltEntry[]{ + new SaltEntry(1, + "hashedId1", + 100, + "salt1", + 1000L, + "previousSalt1", + null, + null + ), + new SaltEntry(2, + "hashedId2", + 200, + "salt2", + 2000L, + "previousSalt2", + null, + null + ), + }; + var actual = SaltSerializer.toCsv(salts); + + assertThat(actual).isEqualTo(expected); + } + + @Test + void toCsv_toleratesNullsInKeys() { + var expected = """ +1,100,salt1,1000,previousSalt1,11,,,111,, +"""; + + var salts = new SaltEntry[]{ + new SaltEntry(1, + "hashedId1", + 100, + "salt1", + 1000L, + "previousSalt1", + new KeyMaterial(11, null, null), + new KeyMaterial(111, null, null) + ), + }; + var actual = SaltSerializer.toCsv(salts); + + assertThat(actual).isEqualTo(expected); + } +} \ No newline at end of file