From 32119f913d437556af156c30cadbf5c3ac709182 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Tue, 3 Dec 2024 14:23:48 -0700 Subject: [PATCH 01/13] Adding the start of what is needed for salt encryption --- .../store/EncryptedScopedStoreReader.java | 9 +--- .../store/RotatingEncryptedSaltProvider.java | 41 +++++++++++++++++++ .../shared/store/RotatingSaltProvider.java | 27 ++++++++---- .../RotatingCloudEncryptionKeyProvider.java | 4 ++ .../store/EncryptedScopedStoreReaderTest.java | 5 +++ 5 files changed, 70 insertions(+), 16 deletions(-) create mode 100644 src/main/java/com/uid2/shared/store/RotatingEncryptedSaltProvider.java diff --git a/src/main/java/com/uid2/shared/store/EncryptedScopedStoreReader.java b/src/main/java/com/uid2/shared/store/EncryptedScopedStoreReader.java index 32b2b5d5..59a5f1e8 100644 --- a/src/main/java/com/uid2/shared/store/EncryptedScopedStoreReader.java +++ b/src/main/java/com/uid2/shared/store/EncryptedScopedStoreReader.java @@ -50,14 +50,7 @@ protected String getDecryptedContent(String encryptedContent) throws Exception { JsonObject json = new JsonObject(encryptedContent); int keyId = json.getInteger("key_id"); String encryptedPayload = json.getString("encrypted_payload"); - Map cloudEncryptionKeys = cloudEncryptionKeyProvider.getAll(); - CloudEncryptionKey decryptionKey = null; - for (CloudEncryptionKey key : cloudEncryptionKeys.values()) { - if (key.getId() == keyId) { - decryptionKey = key; - break; - } - } + CloudEncryptionKey decryptionKey = cloudEncryptionKeyProvider.getKey(keyId); if (decryptionKey == null) { throw new IllegalStateException("No matching S3 key found for decryption for key ID: " + keyId); diff --git a/src/main/java/com/uid2/shared/store/RotatingEncryptedSaltProvider.java b/src/main/java/com/uid2/shared/store/RotatingEncryptedSaltProvider.java new file mode 100644 index 00000000..300bd8cf --- /dev/null +++ b/src/main/java/com/uid2/shared/store/RotatingEncryptedSaltProvider.java @@ -0,0 +1,41 @@ +package com.uid2.shared.store; + +import com.uid2.shared.cloud.DownloadCloudStorage; +import com.uid2.shared.encryption.AesGcm; +import com.uid2.shared.model.CloudEncryptionKey; +import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; + +import io.vertx.core.json.JsonObject; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +public class RotatingEncryptedSaltProvider extends RotatingSaltProvider{ + private final RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider; + + public RotatingEncryptedSaltProvider(DownloadCloudStorage fileStreamProvider, String metadataPath, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) { + super(fileStreamProvider, metadataPath); + this.cloudEncryptionKeyProvider = cloudEncryptionKeyProvider; + } + + @Override + protected String readInputStream(InputStream inputStream) throws IOException { + String encryptedContent = super.readInputStream(inputStream); + + JsonObject json = new JsonObject(encryptedContent); + int keyId = json.getInteger("key_id"); + String encryptedPayload = json.getString("encrypted_payload"); + CloudEncryptionKey decryptionKey = cloudEncryptionKeyProvider.getKey(keyId); + + if (decryptionKey == null) { + throw new IllegalStateException("No matching S3 key found for decryption for key ID: " + keyId); + } + + byte[] secret = Base64.getDecoder().decode(decryptionKey.getSecret()); + byte[] encryptedBytes = Base64.getDecoder().decode(encryptedPayload); + byte[] decryptedBytes = AesGcm.decrypt(encryptedBytes, 0, secret); + + return new String(decryptedBytes, StandardCharsets.UTF_8); + } +} diff --git a/src/main/java/com/uid2/shared/store/RotatingSaltProvider.java b/src/main/java/com/uid2/shared/store/RotatingSaltProvider.java index 57d2b9b1..a723722d 100644 --- a/src/main/java/com/uid2/shared/store/RotatingSaltProvider.java +++ b/src/main/java/com/uid2/shared/store/RotatingSaltProvider.java @@ -13,6 +13,7 @@ import org.hashids.Hashids; import java.io.BufferedReader; +import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; @@ -21,6 +22,7 @@ import java.util.*; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import java.util.stream.Stream; /* 1. metadata.json format @@ -132,20 +134,29 @@ private SaltSnapshot loadSnapshot(JsonObject spec, String firstLevelSalt, SaltEn final String path = spec.getString("location"); int idx = 0; final SaltEntry[] entries = new SaltEntry[spec.getInteger("size")]; - - try (InputStream inputStream = this.contentStreamProvider.download(path); - InputStreamReader inputStreamReader = new InputStreamReader(inputStream, StandardCharsets.UTF_8); - BufferedReader reader = new BufferedReader(inputStreamReader)) { - for (String l; (l = reader.readLine()) != null; ++idx) { - final SaltEntry entry = entryBuilder.toEntry(l); - entries[idx] = entry; - } + Stream stream = readInputStream(this.contentStreamProvider.download(path)).lines(); + for (String l : stream.toList()) { + final SaltEntry entry = entryBuilder.toEntry(l); + entries[idx] = entry; + idx++; } LOGGER.info("Loaded " + idx + " salts"); return new SaltSnapshot(effective, expires, entries, firstLevelSalt); } + protected String readInputStream(InputStream inputStream) throws IOException { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + StringBuilder stringBuilder = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + stringBuilder.append(line); + stringBuilder.append(System.lineSeparator()); + } + return stringBuilder.toString(); + } + } + public static class SaltSnapshot implements ISaltSnapshot { private final Instant effective; private final Instant expires; diff --git a/src/main/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProvider.java b/src/main/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProvider.java index 3716cdd5..6d144fb8 100644 --- a/src/main/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProvider.java +++ b/src/main/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProvider.java @@ -61,6 +61,10 @@ public Map getAll() { return keys != null ? keys : new HashMap<>(); } + public CloudEncryptionKey getKey(int id) { + return reader.getSnapshot().get(id); + } + public void updateSiteToKeysMapping() { Map allKeys = getAll(); siteToKeysMap.clear(); diff --git a/src/test/java/com/uid2/shared/store/EncryptedScopedStoreReaderTest.java b/src/test/java/com/uid2/shared/store/EncryptedScopedStoreReaderTest.java index 3ac00f0a..ed6861bf 100644 --- a/src/test/java/com/uid2/shared/store/EncryptedScopedStoreReaderTest.java +++ b/src/test/java/com/uid2/shared/store/EncryptedScopedStoreReaderTest.java @@ -20,6 +20,8 @@ import static com.uid2.shared.TestUtilites.toInputStream; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -52,6 +54,7 @@ void setUp() { Map mockKeyMap = new HashMap<>(); mockKeyMap.put(encryptionKey.getId(), encryptionKey); when(keyProvider.getAll()).thenReturn(mockKeyMap); + when(keyProvider.getKey(1)).thenReturn(mockKeyMap.get(1)); } @Test @@ -127,6 +130,7 @@ void testDecryptionOfEncryptedContent() throws Exception { void testHandlingInvalidEncryptionKey() throws Exception { // Set key provider to return an empty map when(keyProvider.getAll()).thenReturn(new HashMap<>()); + when(keyProvider.getKey(anyInt())).thenReturn(null); String secretKey = encryptionKey.getSecret(); byte[] secretKeyBytes = Base64.getDecoder().decode(secretKey); @@ -159,6 +163,7 @@ void testLoadWithMultipleEncryptionKeys() throws Exception { mockKeyMap.put(encryptionKey.getId(), encryptionKey); mockKeyMap.put(newKey.getId(), newKey); when(keyProvider.getAll()).thenReturn(mockKeyMap); + when(keyProvider.getKey(2)).thenReturn(mockKeyMap.get(2)); byte[] encryptedPayload = AesGcm.encrypt("value1,value2".getBytes(StandardCharsets.UTF_8), newKeyBytes); String encryptedPayloadBase64 = Base64.getEncoder().encodeToString(encryptedPayload); From 7fe538c41c8dee45e89aac960b955d14e26345c8 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 3 Dec 2024 21:27:04 +0000 Subject: [PATCH 02/13] [CI Pipeline] Released Snapshot version: 8.0.10-alpha-172-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 27fdc4e0..1d3b4a74 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.uid2 uid2-shared - 8.0.9 + 8.0.10-alpha-172-SNAPSHOT ${project.groupId}:${project.artifactId} Library for all the shared uid2 operations https://github.com/IABTechLab/uid2docs From 3a808a2fe11275ebee26ebe953528ca32fc1957e Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Fri, 6 Dec 2024 15:54:23 -0700 Subject: [PATCH 03/13] Added more work --- .../uid2/shared/store/RotatingEncryptedSaltProvider.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/shared/store/RotatingEncryptedSaltProvider.java b/src/main/java/com/uid2/shared/store/RotatingEncryptedSaltProvider.java index 300bd8cf..74fd171c 100644 --- a/src/main/java/com/uid2/shared/store/RotatingEncryptedSaltProvider.java +++ b/src/main/java/com/uid2/shared/store/RotatingEncryptedSaltProvider.java @@ -5,13 +5,15 @@ import com.uid2.shared.model.CloudEncryptionKey; import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; +import com.uid2.shared.store.reader.StoreReader; import io.vertx.core.json.JsonObject; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.Base64; +import java.util.Collection; -public class RotatingEncryptedSaltProvider extends RotatingSaltProvider{ +public class RotatingEncryptedSaltProvider extends RotatingSaltProvider implements StoreReader> { private final RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider; public RotatingEncryptedSaltProvider(DownloadCloudStorage fileStreamProvider, String metadataPath, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) { @@ -38,4 +40,9 @@ protected String readInputStream(InputStream inputStream) throws IOException { return new String(decryptedBytes, StandardCharsets.UTF_8); } + + @Override + public Collection getAll() { + return super.getSnapshots(); + } } From e741256c2f95eecd59e87d03d4ad7e49fe5929ff Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Tue, 10 Dec 2024 13:24:20 -0700 Subject: [PATCH 04/13] Addressing PR comments --- .../store/EncryptedRotatingSaltProvider.java | 33 ++ .../store/EncryptedScopedStoreReader.java | 39 +-- .../shared/store/RotatingSaltProvider.java | 26 +- .../RotatingCloudEncryptionKeyProvider.java | 7 +- .../CloudEncryptionHelpers.java} | 37 +-- .../EncryptedRotatingSaltProviderTest.java | 291 ++++++++++++++++++ .../store/EncryptedScopedStoreReaderTest.java | 20 -- ...otatingCloudEncryptionKeyProviderTest.java | 29 ++ .../util/CloudEncryptionHelperTest.java | 59 ++++ 9 files changed, 448 insertions(+), 93 deletions(-) create mode 100644 src/main/java/com/uid2/shared/store/EncryptedRotatingSaltProvider.java rename src/main/java/com/uid2/shared/{store/RotatingEncryptedSaltProvider.java => util/CloudEncryptionHelpers.java} (51%) create mode 100644 src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java create mode 100644 src/test/java/com/uid2/shared/util/CloudEncryptionHelperTest.java diff --git a/src/main/java/com/uid2/shared/store/EncryptedRotatingSaltProvider.java b/src/main/java/com/uid2/shared/store/EncryptedRotatingSaltProvider.java new file mode 100644 index 00000000..f9473d8f --- /dev/null +++ b/src/main/java/com/uid2/shared/store/EncryptedRotatingSaltProvider.java @@ -0,0 +1,33 @@ +package com.uid2.shared.store; + +import com.uid2.shared.cloud.DownloadCloudStorage; +import com.uid2.shared.model.SaltEntry; +import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Collection; + +import static com.uid2.shared.util.CloudEncryptionHelpers.decryptInputStream; + +public class EncryptedRotatingSaltProvider extends RotatingSaltProvider { + private final RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider; + + public EncryptedRotatingSaltProvider(DownloadCloudStorage fileStreamProvider, String metadataPath, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) { + super(fileStreamProvider, metadataPath); + this.cloudEncryptionKeyProvider = cloudEncryptionKeyProvider; + } + + @Override + protected SaltEntry[] readInputStream(InputStream inputStream, SaltEntryBuilder entryBuilder, Integer size) throws IOException { + String decrypted = decryptInputStream(inputStream, cloudEncryptionKeyProvider); + SaltEntry[] entries = new SaltEntry[size]; + int idx = 0; + for (String line : decrypted.split("\n")) { + final SaltEntry entry = entryBuilder.toEntry(line); + entries[idx] = entry; + idx++; + } + return entries; + } +} diff --git a/src/main/java/com/uid2/shared/store/EncryptedScopedStoreReader.java b/src/main/java/com/uid2/shared/store/EncryptedScopedStoreReader.java index 59a5f1e8..98ebb785 100644 --- a/src/main/java/com/uid2/shared/store/EncryptedScopedStoreReader.java +++ b/src/main/java/com/uid2/shared/store/EncryptedScopedStoreReader.java @@ -1,22 +1,18 @@ package com.uid2.shared.store; import com.uid2.shared.cloud.DownloadCloudStorage; -import com.uid2.shared.model.CloudEncryptionKey; import com.uid2.shared.store.parser.Parser; import com.uid2.shared.store.parser.ParsingResult; import com.uid2.shared.store.scope.StoreScope; import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; -import io.vertx.core.json.JsonObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; -import com.uid2.shared.encryption.AesGcm; - import java.nio.charset.StandardCharsets; -import java.util.Base64; -import java.util.Map; + +import static com.uid2.shared.util.CloudEncryptionHelpers.decryptInputStream; public class EncryptedScopedStoreReader extends ScopedStoreReader { private static final Logger LOGGER = LoggerFactory.getLogger(EncryptedScopedStoreReader.class); @@ -31,8 +27,7 @@ public EncryptedScopedStoreReader(DownloadCloudStorage fileStreamProvider, Store @Override protected long loadContent(String path) throws Exception { try (InputStream inputStream = this.contentStreamProvider.download(path)) { - String encryptedContent = inputStreamToString(inputStream); - String decryptedContent = getDecryptedContent(encryptedContent); + String decryptedContent = decryptInputStream(inputStream, cloudEncryptionKeyProvider); ParsingResult parsed = this.parser.deserialize(new ByteArrayInputStream(decryptedContent.getBytes(StandardCharsets.UTF_8))); latestSnapshot.set(parsed.getData()); @@ -45,32 +40,4 @@ protected long loadContent(String path) throws Exception { throw e; } } - - protected String getDecryptedContent(String encryptedContent) throws Exception { - JsonObject json = new JsonObject(encryptedContent); - int keyId = json.getInteger("key_id"); - String encryptedPayload = json.getString("encrypted_payload"); - CloudEncryptionKey decryptionKey = cloudEncryptionKeyProvider.getKey(keyId); - - if (decryptionKey == null) { - throw new IllegalStateException("No matching S3 key found for decryption for key ID: " + keyId); - } - - byte[] secret = Base64.getDecoder().decode(decryptionKey.getSecret()); - byte[] encryptedBytes = Base64.getDecoder().decode(encryptedPayload); - byte[] decryptedBytes = AesGcm.decrypt(encryptedBytes, 0, secret); - - return new String(decryptedBytes, StandardCharsets.UTF_8); - } - - public static String inputStreamToString(InputStream inputStream) throws IOException { - try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { - StringBuilder stringBuilder = new StringBuilder(); - String line; - while ((line = reader.readLine()) != null) { - stringBuilder.append(line); - } - return stringBuilder.toString(); - } - } } \ No newline at end of file diff --git a/src/main/java/com/uid2/shared/store/RotatingSaltProvider.java b/src/main/java/com/uid2/shared/store/RotatingSaltProvider.java index a723722d..64da06ee 100644 --- a/src/main/java/com/uid2/shared/store/RotatingSaltProvider.java +++ b/src/main/java/com/uid2/shared/store/RotatingSaltProvider.java @@ -132,28 +132,24 @@ private SaltSnapshot loadSnapshot(JsonObject spec, String firstLevelSalt, SaltEn final Instant expires = Instant.ofEpochMilli(spec.getLong("expires", defaultExpires.toEpochMilli())); final String path = spec.getString("location"); - int idx = 0; - final SaltEntry[] entries = new SaltEntry[spec.getInteger("size")]; - Stream stream = readInputStream(this.contentStreamProvider.download(path)).lines(); - for (String l : stream.toList()) { - final SaltEntry entry = entryBuilder.toEntry(l); - entries[idx] = entry; - idx++; - } + Integer size = spec.getInteger("size"); + SaltEntry[] entries = readInputStream(this.contentStreamProvider.download(path), entryBuilder, size); - LOGGER.info("Loaded " + idx + " salts"); + LOGGER.info("Loaded " + size + " salts"); return new SaltSnapshot(effective, expires, entries, firstLevelSalt); } - protected String readInputStream(InputStream inputStream) throws IOException { + protected SaltEntry[] readInputStream(InputStream inputStream, SaltEntryBuilder entryBuilder, Integer size) throws IOException { try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { - StringBuilder stringBuilder = new StringBuilder(); String line; + SaltEntry[] entries = new SaltEntry[size]; + int idx = 0; while ((line = reader.readLine()) != null) { - stringBuilder.append(line); - stringBuilder.append(System.lineSeparator()); + final SaltEntry entry = entryBuilder.toEntry(line); + entries[idx] = entry; + idx++; } - return stringBuilder.toString(); + return entries; } } @@ -225,7 +221,7 @@ public String encode(long id) { } } - static final class SaltEntryBuilder { + protected static final class SaltEntryBuilder { private final IdHashingScheme idHashingScheme; public SaltEntryBuilder(IdHashingScheme idHashingScheme) { diff --git a/src/main/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProvider.java b/src/main/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProvider.java index 6d144fb8..c8075594 100644 --- a/src/main/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProvider.java +++ b/src/main/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProvider.java @@ -62,7 +62,12 @@ public Map getAll() { } public CloudEncryptionKey getKey(int id) { - return reader.getSnapshot().get(id); + Map snapshot = reader.getSnapshot(); + if(snapshot == null) { + return null; + } + + return snapshot.get(id); } public void updateSiteToKeysMapping() { diff --git a/src/main/java/com/uid2/shared/store/RotatingEncryptedSaltProvider.java b/src/main/java/com/uid2/shared/util/CloudEncryptionHelpers.java similarity index 51% rename from src/main/java/com/uid2/shared/store/RotatingEncryptedSaltProvider.java rename to src/main/java/com/uid2/shared/util/CloudEncryptionHelpers.java index 74fd171c..f1037587 100644 --- a/src/main/java/com/uid2/shared/store/RotatingEncryptedSaltProvider.java +++ b/src/main/java/com/uid2/shared/util/CloudEncryptionHelpers.java @@ -1,30 +1,19 @@ -package com.uid2.shared.store; +package com.uid2.shared.util; -import com.uid2.shared.cloud.DownloadCloudStorage; +import java.io.InputStream; import com.uid2.shared.encryption.AesGcm; import com.uid2.shared.model.CloudEncryptionKey; -import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; -import com.uid2.shared.store.reader.StoreReader; +import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; import io.vertx.core.json.JsonObject; -import java.io.IOException; -import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.Base64; -import java.util.Collection; -public class RotatingEncryptedSaltProvider extends RotatingSaltProvider implements StoreReader> { - private final RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider; - - public RotatingEncryptedSaltProvider(DownloadCloudStorage fileStreamProvider, String metadataPath, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) { - super(fileStreamProvider, metadataPath); - this.cloudEncryptionKeyProvider = cloudEncryptionKeyProvider; - } - - @Override - protected String readInputStream(InputStream inputStream) throws IOException { - String encryptedContent = super.readInputStream(inputStream); +import java.io.*; +public class CloudEncryptionHelpers { + public static String decryptInputStream(InputStream inputStream, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) throws IOException { + String encryptedContent = inputStreamToString(inputStream); JsonObject json = new JsonObject(encryptedContent); int keyId = json.getInteger("key_id"); String encryptedPayload = json.getString("encrypted_payload"); @@ -41,8 +30,14 @@ protected String readInputStream(InputStream inputStream) throws IOException { return new String(decryptedBytes, StandardCharsets.UTF_8); } - @Override - public Collection getAll() { - return super.getSnapshots(); + public static String inputStreamToString(InputStream inputStream) throws IOException { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + StringBuilder stringBuilder = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + stringBuilder.append(line); + } + return stringBuilder.toString(); + } } } diff --git a/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java b/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java new file mode 100644 index 00000000..21652050 --- /dev/null +++ b/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java @@ -0,0 +1,291 @@ +package com.uid2.shared.store; + +import com.uid2.shared.cloud.ICloudStorage; +import com.uid2.shared.encryption.AesGcm; +import com.uid2.shared.model.CloudEncryptionKey; +import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +public class EncryptedRotatingSaltProviderTest { + private AutoCloseable mocks; + @Mock + private ICloudStorage cloudStorage; + + @Mock + private RotatingCloudEncryptionKeyProvider keyProvider; + private CloudEncryptionKey encryptionKey; + + @BeforeEach + public void setup() { + + mocks = MockitoAnnotations.openMocks(this); + + byte[] keyBytes = new byte[32]; + new Random().nextBytes(keyBytes); + String base64Key = Base64.getEncoder().encodeToString(keyBytes); + encryptionKey = new CloudEncryptionKey(1, 1, 0, 0, base64Key); + + Map mockKeyMap = new HashMap<>(); + mockKeyMap.put(encryptionKey.getId(), encryptionKey); + when(keyProvider.getAll()).thenReturn(mockKeyMap); + when(keyProvider.getKey(1)).thenReturn(mockKeyMap.get(1)); + } + + @AfterEach + public void teardown() throws Exception { + mocks.close(); + } + + private InputStream getEncryptedStream(String content) { + String secretKey = encryptionKey.getSecret(); + byte[] secretKeyBytes = Base64.getDecoder().decode(secretKey); + byte[] encryptedPayload = AesGcm.encrypt(content.getBytes(StandardCharsets.UTF_8), secretKeyBytes); + String encryptedPayloadBase64 = Base64.getEncoder().encodeToString(encryptedPayload); + + JsonObject encryptedJson = new JsonObject() + .put("key_id", encryptionKey.getId()) + .put("encrypted_payload", encryptedPayloadBase64); + + String encryptedContent = encryptedJson.encodePrettily(); + return new ByteArrayInputStream(encryptedContent.getBytes(StandardCharsets.UTF_8)); + } + + @Test + public void loadSaltSingleVersion() throws Exception { + final String FIRST_LEVEL_SALT = "first_level_salt_value"; + final String ID_PREFIX = "a"; + final String ID_SECRET = "m3yMIcbg9vCaFLJsn4m4PfruZnvAZ72OxmFG5QsGMOw="; + + final Instant generatedTime = Instant.now().minus(1, ChronoUnit.DAYS); + final Instant expireTime = Instant.now().plus(365, ChronoUnit.DAYS); + + final JsonObject metadataJson = new JsonObject(); + { + metadataJson.put("version", 2); + metadataJson.put("generated", generatedTime.getEpochSecond() * 1000L); + metadataJson.put("first_level", FIRST_LEVEL_SALT); + metadataJson.put("id_prefix", ID_PREFIX); + metadataJson.put("id_secret", ID_SECRET); + final JsonArray saltsRefList = new JsonArray(); + { + final JsonObject saltsRef = new JsonObject(); + saltsRef.put("effective", generatedTime.getEpochSecond() * 1000L); + saltsRef.put("expires", expireTime.getEpochSecond() * 1000L); + saltsRef.put("location", "salts.txt"); + saltsRef.put("size", 8); + saltsRefList.add(saltsRef); + } + metadataJson.put("salts", saltsRefList); + } + + final String effectiveTimeString = String.valueOf(generatedTime.getEpochSecond() * 1000L); + final String salts = + "1000000," + effectiveTimeString + ",y5YitNf/KFtceipDz8nqsFVmBZsK3KY7s8bOVM4gMD4=\n" + + "1000001," + effectiveTimeString + ",z1uBoGyyzgna9i0o/r5eiD/wAhDX/2Q/6zX1p6hsF7I=\n" + + "1000002," + effectiveTimeString + ",+a5LPajo7uPfNcc9HH0Tn25b3RnSNZwe8YaAKcyeHaA=\n" + + "1000003," + effectiveTimeString + ",wAL6U+lu9gcMhSEySzWG9RQyoo446zAyGWKTW8VVoVw=\n" + + "1000004," + effectiveTimeString + ",eP9ZvW4igLQZ4QfzlyiXgKYFDZgmGOefaKDLEL0zuwE=\n" + + "1000005," + effectiveTimeString + ",UebesrNN0bQkm/QR7Jx7eav+UDXN5Gbq3zs1fLBMRy0=\n" + + "1000006," + effectiveTimeString + ",MtpALOziEJMtPlCQHk6RHALuWvRvRZpCDBmO0xPAia0=\n" + + "1000007," + effectiveTimeString + ",7tjv+KXaSztTZHEHULacotHQ7IpGBcw6IymoRLObkT4="; + + when(cloudStorage.download("metadata")) + .thenReturn(new ByteArrayInputStream(metadataJson.toString().getBytes(StandardCharsets.US_ASCII))); + when(cloudStorage.download("salts.txt")) + .thenReturn(getEncryptedStream(salts)); + + EncryptedRotatingSaltProvider saltsProvider = new EncryptedRotatingSaltProvider( + cloudStorage, "metadata", keyProvider); + + final JsonObject loadedMetadata = saltsProvider.getMetadata(); + saltsProvider.loadContent(loadedMetadata); + assertEquals(2, saltsProvider.getVersion(loadedMetadata)); + + final ISaltProvider.ISaltSnapshot snapshot = saltsProvider.getSnapshot(Instant.now()); + assertEquals(FIRST_LEVEL_SALT, snapshot.getFirstLevelSalt()); + assertTrue(snapshot.getModifiedSince(Instant.now().minus(1, ChronoUnit.HOURS)).isEmpty()); + } + + @Test + public void loadSaltMultipleVersions() throws Exception { + final String FIRST_LEVEL_SALT = "first_level_salt_value"; + final String ID_PREFIX = "a"; + final String ID_SECRET = "m3yMIcbg9vCaFLJsn4m4PfruZnvAZ72OxmFG5QsGMOw="; + + final Instant generatedTimeV1 = Instant.now().minus(2, ChronoUnit.DAYS); + final Instant expireTimeV1 = Instant.now().plus(365, ChronoUnit.DAYS); + final Instant generatedTimeV2 = Instant.now().minus(1, ChronoUnit.DAYS); + final Instant expireTimeV2 = Instant.now().plus(366, ChronoUnit.DAYS); + + final JsonObject metadataJson = new JsonObject(); + { + metadataJson.put("version", 2); + metadataJson.put("generated", generatedTimeV1.getEpochSecond() * 1000L); + metadataJson.put("first_level", FIRST_LEVEL_SALT); + metadataJson.put("id_prefix", ID_PREFIX); + metadataJson.put("id_secret", ID_SECRET); + final JsonArray saltsRefList = new JsonArray(); + { + final JsonObject saltsRef = new JsonObject(); + saltsRef.put("effective", generatedTimeV1.getEpochSecond() * 1000L); + saltsRef.put("expires", expireTimeV1.getEpochSecond() * 1000L); + saltsRef.put("location", "saltsV1.txt"); + saltsRef.put("size", 8); + saltsRefList.add(saltsRef); + } + { + final JsonObject saltsRef = new JsonObject(); + saltsRef.put("effective", generatedTimeV2.getEpochSecond() * 1000L); + saltsRef.put("expires", expireTimeV2.getEpochSecond() * 1000L); + saltsRef.put("location", "saltsV2.txt"); + saltsRef.put("size", 8); + saltsRefList.add(saltsRef); + } + metadataJson.put("salts", saltsRefList); + } + + final String effectiveTimeStringV1 = String.valueOf(generatedTimeV1.getEpochSecond() * 1000L); + final String saltsV1 = + "1000000," + effectiveTimeStringV1 + ",y5YitNf/KFtceipDz8nqsFVmBZsK3KY7s8bOVM4gMD4=\n" + + "1000001," + effectiveTimeStringV1 + ",z1uBoGyyzgna9i0o/r5eiD/wAhDX/2Q/6zX1p6hsF7I=\n" + + "1000002," + effectiveTimeStringV1 + ",+a5LPajo7uPfNcc9HH0Tn25b3RnSNZwe8YaAKcyeHaA=\n" + + "1000003," + effectiveTimeStringV1 + ",wAL6U+lu9gcMhSEySzWG9RQyoo446zAyGWKTW8VVoVw=\n" + + "1000004," + effectiveTimeStringV1 + ",eP9ZvW4igLQZ4QfzlyiXgKYFDZgmGOefaKDLEL0zuwE=\n" + + "1000005," + effectiveTimeStringV1 + ",UebesrNN0bQkm/QR7Jx7eav+UDXN5Gbq3zs1fLBMRy0=\n" + + "1000006," + effectiveTimeStringV1 + ",MtpALOziEJMtPlCQHk6RHALuWvRvRZpCDBmO0xPAia0=\n" + + "1000007," + effectiveTimeStringV1 + ",7tjv+KXaSztTZHEHULacotHQ7IpGBcw6IymoRLObkT4="; + + // update key 1000002 + final String effectiveTimeStringV2 = String.valueOf(generatedTimeV2.getEpochSecond() * 1000L); + final String saltsV2 = + "1000000," + effectiveTimeStringV1 + ",y5YitNf/KFtceipDz8nqsFVmBZsK3KY7s8bOVM4gMD4=\n" + + "1000001," + effectiveTimeStringV1 + ",z1uBoGyyzgna9i0o/r5eiD/wAhDX/2Q/6zX1p6hsF7I=\n" + + "1000002," + effectiveTimeStringV2 + ",AP73KwZscb1ltQQH/B7fdbHUnMmbJNlRULxzklXUqaA=\n" + + "1000003," + effectiveTimeStringV1 + ",wAL6U+lu9gcMhSEySzWG9RQyoo446zAyGWKTW8VVoVw=\n" + + "1000004," + effectiveTimeStringV1 + ",eP9ZvW4igLQZ4QfzlyiXgKYFDZgmGOefaKDLEL0zuwE=\n" + + "1000005," + effectiveTimeStringV1 + ",UebesrNN0bQkm/QR7Jx7eav+UDXN5Gbq3zs1fLBMRy0=\n" + + "1000006," + effectiveTimeStringV1 + ",MtpALOziEJMtPlCQHk6RHALuWvRvRZpCDBmO0xPAia0=\n" + + "1000007," + effectiveTimeStringV1 + ",7tjv+KXaSztTZHEHULacotHQ7IpGBcw6IymoRLObkT4="; + + when(cloudStorage.download("metadata")) + .thenReturn(new ByteArrayInputStream(metadataJson.toString().getBytes(StandardCharsets.US_ASCII))); + when(cloudStorage.download("saltsV1.txt")) + .thenReturn(getEncryptedStream(saltsV1)); + when(cloudStorage.download("saltsV2.txt")) + .thenReturn(getEncryptedStream(saltsV2)); + + EncryptedRotatingSaltProvider saltsProvider = new EncryptedRotatingSaltProvider( + cloudStorage, "metadata", keyProvider); + + final JsonObject loadedMetadata = saltsProvider.getMetadata(); + saltsProvider.loadContent(loadedMetadata); + assertEquals(2, saltsProvider.getVersion(loadedMetadata)); + + final ISaltProvider.ISaltSnapshot snapshot = saltsProvider.getSnapshot(Instant.now()); + assertEquals(FIRST_LEVEL_SALT, snapshot.getFirstLevelSalt()); + assertTrue(snapshot.getModifiedSince(Instant.now().minus(1, ChronoUnit.HOURS)).isEmpty()); + assertEquals(1, snapshot.getModifiedSince(Instant.now().minus(30, ChronoUnit.HOURS)).size()); + assertEquals(1000002, snapshot.getModifiedSince(Instant.now().minus(30, ChronoUnit.HOURS)).get(0).getId()); + } + + @Test + public void loadSaltMultipleVersionsExpired() throws Exception { + final String FIRST_LEVEL_SALT = "first_level_salt_value"; + final String ID_PREFIX = "a"; + final String ID_SECRET = "m3yMIcbg9vCaFLJsn4m4PfruZnvAZ72OxmFG5QsGMOw="; + + final Instant generatedTimeV1 = Instant.now().minus(3, ChronoUnit.DAYS); + final Instant expireTimeV1 = Instant.now().minus(2, ChronoUnit.DAYS); + final Instant generatedTimeV2 = Instant.now().minus(2, ChronoUnit.DAYS); + final Instant expireTimeV2 = Instant.now().minus(1, ChronoUnit.DAYS); + + final JsonObject metadataJson = new JsonObject(); + { + metadataJson.put("version", 2); + metadataJson.put("generated", generatedTimeV1.getEpochSecond() * 1000L); + metadataJson.put("first_level", FIRST_LEVEL_SALT); + metadataJson.put("id_prefix", ID_PREFIX); + metadataJson.put("id_secret", ID_SECRET); + final JsonArray saltsRefList = new JsonArray(); + { + final JsonObject saltsRef = new JsonObject(); + saltsRef.put("effective", generatedTimeV1.getEpochSecond() * 1000L); + saltsRef.put("expires", expireTimeV1.getEpochSecond() * 1000L); + saltsRef.put("location", "saltsV1.txt"); + saltsRef.put("size", 8); + saltsRefList.add(saltsRef); + } + { + final JsonObject saltsRef = new JsonObject(); + saltsRef.put("effective", generatedTimeV2.getEpochSecond() * 1000L); + saltsRef.put("expires", expireTimeV2.getEpochSecond() * 1000L); + saltsRef.put("location", "saltsV2.txt"); + saltsRef.put("size", 8); + saltsRefList.add(saltsRef); + } + metadataJson.put("salts", saltsRefList); + } + + final String effectiveTimeStringV1 = String.valueOf(generatedTimeV1.getEpochSecond() * 1000L); + final String saltsV1 = + "1000000," + effectiveTimeStringV1 + ",y5YitNf/KFtceipDz8nqsFVmBZsK3KY7s8bOVM4gMD4=\n" + + "1000001," + effectiveTimeStringV1 + ",z1uBoGyyzgna9i0o/r5eiD/wAhDX/2Q/6zX1p6hsF7I=\n" + + "1000002," + effectiveTimeStringV1 + ",+a5LPajo7uPfNcc9HH0Tn25b3RnSNZwe8YaAKcyeHaA=\n" + + "1000003," + effectiveTimeStringV1 + ",wAL6U+lu9gcMhSEySzWG9RQyoo446zAyGWKTW8VVoVw=\n" + + "1000004," + effectiveTimeStringV1 + ",eP9ZvW4igLQZ4QfzlyiXgKYFDZgmGOefaKDLEL0zuwE=\n" + + "1000005," + effectiveTimeStringV1 + ",UebesrNN0bQkm/QR7Jx7eav+UDXN5Gbq3zs1fLBMRy0=\n" + + "1000006," + effectiveTimeStringV1 + ",MtpALOziEJMtPlCQHk6RHALuWvRvRZpCDBmO0xPAia0=\n" + + "1000007," + effectiveTimeStringV1 + ",7tjv+KXaSztTZHEHULacotHQ7IpGBcw6IymoRLObkT4="; + + // update key 1000002 + final String effectiveTimeStringV2 = String.valueOf(generatedTimeV2.getEpochSecond() * 1000L); + final String saltsV2 = + "1000000," + effectiveTimeStringV1 + ",y5YitNf/KFtceipDz8nqsFVmBZsK3KY7s8bOVM4gMD4=\n" + + "1000001," + effectiveTimeStringV1 + ",z1uBoGyyzgna9i0o/r5eiD/wAhDX/2Q/6zX1p6hsF7I=\n" + + "1000002," + effectiveTimeStringV2 + ",AP73KwZscb1ltQQH/B7fdbHUnMmbJNlRULxzklXUqaA=\n" + + "1000003," + effectiveTimeStringV1 + ",wAL6U+lu9gcMhSEySzWG9RQyoo446zAyGWKTW8VVoVw=\n" + + "1000004," + effectiveTimeStringV1 + ",eP9ZvW4igLQZ4QfzlyiXgKYFDZgmGOefaKDLEL0zuwE=\n" + + "1000005," + effectiveTimeStringV1 + ",UebesrNN0bQkm/QR7Jx7eav+UDXN5Gbq3zs1fLBMRy0=\n" + + "1000006," + effectiveTimeStringV1 + ",MtpALOziEJMtPlCQHk6RHALuWvRvRZpCDBmO0xPAia0=\n" + + "1000007," + effectiveTimeStringV1 + ",7tjv+KXaSztTZHEHULacotHQ7IpGBcw6IymoRLObkT4="; + + when(cloudStorage.download("metadata")) + .thenReturn(new ByteArrayInputStream(metadataJson.toString().getBytes(StandardCharsets.US_ASCII))); + when(cloudStorage.download("saltsV1.txt")) + .thenReturn(getEncryptedStream(saltsV1)); + when(cloudStorage.download("saltsV2.txt")) + .thenReturn(getEncryptedStream(saltsV2)); + + EncryptedRotatingSaltProvider saltsProvider = new EncryptedRotatingSaltProvider( + cloudStorage, "metadata", keyProvider); + + final JsonObject loadedMetadata = saltsProvider.getMetadata(); + saltsProvider.loadContent(loadedMetadata); + assertEquals(2, saltsProvider.getVersion(loadedMetadata)); + + final ISaltProvider.ISaltSnapshot snapshot = saltsProvider.getSnapshot(Instant.now()); + assertEquals(FIRST_LEVEL_SALT, snapshot.getFirstLevelSalt()); + assertTrue(snapshot.getModifiedSince(Instant.now().minus(1, ChronoUnit.HOURS)).isEmpty()); + assertEquals(1, snapshot.getModifiedSince(Instant.now().minus(49, ChronoUnit.HOURS)).size()); + assertEquals(1000002, snapshot.getModifiedSince(Instant.now().minus(49, ChronoUnit.HOURS)).get(0).getId()); + } + + +} diff --git a/src/test/java/com/uid2/shared/store/EncryptedScopedStoreReaderTest.java b/src/test/java/com/uid2/shared/store/EncryptedScopedStoreReaderTest.java index ed6861bf..24a13156 100644 --- a/src/test/java/com/uid2/shared/store/EncryptedScopedStoreReaderTest.java +++ b/src/test/java/com/uid2/shared/store/EncryptedScopedStoreReaderTest.java @@ -106,26 +106,6 @@ void raisesExceptionWhenNoDecryptionKeyFound() throws Exception { .hasMessageContaining("No matching S3 key found for decryption"); } - @Test - void testDecryptionOfEncryptedContent() throws Exception { - // Simulate encrypted content - String secretKey = encryptionKey.getSecret(); - byte[] secretKeyBytes = Base64.getDecoder().decode(secretKey); - byte[] encryptedPayload = AesGcm.encrypt("value1,value2".getBytes(StandardCharsets.UTF_8), secretKeyBytes); - String encryptedPayloadBase64 = Base64.getEncoder().encodeToString(encryptedPayload); - - JsonObject encryptedJson = new JsonObject() - .put("key_id", encryptionKey.getId()) - .put("encrypted_payload", encryptedPayloadBase64); - - String encryptedContent = encryptedJson.encodePrettily(); - EncryptedScopedStoreReader> reader = new EncryptedScopedStoreReader<>(storage, scope, parser, dataType, keyProvider); - - String decryptedContent = reader.getDecryptedContent(encryptedContent); - - assertThat(decryptedContent).isEqualTo("value1,value2"); - } - @Test void testHandlingInvalidEncryptionKey() throws Exception { // Set key provider to return an empty map diff --git a/src/test/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProviderTest.java b/src/test/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProviderTest.java index f12c52ae..c2c94308 100644 --- a/src/test/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProviderTest.java +++ b/src/test/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProviderTest.java @@ -135,6 +135,35 @@ void testGetAllNullSnapshot() { assertTrue(keys.isEmpty()); } + @Test + void testGet() { + Map expectedKeys = new HashMap<>(); + CloudEncryptionKey key1 = new CloudEncryptionKey(1, 123, 1687635529, 1687808329, "S3keySecretByteHere1"); + CloudEncryptionKey key2 = new CloudEncryptionKey(2, 123, 1687808429, 1687808329, "S3keySecretByteHere2"); + expectedKeys.put(1, key1); + expectedKeys.put(2, key2); + when(reader.getSnapshot()).thenReturn(expectedKeys); + + CloudEncryptionKey key = rotatingCloudEncryptionKeyProvider.getKey(1); + assertEquals(key1, key); + } + + @Test + void testGetEmpty() { + when(reader.getSnapshot()).thenReturn(new HashMap<>()); + + CloudEncryptionKey key = rotatingCloudEncryptionKeyProvider.getKey(1); + assertNull(key); + } + + @Test + void testGetNullSnapshot() { + when(reader.getSnapshot()).thenReturn(null); + + CloudEncryptionKey key = rotatingCloudEncryptionKeyProvider.getKey(1); + assertNull(key); + } + @Test void testUpdateExistingKey() throws Exception { Map existingKeys = new HashMap<>(); diff --git a/src/test/java/com/uid2/shared/util/CloudEncryptionHelperTest.java b/src/test/java/com/uid2/shared/util/CloudEncryptionHelperTest.java new file mode 100644 index 00000000..047b3d12 --- /dev/null +++ b/src/test/java/com/uid2/shared/util/CloudEncryptionHelperTest.java @@ -0,0 +1,59 @@ +package com.uid2.shared.util; + +import com.uid2.shared.encryption.AesGcm; +import com.uid2.shared.model.CloudEncryptionKey; +import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; +import io.vertx.core.json.JsonObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.*; + +import static com.uid2.shared.util.CloudEncryptionHelpers.decryptInputStream; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class CloudEncryptionHelperTest { + private RotatingCloudEncryptionKeyProvider keyProvider; + private CloudEncryptionKey encryptionKey; + + @BeforeEach + void setUp() { + keyProvider = mock(RotatingCloudEncryptionKeyProvider.class); + + // Generate a valid 32-byte AES key + byte[] keyBytes = new byte[32]; + new Random().nextBytes(keyBytes); + String base64Key = Base64.getEncoder().encodeToString(keyBytes); + encryptionKey = new CloudEncryptionKey(1, 1, 0, 0, base64Key); + + Map mockKeyMap = new HashMap<>(); + mockKeyMap.put(encryptionKey.getId(), encryptionKey); + when(keyProvider.getAll()).thenReturn(mockKeyMap); + when(keyProvider.getKey(1)).thenReturn(mockKeyMap.get(1)); + } + + @Test + void testDecryptionOfEncryptedContent() throws Exception { + // Simulate encrypted content + String secretKey = encryptionKey.getSecret(); + byte[] secretKeyBytes = Base64.getDecoder().decode(secretKey); + byte[] encryptedPayload = AesGcm.encrypt("value1,value2".getBytes(StandardCharsets.UTF_8), secretKeyBytes); + String encryptedPayloadBase64 = Base64.getEncoder().encodeToString(encryptedPayload); + + JsonObject encryptedJson = new JsonObject() + .put("key_id", encryptionKey.getId()) + .put("encrypted_payload", encryptedPayloadBase64); + + String encryptedContent = encryptedJson.encodePrettily(); + + InputStream encryptedInputStream = new ByteArrayInputStream(encryptedContent.getBytes(StandardCharsets.UTF_8)); + + String decryptedContent = decryptInputStream(encryptedInputStream, keyProvider); + + assertThat(decryptedContent).isEqualTo("value1,value2"); + } +} From b2e9b68035016d2e2b9e76c479130f409b6fedb4 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 10 Dec 2024 20:27:15 +0000 Subject: [PATCH 05/13] [CI Pipeline] Released Snapshot version: 8.0.11-alpha-173-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1d3b4a74..17981f6d 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.uid2 uid2-shared - 8.0.10-alpha-172-SNAPSHOT + 8.0.11-alpha-173-SNAPSHOT ${project.groupId}:${project.artifactId} Library for all the shared uid2 operations https://github.com/IABTechLab/uid2docs From 1a23d0d256d885cf289d85b949a7d03f179adc6b Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Wed, 11 Dec 2024 14:57:23 -0700 Subject: [PATCH 06/13] Had to adjust to work with 1mil salts --- .../shared/util/CloudEncryptionHelpers.java | 42 +++++++++------ .../EncryptedRotatingSaltProviderTest.java | 51 +++++++++++++++++++ 2 files changed, 77 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/uid2/shared/util/CloudEncryptionHelpers.java b/src/main/java/com/uid2/shared/util/CloudEncryptionHelpers.java index f1037587..af31c037 100644 --- a/src/main/java/com/uid2/shared/util/CloudEncryptionHelpers.java +++ b/src/main/java/com/uid2/shared/util/CloudEncryptionHelpers.java @@ -1,6 +1,10 @@ package com.uid2.shared.util; import java.io.InputStream; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; import com.uid2.shared.encryption.AesGcm; import com.uid2.shared.model.CloudEncryptionKey; @@ -13,10 +17,27 @@ public class CloudEncryptionHelpers { public static String decryptInputStream(InputStream inputStream, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) throws IOException { - String encryptedContent = inputStreamToString(inputStream); - JsonObject json = new JsonObject(encryptedContent); - int keyId = json.getInteger("key_id"); - String encryptedPayload = json.getString("encrypted_payload"); + JsonFactory factory = new JsonFactory(); + JsonParser parser = factory.createParser(inputStream); + int keyId = -1; + byte[] encryptedPayload = null; + parser.nextToken(); + while (parser.nextToken() != JsonToken.END_OBJECT) { + String fieldName = parser.getCurrentName(); + if(fieldName.equals("key_id")) { + parser.nextToken(); + keyId = parser.getIntValue(); + } + if(fieldName.equals("encrypted_payload")) { + parser.nextToken(); + encryptedPayload = parser.getBinaryValue(); + } + } + + if(keyId == -1 || encryptedPayload == null) { + throw new IllegalStateException("failed to parse json"); + } + CloudEncryptionKey decryptionKey = cloudEncryptionKeyProvider.getKey(keyId); if (decryptionKey == null) { @@ -24,20 +45,9 @@ public static String decryptInputStream(InputStream inputStream, RotatingCloudEn } byte[] secret = Base64.getDecoder().decode(decryptionKey.getSecret()); - byte[] encryptedBytes = Base64.getDecoder().decode(encryptedPayload); + byte[] encryptedBytes = encryptedPayload; byte[] decryptedBytes = AesGcm.decrypt(encryptedBytes, 0, secret); return new String(decryptedBytes, StandardCharsets.UTF_8); } - - public static String inputStreamToString(InputStream inputStream) throws IOException { - try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { - StringBuilder stringBuilder = new StringBuilder(); - String line; - while ((line = reader.readLine()) != null) { - stringBuilder.append(line); - } - return stringBuilder.toString(); - } - } } diff --git a/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java b/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java index 21652050..e6294b24 100644 --- a/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java +++ b/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java @@ -123,6 +123,57 @@ public void loadSaltSingleVersion() throws Exception { assertTrue(snapshot.getModifiedSince(Instant.now().minus(1, ChronoUnit.HOURS)).isEmpty()); } + @Test + public void loadSaltSingleVersion1mil() throws Exception { + final String FIRST_LEVEL_SALT = "first_level_salt_value"; + final String ID_PREFIX = "a"; + final String ID_SECRET = "m3yMIcbg9vCaFLJsn4m4PfruZnvAZ72OxmFG5QsGMOw="; + + final Instant generatedTime = Instant.now().minus(1, ChronoUnit.DAYS); + final Instant expireTime = Instant.now().plus(365, ChronoUnit.DAYS); + + final JsonObject metadataJson = new JsonObject(); + { + metadataJson.put("version", 2); + metadataJson.put("generated", generatedTime.getEpochSecond() * 1000L); + metadataJson.put("first_level", FIRST_LEVEL_SALT); + metadataJson.put("id_prefix", ID_PREFIX); + metadataJson.put("id_secret", ID_SECRET); + final JsonArray saltsRefList = new JsonArray(); + { + final JsonObject saltsRef = new JsonObject(); + saltsRef.put("effective", generatedTime.getEpochSecond() * 1000L); + saltsRef.put("expires", expireTime.getEpochSecond() * 1000L); + saltsRef.put("location", "salts.txt"); + saltsRef.put("size", 1000000); + saltsRefList.add(saltsRef); + } + metadataJson.put("salts", saltsRefList); + } + + final String effectiveTimeString = String.valueOf(generatedTime.getEpochSecond() * 1000L); + StringBuilder salts = new StringBuilder(); + for (int i = 0; i < 1000000; i++) { + salts.append(i).append(",").append(effectiveTimeString).append(",").append("salt-string").append("\n"); + } + + when(cloudStorage.download("metadata")) + .thenReturn(new ByteArrayInputStream(metadataJson.toString().getBytes(StandardCharsets.US_ASCII))); + when(cloudStorage.download("salts.txt")) + .thenReturn(getEncryptedStream(salts.toString())); + + EncryptedRotatingSaltProvider saltsProvider = new EncryptedRotatingSaltProvider( + cloudStorage, "metadata", keyProvider); + + final JsonObject loadedMetadata = saltsProvider.getMetadata(); + saltsProvider.loadContent(loadedMetadata); + assertEquals(2, saltsProvider.getVersion(loadedMetadata)); + + final ISaltProvider.ISaltSnapshot snapshot = saltsProvider.getSnapshot(Instant.now()); + assertEquals(FIRST_LEVEL_SALT, snapshot.getFirstLevelSalt()); + assertTrue(snapshot.getModifiedSince(Instant.now().minus(1, ChronoUnit.HOURS)).isEmpty()); + } + @Test public void loadSaltMultipleVersions() throws Exception { final String FIRST_LEVEL_SALT = "first_level_salt_value"; From 65abd71149e7dfa85eb09e9f88702c9d68b78dad Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 11 Dec 2024 22:00:59 +0000 Subject: [PATCH 07/13] [CI Pipeline] Released Snapshot version: 8.0.12-alpha-174-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 17981f6d..21ad6ceb 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.uid2 uid2-shared - 8.0.11-alpha-173-SNAPSHOT + 8.0.12-alpha-174-SNAPSHOT ${project.groupId}:${project.artifactId} Library for all the shared uid2 operations https://github.com/IABTechLab/uid2docs From e66cd22fea7a934b6d38caac7949d3507c4e8c02 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Wed, 11 Dec 2024 15:47:44 -0700 Subject: [PATCH 08/13] Adding encrypted version of ClientSideKeypairStore --- .../shared/store/reader/RotatingClientSideKeypairStore.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/uid2/shared/store/reader/RotatingClientSideKeypairStore.java b/src/main/java/com/uid2/shared/store/reader/RotatingClientSideKeypairStore.java index 148538a3..43bf9b17 100644 --- a/src/main/java/com/uid2/shared/store/reader/RotatingClientSideKeypairStore.java +++ b/src/main/java/com/uid2/shared/store/reader/RotatingClientSideKeypairStore.java @@ -3,6 +3,7 @@ import com.uid2.shared.cloud.DownloadCloudStorage; import com.uid2.shared.model.ClientSideKeypair; import com.uid2.shared.store.CloudPath; +import com.uid2.shared.store.EncryptedScopedStoreReader; import com.uid2.shared.store.IClientSideKeypairStore; import com.uid2.shared.store.ScopedStoreReader; import com.uid2.shared.store.parser.ClientSideKeypairParser; @@ -19,6 +20,10 @@ public RotatingClientSideKeypairStore(DownloadCloudStorage fileStreamProvider, S this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new ClientSideKeypairParser(), "client_side_keypairs"); } + public RotatingClientSideKeypairStore(DownloadCloudStorage fileStreamProvider, StoreScope scope, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) { + this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new ClientSideKeypairParser(), "client_side_keypairs", cloudEncryptionKeyProvider); + } + @Override public long getVersion(JsonObject metadata) { return metadata.getLong("version"); From 9f9b37bfa8619f5bc805bd43e7d5e4a6d98c5541 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 11 Dec 2024 22:52:07 +0000 Subject: [PATCH 09/13] [CI Pipeline] Released Snapshot version: 8.0.13-alpha-175-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 21ad6ceb..75014062 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.uid2 uid2-shared - 8.0.12-alpha-174-SNAPSHOT + 8.0.13-alpha-175-SNAPSHOT ${project.groupId}:${project.artifactId} Library for all the shared uid2 operations https://github.com/IABTechLab/uid2docs From b9f29ed9911654d38b99d4af4b8aac38ffa7b243 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Thu, 12 Dec 2024 11:40:32 -0700 Subject: [PATCH 10/13] Have the scope resolve the metadata path instead --- .../store/EncryptedRotatingSaltProvider.java | 6 +++-- .../EncryptedRotatingSaltProviderTest.java | 25 +++++++++++++------ 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/uid2/shared/store/EncryptedRotatingSaltProvider.java b/src/main/java/com/uid2/shared/store/EncryptedRotatingSaltProvider.java index f9473d8f..95b94731 100644 --- a/src/main/java/com/uid2/shared/store/EncryptedRotatingSaltProvider.java +++ b/src/main/java/com/uid2/shared/store/EncryptedRotatingSaltProvider.java @@ -1,8 +1,10 @@ package com.uid2.shared.store; +import com.uid2.shared.Const; import com.uid2.shared.cloud.DownloadCloudStorage; import com.uid2.shared.model.SaltEntry; import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; +import com.uid2.shared.store.scope.StoreScope; import java.io.IOException; import java.io.InputStream; @@ -13,8 +15,8 @@ public class EncryptedRotatingSaltProvider extends RotatingSaltProvider { private final RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider; - public EncryptedRotatingSaltProvider(DownloadCloudStorage fileStreamProvider, String metadataPath, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) { - super(fileStreamProvider, metadataPath); + public EncryptedRotatingSaltProvider(DownloadCloudStorage fileStreamProvider, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider, StoreScope scope) { + super(fileStreamProvider, scope.getMetadataPath().toString()); this.cloudEncryptionKeyProvider = cloudEncryptionKeyProvider; } diff --git a/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java b/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java index e6294b24..79964648 100644 --- a/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java +++ b/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java @@ -4,6 +4,7 @@ import com.uid2.shared.encryption.AesGcm; import com.uid2.shared.model.CloudEncryptionKey; import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; +import com.uid2.shared.store.scope.EncryptedScope; import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; import org.junit.jupiter.api.AfterEach; @@ -67,6 +68,14 @@ private InputStream getEncryptedStream(String content) { return new ByteArrayInputStream(encryptedContent.getBytes(StandardCharsets.UTF_8)); } + @Test + public void metadataPath() { + EncryptedRotatingSaltProvider saltsProvider = new EncryptedRotatingSaltProvider( + cloudStorage, keyProvider, new EncryptedScope(new CloudPath("salts/metadata.json"), 1, true)); + + assertEquals("salts/encrypted/1_public/metadata.json", saltsProvider.getMetadataPath()); + } + @Test public void loadSaltSingleVersion() throws Exception { final String FIRST_LEVEL_SALT = "first_level_salt_value"; @@ -106,13 +115,13 @@ public void loadSaltSingleVersion() throws Exception { "1000006," + effectiveTimeString + ",MtpALOziEJMtPlCQHk6RHALuWvRvRZpCDBmO0xPAia0=\n" + "1000007," + effectiveTimeString + ",7tjv+KXaSztTZHEHULacotHQ7IpGBcw6IymoRLObkT4="; - when(cloudStorage.download("metadata")) + when(cloudStorage.download("sites/encrypted/1_public/metadata.json")) .thenReturn(new ByteArrayInputStream(metadataJson.toString().getBytes(StandardCharsets.US_ASCII))); when(cloudStorage.download("salts.txt")) .thenReturn(getEncryptedStream(salts)); EncryptedRotatingSaltProvider saltsProvider = new EncryptedRotatingSaltProvider( - cloudStorage, "metadata", keyProvider); + cloudStorage, keyProvider, new EncryptedScope(new CloudPath("sites/metadata.json"), 1, true)); final JsonObject loadedMetadata = saltsProvider.getMetadata(); saltsProvider.loadContent(loadedMetadata); @@ -157,13 +166,13 @@ public void loadSaltSingleVersion1mil() throws Exception { salts.append(i).append(",").append(effectiveTimeString).append(",").append("salt-string").append("\n"); } - when(cloudStorage.download("metadata")) + when(cloudStorage.download("sites/encrypted/1_public/metadata.json")) .thenReturn(new ByteArrayInputStream(metadataJson.toString().getBytes(StandardCharsets.US_ASCII))); when(cloudStorage.download("salts.txt")) .thenReturn(getEncryptedStream(salts.toString())); EncryptedRotatingSaltProvider saltsProvider = new EncryptedRotatingSaltProvider( - cloudStorage, "metadata", keyProvider); + cloudStorage, keyProvider, new EncryptedScope(new CloudPath("sites/metadata.json"), 1, true)); final JsonObject loadedMetadata = saltsProvider.getMetadata(); saltsProvider.loadContent(loadedMetadata); @@ -235,7 +244,7 @@ public void loadSaltMultipleVersions() throws Exception { "1000006," + effectiveTimeStringV1 + ",MtpALOziEJMtPlCQHk6RHALuWvRvRZpCDBmO0xPAia0=\n" + "1000007," + effectiveTimeStringV1 + ",7tjv+KXaSztTZHEHULacotHQ7IpGBcw6IymoRLObkT4="; - when(cloudStorage.download("metadata")) + when(cloudStorage.download("sites/encrypted/1_public/metadata.json")) .thenReturn(new ByteArrayInputStream(metadataJson.toString().getBytes(StandardCharsets.US_ASCII))); when(cloudStorage.download("saltsV1.txt")) .thenReturn(getEncryptedStream(saltsV1)); @@ -243,7 +252,7 @@ public void loadSaltMultipleVersions() throws Exception { .thenReturn(getEncryptedStream(saltsV2)); EncryptedRotatingSaltProvider saltsProvider = new EncryptedRotatingSaltProvider( - cloudStorage, "metadata", keyProvider); + cloudStorage, keyProvider, new EncryptedScope(new CloudPath("sites/metadata.json"), 1, true)); final JsonObject loadedMetadata = saltsProvider.getMetadata(); saltsProvider.loadContent(loadedMetadata); @@ -317,7 +326,7 @@ public void loadSaltMultipleVersionsExpired() throws Exception { "1000006," + effectiveTimeStringV1 + ",MtpALOziEJMtPlCQHk6RHALuWvRvRZpCDBmO0xPAia0=\n" + "1000007," + effectiveTimeStringV1 + ",7tjv+KXaSztTZHEHULacotHQ7IpGBcw6IymoRLObkT4="; - when(cloudStorage.download("metadata")) + when(cloudStorage.download("sites/encrypted/1_public/metadata.json")) .thenReturn(new ByteArrayInputStream(metadataJson.toString().getBytes(StandardCharsets.US_ASCII))); when(cloudStorage.download("saltsV1.txt")) .thenReturn(getEncryptedStream(saltsV1)); @@ -325,7 +334,7 @@ public void loadSaltMultipleVersionsExpired() throws Exception { .thenReturn(getEncryptedStream(saltsV2)); EncryptedRotatingSaltProvider saltsProvider = new EncryptedRotatingSaltProvider( - cloudStorage, "metadata", keyProvider); + cloudStorage, keyProvider, new EncryptedScope(new CloudPath("sites/metadata.json"), 1, true)); final JsonObject loadedMetadata = saltsProvider.getMetadata(); saltsProvider.loadContent(loadedMetadata); From 995166778032f2a3e74c3c980564cf1a531ea750 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 12 Dec 2024 18:44:23 +0000 Subject: [PATCH 11/13] [CI Pipeline] Released Snapshot version: 8.0.14-alpha-176-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 75014062..1572142e 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.uid2 uid2-shared - 8.0.13-alpha-175-SNAPSHOT + 8.0.14-alpha-176-SNAPSHOT ${project.groupId}:${project.artifactId} Library for all the shared uid2 operations https://github.com/IABTechLab/uid2docs From 445f111baa760541e2b9ac377c16c3b49392d782 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Fri, 13 Dec 2024 12:33:31 -0700 Subject: [PATCH 12/13] Updating rotating cloud encryptionKeyProvider --- .../store/reader/RotatingCloudEncryptionKeyProvider.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProvider.java b/src/main/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProvider.java index c8075594..76a4423c 100644 --- a/src/main/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProvider.java +++ b/src/main/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProvider.java @@ -24,7 +24,7 @@ import java.time.Instant; public class RotatingCloudEncryptionKeyProvider implements StoreReader> { - ScopedStoreReader> reader; + protected ScopedStoreReader> reader; private static final Logger LOGGER = LoggerFactory.getLogger(RotatingCloudEncryptionKeyProvider.class); public Map> siteToKeysMap = new HashMap<>(); @@ -33,6 +33,11 @@ public RotatingCloudEncryptionKeyProvider(DownloadCloudStorage fileStreamProvide this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new CloudEncryptionKeyParser(), "cloud_encryption_keys"); } + + public RotatingCloudEncryptionKeyProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope, ScopedStoreReader> reader) { + this.reader = reader; + } + @Override public JsonObject getMetadata() throws Exception { return reader.getMetadata(); From 73cf98401fb71ef5d3f2282c9f4516d24ad994a3 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 13 Dec 2024 19:37:02 +0000 Subject: [PATCH 13/13] [CI Pipeline] Released Snapshot version: 8.0.15-alpha-177-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1572142e..e76bf544 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.uid2 uid2-shared - 8.0.14-alpha-176-SNAPSHOT + 8.0.15-alpha-177-SNAPSHOT ${project.groupId}:${project.artifactId} Library for all the shared uid2 operations https://github.com/IABTechLab/uid2docs