diff --git a/pom.xml b/pom.xml index 27fdc4e0..e76bf544 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.uid2 uid2-shared - 8.0.9 + 8.0.15-alpha-177-SNAPSHOT ${project.groupId}:${project.artifactId} Library for all the shared uid2 operations https://github.com/IABTechLab/uid2docs diff --git a/src/main/java/com/uid2/shared/store/EncryptedRotatingSaltProvider.java b/src/main/java/com/uid2/shared/store/EncryptedRotatingSaltProvider.java new file mode 100644 index 00000000..95b94731 --- /dev/null +++ b/src/main/java/com/uid2/shared/store/EncryptedRotatingSaltProvider.java @@ -0,0 +1,35 @@ +package com.uid2.shared.store; + +import com.uid2.shared.Const; +import com.uid2.shared.cloud.DownloadCloudStorage; +import com.uid2.shared.model.SaltEntry; +import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; +import com.uid2.shared.store.scope.StoreScope; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Collection; + +import static com.uid2.shared.util.CloudEncryptionHelpers.decryptInputStream; + +public class EncryptedRotatingSaltProvider extends RotatingSaltProvider { + private final RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider; + + public EncryptedRotatingSaltProvider(DownloadCloudStorage fileStreamProvider, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider, StoreScope scope) { + super(fileStreamProvider, scope.getMetadataPath().toString()); + this.cloudEncryptionKeyProvider = cloudEncryptionKeyProvider; + } + + @Override + protected SaltEntry[] readInputStream(InputStream inputStream, SaltEntryBuilder entryBuilder, Integer size) throws IOException { + String decrypted = decryptInputStream(inputStream, cloudEncryptionKeyProvider); + SaltEntry[] entries = new SaltEntry[size]; + int idx = 0; + for (String line : decrypted.split("\n")) { + final SaltEntry entry = entryBuilder.toEntry(line); + entries[idx] = entry; + idx++; + } + return entries; + } +} diff --git a/src/main/java/com/uid2/shared/store/EncryptedScopedStoreReader.java b/src/main/java/com/uid2/shared/store/EncryptedScopedStoreReader.java index 32b2b5d5..98ebb785 100644 --- a/src/main/java/com/uid2/shared/store/EncryptedScopedStoreReader.java +++ b/src/main/java/com/uid2/shared/store/EncryptedScopedStoreReader.java @@ -1,22 +1,18 @@ package com.uid2.shared.store; import com.uid2.shared.cloud.DownloadCloudStorage; -import com.uid2.shared.model.CloudEncryptionKey; import com.uid2.shared.store.parser.Parser; import com.uid2.shared.store.parser.ParsingResult; import com.uid2.shared.store.scope.StoreScope; import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; -import io.vertx.core.json.JsonObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; -import com.uid2.shared.encryption.AesGcm; - import java.nio.charset.StandardCharsets; -import java.util.Base64; -import java.util.Map; + +import static com.uid2.shared.util.CloudEncryptionHelpers.decryptInputStream; public class EncryptedScopedStoreReader extends ScopedStoreReader { private static final Logger LOGGER = LoggerFactory.getLogger(EncryptedScopedStoreReader.class); @@ -31,8 +27,7 @@ public EncryptedScopedStoreReader(DownloadCloudStorage fileStreamProvider, Store @Override protected long loadContent(String path) throws Exception { try (InputStream inputStream = this.contentStreamProvider.download(path)) { - String encryptedContent = inputStreamToString(inputStream); - String decryptedContent = getDecryptedContent(encryptedContent); + String decryptedContent = decryptInputStream(inputStream, cloudEncryptionKeyProvider); ParsingResult parsed = this.parser.deserialize(new ByteArrayInputStream(decryptedContent.getBytes(StandardCharsets.UTF_8))); latestSnapshot.set(parsed.getData()); @@ -45,39 +40,4 @@ protected long loadContent(String path) throws Exception { throw e; } } - - protected String getDecryptedContent(String encryptedContent) throws Exception { - JsonObject json = new JsonObject(encryptedContent); - int keyId = json.getInteger("key_id"); - String encryptedPayload = json.getString("encrypted_payload"); - Map cloudEncryptionKeys = cloudEncryptionKeyProvider.getAll(); - CloudEncryptionKey decryptionKey = null; - for (CloudEncryptionKey key : cloudEncryptionKeys.values()) { - if (key.getId() == keyId) { - decryptionKey = key; - break; - } - } - - if (decryptionKey == null) { - throw new IllegalStateException("No matching S3 key found for decryption for key ID: " + keyId); - } - - byte[] secret = Base64.getDecoder().decode(decryptionKey.getSecret()); - byte[] encryptedBytes = Base64.getDecoder().decode(encryptedPayload); - byte[] decryptedBytes = AesGcm.decrypt(encryptedBytes, 0, secret); - - return new String(decryptedBytes, StandardCharsets.UTF_8); - } - - public static String inputStreamToString(InputStream inputStream) throws IOException { - try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { - StringBuilder stringBuilder = new StringBuilder(); - String line; - while ((line = reader.readLine()) != null) { - stringBuilder.append(line); - } - return stringBuilder.toString(); - } - } } \ No newline at end of file diff --git a/src/main/java/com/uid2/shared/store/RotatingSaltProvider.java b/src/main/java/com/uid2/shared/store/RotatingSaltProvider.java index 57d2b9b1..64da06ee 100644 --- a/src/main/java/com/uid2/shared/store/RotatingSaltProvider.java +++ b/src/main/java/com/uid2/shared/store/RotatingSaltProvider.java @@ -13,6 +13,7 @@ import org.hashids.Hashids; import java.io.BufferedReader; +import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; @@ -21,6 +22,7 @@ import java.util.*; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import java.util.stream.Stream; /* 1. metadata.json format @@ -130,20 +132,25 @@ private SaltSnapshot loadSnapshot(JsonObject spec, String firstLevelSalt, SaltEn final Instant expires = Instant.ofEpochMilli(spec.getLong("expires", defaultExpires.toEpochMilli())); final String path = spec.getString("location"); - int idx = 0; - final SaltEntry[] entries = new SaltEntry[spec.getInteger("size")]; - - try (InputStream inputStream = this.contentStreamProvider.download(path); - InputStreamReader inputStreamReader = new InputStreamReader(inputStream, StandardCharsets.UTF_8); - BufferedReader reader = new BufferedReader(inputStreamReader)) { - for (String l; (l = reader.readLine()) != null; ++idx) { - final SaltEntry entry = entryBuilder.toEntry(l); + Integer size = spec.getInteger("size"); + SaltEntry[] entries = readInputStream(this.contentStreamProvider.download(path), entryBuilder, size); + + LOGGER.info("Loaded " + size + " salts"); + return new SaltSnapshot(effective, expires, entries, firstLevelSalt); + } + + protected SaltEntry[] readInputStream(InputStream inputStream, SaltEntryBuilder entryBuilder, Integer size) throws IOException { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + String line; + SaltEntry[] entries = new SaltEntry[size]; + int idx = 0; + while ((line = reader.readLine()) != null) { + final SaltEntry entry = entryBuilder.toEntry(line); entries[idx] = entry; + idx++; } + return entries; } - - LOGGER.info("Loaded " + idx + " salts"); - return new SaltSnapshot(effective, expires, entries, firstLevelSalt); } public static class SaltSnapshot implements ISaltSnapshot { @@ -214,7 +221,7 @@ public String encode(long id) { } } - static final class SaltEntryBuilder { + protected static final class SaltEntryBuilder { private final IdHashingScheme idHashingScheme; public SaltEntryBuilder(IdHashingScheme idHashingScheme) { diff --git a/src/main/java/com/uid2/shared/store/reader/RotatingClientSideKeypairStore.java b/src/main/java/com/uid2/shared/store/reader/RotatingClientSideKeypairStore.java index 148538a3..43bf9b17 100644 --- a/src/main/java/com/uid2/shared/store/reader/RotatingClientSideKeypairStore.java +++ b/src/main/java/com/uid2/shared/store/reader/RotatingClientSideKeypairStore.java @@ -3,6 +3,7 @@ import com.uid2.shared.cloud.DownloadCloudStorage; import com.uid2.shared.model.ClientSideKeypair; import com.uid2.shared.store.CloudPath; +import com.uid2.shared.store.EncryptedScopedStoreReader; import com.uid2.shared.store.IClientSideKeypairStore; import com.uid2.shared.store.ScopedStoreReader; import com.uid2.shared.store.parser.ClientSideKeypairParser; @@ -19,6 +20,10 @@ public RotatingClientSideKeypairStore(DownloadCloudStorage fileStreamProvider, S this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new ClientSideKeypairParser(), "client_side_keypairs"); } + public RotatingClientSideKeypairStore(DownloadCloudStorage fileStreamProvider, StoreScope scope, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) { + this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new ClientSideKeypairParser(), "client_side_keypairs", cloudEncryptionKeyProvider); + } + @Override public long getVersion(JsonObject metadata) { return metadata.getLong("version"); diff --git a/src/main/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProvider.java b/src/main/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProvider.java index 3716cdd5..76a4423c 100644 --- a/src/main/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProvider.java +++ b/src/main/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProvider.java @@ -24,7 +24,7 @@ import java.time.Instant; public class RotatingCloudEncryptionKeyProvider implements StoreReader> { - ScopedStoreReader> reader; + protected ScopedStoreReader> reader; private static final Logger LOGGER = LoggerFactory.getLogger(RotatingCloudEncryptionKeyProvider.class); public Map> siteToKeysMap = new HashMap<>(); @@ -33,6 +33,11 @@ public RotatingCloudEncryptionKeyProvider(DownloadCloudStorage fileStreamProvide this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new CloudEncryptionKeyParser(), "cloud_encryption_keys"); } + + public RotatingCloudEncryptionKeyProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope, ScopedStoreReader> reader) { + this.reader = reader; + } + @Override public JsonObject getMetadata() throws Exception { return reader.getMetadata(); @@ -61,6 +66,15 @@ public Map getAll() { return keys != null ? keys : new HashMap<>(); } + public CloudEncryptionKey getKey(int id) { + Map snapshot = reader.getSnapshot(); + if(snapshot == null) { + return null; + } + + return snapshot.get(id); + } + public void updateSiteToKeysMapping() { Map allKeys = getAll(); siteToKeysMap.clear(); diff --git a/src/main/java/com/uid2/shared/util/CloudEncryptionHelpers.java b/src/main/java/com/uid2/shared/util/CloudEncryptionHelpers.java new file mode 100644 index 00000000..af31c037 --- /dev/null +++ b/src/main/java/com/uid2/shared/util/CloudEncryptionHelpers.java @@ -0,0 +1,53 @@ +package com.uid2.shared.util; + +import java.io.InputStream; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.uid2.shared.encryption.AesGcm; +import com.uid2.shared.model.CloudEncryptionKey; + +import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; +import io.vertx.core.json.JsonObject; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +import java.io.*; + +public class CloudEncryptionHelpers { + public static String decryptInputStream(InputStream inputStream, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) throws IOException { + JsonFactory factory = new JsonFactory(); + JsonParser parser = factory.createParser(inputStream); + int keyId = -1; + byte[] encryptedPayload = null; + parser.nextToken(); + while (parser.nextToken() != JsonToken.END_OBJECT) { + String fieldName = parser.getCurrentName(); + if(fieldName.equals("key_id")) { + parser.nextToken(); + keyId = parser.getIntValue(); + } + if(fieldName.equals("encrypted_payload")) { + parser.nextToken(); + encryptedPayload = parser.getBinaryValue(); + } + } + + if(keyId == -1 || encryptedPayload == null) { + throw new IllegalStateException("failed to parse json"); + } + + CloudEncryptionKey decryptionKey = cloudEncryptionKeyProvider.getKey(keyId); + + if (decryptionKey == null) { + throw new IllegalStateException("No matching S3 key found for decryption for key ID: " + keyId); + } + + byte[] secret = Base64.getDecoder().decode(decryptionKey.getSecret()); + byte[] encryptedBytes = encryptedPayload; + byte[] decryptedBytes = AesGcm.decrypt(encryptedBytes, 0, secret); + + return new String(decryptedBytes, StandardCharsets.UTF_8); + } +} diff --git a/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java b/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java new file mode 100644 index 00000000..79964648 --- /dev/null +++ b/src/test/java/com/uid2/shared/store/EncryptedRotatingSaltProviderTest.java @@ -0,0 +1,351 @@ +package com.uid2.shared.store; + +import com.uid2.shared.cloud.ICloudStorage; +import com.uid2.shared.encryption.AesGcm; +import com.uid2.shared.model.CloudEncryptionKey; +import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; +import com.uid2.shared.store.scope.EncryptedScope; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +public class EncryptedRotatingSaltProviderTest { + private AutoCloseable mocks; + @Mock + private ICloudStorage cloudStorage; + + @Mock + private RotatingCloudEncryptionKeyProvider keyProvider; + private CloudEncryptionKey encryptionKey; + + @BeforeEach + public void setup() { + + mocks = MockitoAnnotations.openMocks(this); + + byte[] keyBytes = new byte[32]; + new Random().nextBytes(keyBytes); + String base64Key = Base64.getEncoder().encodeToString(keyBytes); + encryptionKey = new CloudEncryptionKey(1, 1, 0, 0, base64Key); + + Map mockKeyMap = new HashMap<>(); + mockKeyMap.put(encryptionKey.getId(), encryptionKey); + when(keyProvider.getAll()).thenReturn(mockKeyMap); + when(keyProvider.getKey(1)).thenReturn(mockKeyMap.get(1)); + } + + @AfterEach + public void teardown() throws Exception { + mocks.close(); + } + + private InputStream getEncryptedStream(String content) { + String secretKey = encryptionKey.getSecret(); + byte[] secretKeyBytes = Base64.getDecoder().decode(secretKey); + byte[] encryptedPayload = AesGcm.encrypt(content.getBytes(StandardCharsets.UTF_8), secretKeyBytes); + String encryptedPayloadBase64 = Base64.getEncoder().encodeToString(encryptedPayload); + + JsonObject encryptedJson = new JsonObject() + .put("key_id", encryptionKey.getId()) + .put("encrypted_payload", encryptedPayloadBase64); + + String encryptedContent = encryptedJson.encodePrettily(); + return new ByteArrayInputStream(encryptedContent.getBytes(StandardCharsets.UTF_8)); + } + + @Test + public void metadataPath() { + EncryptedRotatingSaltProvider saltsProvider = new EncryptedRotatingSaltProvider( + cloudStorage, keyProvider, new EncryptedScope(new CloudPath("salts/metadata.json"), 1, true)); + + assertEquals("salts/encrypted/1_public/metadata.json", saltsProvider.getMetadataPath()); + } + + @Test + public void loadSaltSingleVersion() throws Exception { + final String FIRST_LEVEL_SALT = "first_level_salt_value"; + final String ID_PREFIX = "a"; + final String ID_SECRET = "m3yMIcbg9vCaFLJsn4m4PfruZnvAZ72OxmFG5QsGMOw="; + + final Instant generatedTime = Instant.now().minus(1, ChronoUnit.DAYS); + final Instant expireTime = Instant.now().plus(365, ChronoUnit.DAYS); + + final JsonObject metadataJson = new JsonObject(); + { + metadataJson.put("version", 2); + metadataJson.put("generated", generatedTime.getEpochSecond() * 1000L); + metadataJson.put("first_level", FIRST_LEVEL_SALT); + metadataJson.put("id_prefix", ID_PREFIX); + metadataJson.put("id_secret", ID_SECRET); + final JsonArray saltsRefList = new JsonArray(); + { + final JsonObject saltsRef = new JsonObject(); + saltsRef.put("effective", generatedTime.getEpochSecond() * 1000L); + saltsRef.put("expires", expireTime.getEpochSecond() * 1000L); + saltsRef.put("location", "salts.txt"); + saltsRef.put("size", 8); + saltsRefList.add(saltsRef); + } + metadataJson.put("salts", saltsRefList); + } + + final String effectiveTimeString = String.valueOf(generatedTime.getEpochSecond() * 1000L); + final String salts = + "1000000," + effectiveTimeString + ",y5YitNf/KFtceipDz8nqsFVmBZsK3KY7s8bOVM4gMD4=\n" + + "1000001," + effectiveTimeString + ",z1uBoGyyzgna9i0o/r5eiD/wAhDX/2Q/6zX1p6hsF7I=\n" + + "1000002," + effectiveTimeString + ",+a5LPajo7uPfNcc9HH0Tn25b3RnSNZwe8YaAKcyeHaA=\n" + + "1000003," + effectiveTimeString + ",wAL6U+lu9gcMhSEySzWG9RQyoo446zAyGWKTW8VVoVw=\n" + + "1000004," + effectiveTimeString + ",eP9ZvW4igLQZ4QfzlyiXgKYFDZgmGOefaKDLEL0zuwE=\n" + + "1000005," + effectiveTimeString + ",UebesrNN0bQkm/QR7Jx7eav+UDXN5Gbq3zs1fLBMRy0=\n" + + "1000006," + effectiveTimeString + ",MtpALOziEJMtPlCQHk6RHALuWvRvRZpCDBmO0xPAia0=\n" + + "1000007," + effectiveTimeString + ",7tjv+KXaSztTZHEHULacotHQ7IpGBcw6IymoRLObkT4="; + + when(cloudStorage.download("sites/encrypted/1_public/metadata.json")) + .thenReturn(new ByteArrayInputStream(metadataJson.toString().getBytes(StandardCharsets.US_ASCII))); + when(cloudStorage.download("salts.txt")) + .thenReturn(getEncryptedStream(salts)); + + EncryptedRotatingSaltProvider saltsProvider = new EncryptedRotatingSaltProvider( + cloudStorage, keyProvider, new EncryptedScope(new CloudPath("sites/metadata.json"), 1, true)); + + final JsonObject loadedMetadata = saltsProvider.getMetadata(); + saltsProvider.loadContent(loadedMetadata); + assertEquals(2, saltsProvider.getVersion(loadedMetadata)); + + final ISaltProvider.ISaltSnapshot snapshot = saltsProvider.getSnapshot(Instant.now()); + assertEquals(FIRST_LEVEL_SALT, snapshot.getFirstLevelSalt()); + assertTrue(snapshot.getModifiedSince(Instant.now().minus(1, ChronoUnit.HOURS)).isEmpty()); + } + + @Test + public void loadSaltSingleVersion1mil() throws Exception { + final String FIRST_LEVEL_SALT = "first_level_salt_value"; + final String ID_PREFIX = "a"; + final String ID_SECRET = "m3yMIcbg9vCaFLJsn4m4PfruZnvAZ72OxmFG5QsGMOw="; + + final Instant generatedTime = Instant.now().minus(1, ChronoUnit.DAYS); + final Instant expireTime = Instant.now().plus(365, ChronoUnit.DAYS); + + final JsonObject metadataJson = new JsonObject(); + { + metadataJson.put("version", 2); + metadataJson.put("generated", generatedTime.getEpochSecond() * 1000L); + metadataJson.put("first_level", FIRST_LEVEL_SALT); + metadataJson.put("id_prefix", ID_PREFIX); + metadataJson.put("id_secret", ID_SECRET); + final JsonArray saltsRefList = new JsonArray(); + { + final JsonObject saltsRef = new JsonObject(); + saltsRef.put("effective", generatedTime.getEpochSecond() * 1000L); + saltsRef.put("expires", expireTime.getEpochSecond() * 1000L); + saltsRef.put("location", "salts.txt"); + saltsRef.put("size", 1000000); + saltsRefList.add(saltsRef); + } + metadataJson.put("salts", saltsRefList); + } + + final String effectiveTimeString = String.valueOf(generatedTime.getEpochSecond() * 1000L); + StringBuilder salts = new StringBuilder(); + for (int i = 0; i < 1000000; i++) { + salts.append(i).append(",").append(effectiveTimeString).append(",").append("salt-string").append("\n"); + } + + when(cloudStorage.download("sites/encrypted/1_public/metadata.json")) + .thenReturn(new ByteArrayInputStream(metadataJson.toString().getBytes(StandardCharsets.US_ASCII))); + when(cloudStorage.download("salts.txt")) + .thenReturn(getEncryptedStream(salts.toString())); + + EncryptedRotatingSaltProvider saltsProvider = new EncryptedRotatingSaltProvider( + cloudStorage, keyProvider, new EncryptedScope(new CloudPath("sites/metadata.json"), 1, true)); + + final JsonObject loadedMetadata = saltsProvider.getMetadata(); + saltsProvider.loadContent(loadedMetadata); + assertEquals(2, saltsProvider.getVersion(loadedMetadata)); + + final ISaltProvider.ISaltSnapshot snapshot = saltsProvider.getSnapshot(Instant.now()); + assertEquals(FIRST_LEVEL_SALT, snapshot.getFirstLevelSalt()); + assertTrue(snapshot.getModifiedSince(Instant.now().minus(1, ChronoUnit.HOURS)).isEmpty()); + } + + @Test + public void loadSaltMultipleVersions() throws Exception { + final String FIRST_LEVEL_SALT = "first_level_salt_value"; + final String ID_PREFIX = "a"; + final String ID_SECRET = "m3yMIcbg9vCaFLJsn4m4PfruZnvAZ72OxmFG5QsGMOw="; + + final Instant generatedTimeV1 = Instant.now().minus(2, ChronoUnit.DAYS); + final Instant expireTimeV1 = Instant.now().plus(365, ChronoUnit.DAYS); + final Instant generatedTimeV2 = Instant.now().minus(1, ChronoUnit.DAYS); + final Instant expireTimeV2 = Instant.now().plus(366, ChronoUnit.DAYS); + + final JsonObject metadataJson = new JsonObject(); + { + metadataJson.put("version", 2); + metadataJson.put("generated", generatedTimeV1.getEpochSecond() * 1000L); + metadataJson.put("first_level", FIRST_LEVEL_SALT); + metadataJson.put("id_prefix", ID_PREFIX); + metadataJson.put("id_secret", ID_SECRET); + final JsonArray saltsRefList = new JsonArray(); + { + final JsonObject saltsRef = new JsonObject(); + saltsRef.put("effective", generatedTimeV1.getEpochSecond() * 1000L); + saltsRef.put("expires", expireTimeV1.getEpochSecond() * 1000L); + saltsRef.put("location", "saltsV1.txt"); + saltsRef.put("size", 8); + saltsRefList.add(saltsRef); + } + { + final JsonObject saltsRef = new JsonObject(); + saltsRef.put("effective", generatedTimeV2.getEpochSecond() * 1000L); + saltsRef.put("expires", expireTimeV2.getEpochSecond() * 1000L); + saltsRef.put("location", "saltsV2.txt"); + saltsRef.put("size", 8); + saltsRefList.add(saltsRef); + } + metadataJson.put("salts", saltsRefList); + } + + final String effectiveTimeStringV1 = String.valueOf(generatedTimeV1.getEpochSecond() * 1000L); + final String saltsV1 = + "1000000," + effectiveTimeStringV1 + ",y5YitNf/KFtceipDz8nqsFVmBZsK3KY7s8bOVM4gMD4=\n" + + "1000001," + effectiveTimeStringV1 + ",z1uBoGyyzgna9i0o/r5eiD/wAhDX/2Q/6zX1p6hsF7I=\n" + + "1000002," + effectiveTimeStringV1 + ",+a5LPajo7uPfNcc9HH0Tn25b3RnSNZwe8YaAKcyeHaA=\n" + + "1000003," + effectiveTimeStringV1 + ",wAL6U+lu9gcMhSEySzWG9RQyoo446zAyGWKTW8VVoVw=\n" + + "1000004," + effectiveTimeStringV1 + ",eP9ZvW4igLQZ4QfzlyiXgKYFDZgmGOefaKDLEL0zuwE=\n" + + "1000005," + effectiveTimeStringV1 + ",UebesrNN0bQkm/QR7Jx7eav+UDXN5Gbq3zs1fLBMRy0=\n" + + "1000006," + effectiveTimeStringV1 + ",MtpALOziEJMtPlCQHk6RHALuWvRvRZpCDBmO0xPAia0=\n" + + "1000007," + effectiveTimeStringV1 + ",7tjv+KXaSztTZHEHULacotHQ7IpGBcw6IymoRLObkT4="; + + // update key 1000002 + final String effectiveTimeStringV2 = String.valueOf(generatedTimeV2.getEpochSecond() * 1000L); + final String saltsV2 = + "1000000," + effectiveTimeStringV1 + ",y5YitNf/KFtceipDz8nqsFVmBZsK3KY7s8bOVM4gMD4=\n" + + "1000001," + effectiveTimeStringV1 + ",z1uBoGyyzgna9i0o/r5eiD/wAhDX/2Q/6zX1p6hsF7I=\n" + + "1000002," + effectiveTimeStringV2 + ",AP73KwZscb1ltQQH/B7fdbHUnMmbJNlRULxzklXUqaA=\n" + + "1000003," + effectiveTimeStringV1 + ",wAL6U+lu9gcMhSEySzWG9RQyoo446zAyGWKTW8VVoVw=\n" + + "1000004," + effectiveTimeStringV1 + ",eP9ZvW4igLQZ4QfzlyiXgKYFDZgmGOefaKDLEL0zuwE=\n" + + "1000005," + effectiveTimeStringV1 + ",UebesrNN0bQkm/QR7Jx7eav+UDXN5Gbq3zs1fLBMRy0=\n" + + "1000006," + effectiveTimeStringV1 + ",MtpALOziEJMtPlCQHk6RHALuWvRvRZpCDBmO0xPAia0=\n" + + "1000007," + effectiveTimeStringV1 + ",7tjv+KXaSztTZHEHULacotHQ7IpGBcw6IymoRLObkT4="; + + when(cloudStorage.download("sites/encrypted/1_public/metadata.json")) + .thenReturn(new ByteArrayInputStream(metadataJson.toString().getBytes(StandardCharsets.US_ASCII))); + when(cloudStorage.download("saltsV1.txt")) + .thenReturn(getEncryptedStream(saltsV1)); + when(cloudStorage.download("saltsV2.txt")) + .thenReturn(getEncryptedStream(saltsV2)); + + EncryptedRotatingSaltProvider saltsProvider = new EncryptedRotatingSaltProvider( + cloudStorage, keyProvider, new EncryptedScope(new CloudPath("sites/metadata.json"), 1, true)); + + final JsonObject loadedMetadata = saltsProvider.getMetadata(); + saltsProvider.loadContent(loadedMetadata); + assertEquals(2, saltsProvider.getVersion(loadedMetadata)); + + final ISaltProvider.ISaltSnapshot snapshot = saltsProvider.getSnapshot(Instant.now()); + assertEquals(FIRST_LEVEL_SALT, snapshot.getFirstLevelSalt()); + assertTrue(snapshot.getModifiedSince(Instant.now().minus(1, ChronoUnit.HOURS)).isEmpty()); + assertEquals(1, snapshot.getModifiedSince(Instant.now().minus(30, ChronoUnit.HOURS)).size()); + assertEquals(1000002, snapshot.getModifiedSince(Instant.now().minus(30, ChronoUnit.HOURS)).get(0).getId()); + } + + @Test + public void loadSaltMultipleVersionsExpired() throws Exception { + final String FIRST_LEVEL_SALT = "first_level_salt_value"; + final String ID_PREFIX = "a"; + final String ID_SECRET = "m3yMIcbg9vCaFLJsn4m4PfruZnvAZ72OxmFG5QsGMOw="; + + final Instant generatedTimeV1 = Instant.now().minus(3, ChronoUnit.DAYS); + final Instant expireTimeV1 = Instant.now().minus(2, ChronoUnit.DAYS); + final Instant generatedTimeV2 = Instant.now().minus(2, ChronoUnit.DAYS); + final Instant expireTimeV2 = Instant.now().minus(1, ChronoUnit.DAYS); + + final JsonObject metadataJson = new JsonObject(); + { + metadataJson.put("version", 2); + metadataJson.put("generated", generatedTimeV1.getEpochSecond() * 1000L); + metadataJson.put("first_level", FIRST_LEVEL_SALT); + metadataJson.put("id_prefix", ID_PREFIX); + metadataJson.put("id_secret", ID_SECRET); + final JsonArray saltsRefList = new JsonArray(); + { + final JsonObject saltsRef = new JsonObject(); + saltsRef.put("effective", generatedTimeV1.getEpochSecond() * 1000L); + saltsRef.put("expires", expireTimeV1.getEpochSecond() * 1000L); + saltsRef.put("location", "saltsV1.txt"); + saltsRef.put("size", 8); + saltsRefList.add(saltsRef); + } + { + final JsonObject saltsRef = new JsonObject(); + saltsRef.put("effective", generatedTimeV2.getEpochSecond() * 1000L); + saltsRef.put("expires", expireTimeV2.getEpochSecond() * 1000L); + saltsRef.put("location", "saltsV2.txt"); + saltsRef.put("size", 8); + saltsRefList.add(saltsRef); + } + metadataJson.put("salts", saltsRefList); + } + + final String effectiveTimeStringV1 = String.valueOf(generatedTimeV1.getEpochSecond() * 1000L); + final String saltsV1 = + "1000000," + effectiveTimeStringV1 + ",y5YitNf/KFtceipDz8nqsFVmBZsK3KY7s8bOVM4gMD4=\n" + + "1000001," + effectiveTimeStringV1 + ",z1uBoGyyzgna9i0o/r5eiD/wAhDX/2Q/6zX1p6hsF7I=\n" + + "1000002," + effectiveTimeStringV1 + ",+a5LPajo7uPfNcc9HH0Tn25b3RnSNZwe8YaAKcyeHaA=\n" + + "1000003," + effectiveTimeStringV1 + ",wAL6U+lu9gcMhSEySzWG9RQyoo446zAyGWKTW8VVoVw=\n" + + "1000004," + effectiveTimeStringV1 + ",eP9ZvW4igLQZ4QfzlyiXgKYFDZgmGOefaKDLEL0zuwE=\n" + + "1000005," + effectiveTimeStringV1 + ",UebesrNN0bQkm/QR7Jx7eav+UDXN5Gbq3zs1fLBMRy0=\n" + + "1000006," + effectiveTimeStringV1 + ",MtpALOziEJMtPlCQHk6RHALuWvRvRZpCDBmO0xPAia0=\n" + + "1000007," + effectiveTimeStringV1 + ",7tjv+KXaSztTZHEHULacotHQ7IpGBcw6IymoRLObkT4="; + + // update key 1000002 + final String effectiveTimeStringV2 = String.valueOf(generatedTimeV2.getEpochSecond() * 1000L); + final String saltsV2 = + "1000000," + effectiveTimeStringV1 + ",y5YitNf/KFtceipDz8nqsFVmBZsK3KY7s8bOVM4gMD4=\n" + + "1000001," + effectiveTimeStringV1 + ",z1uBoGyyzgna9i0o/r5eiD/wAhDX/2Q/6zX1p6hsF7I=\n" + + "1000002," + effectiveTimeStringV2 + ",AP73KwZscb1ltQQH/B7fdbHUnMmbJNlRULxzklXUqaA=\n" + + "1000003," + effectiveTimeStringV1 + ",wAL6U+lu9gcMhSEySzWG9RQyoo446zAyGWKTW8VVoVw=\n" + + "1000004," + effectiveTimeStringV1 + ",eP9ZvW4igLQZ4QfzlyiXgKYFDZgmGOefaKDLEL0zuwE=\n" + + "1000005," + effectiveTimeStringV1 + ",UebesrNN0bQkm/QR7Jx7eav+UDXN5Gbq3zs1fLBMRy0=\n" + + "1000006," + effectiveTimeStringV1 + ",MtpALOziEJMtPlCQHk6RHALuWvRvRZpCDBmO0xPAia0=\n" + + "1000007," + effectiveTimeStringV1 + ",7tjv+KXaSztTZHEHULacotHQ7IpGBcw6IymoRLObkT4="; + + when(cloudStorage.download("sites/encrypted/1_public/metadata.json")) + .thenReturn(new ByteArrayInputStream(metadataJson.toString().getBytes(StandardCharsets.US_ASCII))); + when(cloudStorage.download("saltsV1.txt")) + .thenReturn(getEncryptedStream(saltsV1)); + when(cloudStorage.download("saltsV2.txt")) + .thenReturn(getEncryptedStream(saltsV2)); + + EncryptedRotatingSaltProvider saltsProvider = new EncryptedRotatingSaltProvider( + cloudStorage, keyProvider, new EncryptedScope(new CloudPath("sites/metadata.json"), 1, true)); + + final JsonObject loadedMetadata = saltsProvider.getMetadata(); + saltsProvider.loadContent(loadedMetadata); + assertEquals(2, saltsProvider.getVersion(loadedMetadata)); + + final ISaltProvider.ISaltSnapshot snapshot = saltsProvider.getSnapshot(Instant.now()); + assertEquals(FIRST_LEVEL_SALT, snapshot.getFirstLevelSalt()); + assertTrue(snapshot.getModifiedSince(Instant.now().minus(1, ChronoUnit.HOURS)).isEmpty()); + assertEquals(1, snapshot.getModifiedSince(Instant.now().minus(49, ChronoUnit.HOURS)).size()); + assertEquals(1000002, snapshot.getModifiedSince(Instant.now().minus(49, ChronoUnit.HOURS)).get(0).getId()); + } + + +} diff --git a/src/test/java/com/uid2/shared/store/EncryptedScopedStoreReaderTest.java b/src/test/java/com/uid2/shared/store/EncryptedScopedStoreReaderTest.java index 3ac00f0a..24a13156 100644 --- a/src/test/java/com/uid2/shared/store/EncryptedScopedStoreReaderTest.java +++ b/src/test/java/com/uid2/shared/store/EncryptedScopedStoreReaderTest.java @@ -20,6 +20,8 @@ import static com.uid2.shared.TestUtilites.toInputStream; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -52,6 +54,7 @@ void setUp() { Map mockKeyMap = new HashMap<>(); mockKeyMap.put(encryptionKey.getId(), encryptionKey); when(keyProvider.getAll()).thenReturn(mockKeyMap); + when(keyProvider.getKey(1)).thenReturn(mockKeyMap.get(1)); } @Test @@ -103,30 +106,11 @@ void raisesExceptionWhenNoDecryptionKeyFound() throws Exception { .hasMessageContaining("No matching S3 key found for decryption"); } - @Test - void testDecryptionOfEncryptedContent() throws Exception { - // Simulate encrypted content - String secretKey = encryptionKey.getSecret(); - byte[] secretKeyBytes = Base64.getDecoder().decode(secretKey); - byte[] encryptedPayload = AesGcm.encrypt("value1,value2".getBytes(StandardCharsets.UTF_8), secretKeyBytes); - String encryptedPayloadBase64 = Base64.getEncoder().encodeToString(encryptedPayload); - - JsonObject encryptedJson = new JsonObject() - .put("key_id", encryptionKey.getId()) - .put("encrypted_payload", encryptedPayloadBase64); - - String encryptedContent = encryptedJson.encodePrettily(); - EncryptedScopedStoreReader> reader = new EncryptedScopedStoreReader<>(storage, scope, parser, dataType, keyProvider); - - String decryptedContent = reader.getDecryptedContent(encryptedContent); - - assertThat(decryptedContent).isEqualTo("value1,value2"); - } - @Test void testHandlingInvalidEncryptionKey() throws Exception { // Set key provider to return an empty map when(keyProvider.getAll()).thenReturn(new HashMap<>()); + when(keyProvider.getKey(anyInt())).thenReturn(null); String secretKey = encryptionKey.getSecret(); byte[] secretKeyBytes = Base64.getDecoder().decode(secretKey); @@ -159,6 +143,7 @@ void testLoadWithMultipleEncryptionKeys() throws Exception { mockKeyMap.put(encryptionKey.getId(), encryptionKey); mockKeyMap.put(newKey.getId(), newKey); when(keyProvider.getAll()).thenReturn(mockKeyMap); + when(keyProvider.getKey(2)).thenReturn(mockKeyMap.get(2)); byte[] encryptedPayload = AesGcm.encrypt("value1,value2".getBytes(StandardCharsets.UTF_8), newKeyBytes); String encryptedPayloadBase64 = Base64.getEncoder().encodeToString(encryptedPayload); diff --git a/src/test/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProviderTest.java b/src/test/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProviderTest.java index f12c52ae..c2c94308 100644 --- a/src/test/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProviderTest.java +++ b/src/test/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProviderTest.java @@ -135,6 +135,35 @@ void testGetAllNullSnapshot() { assertTrue(keys.isEmpty()); } + @Test + void testGet() { + Map expectedKeys = new HashMap<>(); + CloudEncryptionKey key1 = new CloudEncryptionKey(1, 123, 1687635529, 1687808329, "S3keySecretByteHere1"); + CloudEncryptionKey key2 = new CloudEncryptionKey(2, 123, 1687808429, 1687808329, "S3keySecretByteHere2"); + expectedKeys.put(1, key1); + expectedKeys.put(2, key2); + when(reader.getSnapshot()).thenReturn(expectedKeys); + + CloudEncryptionKey key = rotatingCloudEncryptionKeyProvider.getKey(1); + assertEquals(key1, key); + } + + @Test + void testGetEmpty() { + when(reader.getSnapshot()).thenReturn(new HashMap<>()); + + CloudEncryptionKey key = rotatingCloudEncryptionKeyProvider.getKey(1); + assertNull(key); + } + + @Test + void testGetNullSnapshot() { + when(reader.getSnapshot()).thenReturn(null); + + CloudEncryptionKey key = rotatingCloudEncryptionKeyProvider.getKey(1); + assertNull(key); + } + @Test void testUpdateExistingKey() throws Exception { Map existingKeys = new HashMap<>(); diff --git a/src/test/java/com/uid2/shared/util/CloudEncryptionHelperTest.java b/src/test/java/com/uid2/shared/util/CloudEncryptionHelperTest.java new file mode 100644 index 00000000..047b3d12 --- /dev/null +++ b/src/test/java/com/uid2/shared/util/CloudEncryptionHelperTest.java @@ -0,0 +1,59 @@ +package com.uid2.shared.util; + +import com.uid2.shared.encryption.AesGcm; +import com.uid2.shared.model.CloudEncryptionKey; +import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; +import io.vertx.core.json.JsonObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.*; + +import static com.uid2.shared.util.CloudEncryptionHelpers.decryptInputStream; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class CloudEncryptionHelperTest { + private RotatingCloudEncryptionKeyProvider keyProvider; + private CloudEncryptionKey encryptionKey; + + @BeforeEach + void setUp() { + keyProvider = mock(RotatingCloudEncryptionKeyProvider.class); + + // Generate a valid 32-byte AES key + byte[] keyBytes = new byte[32]; + new Random().nextBytes(keyBytes); + String base64Key = Base64.getEncoder().encodeToString(keyBytes); + encryptionKey = new CloudEncryptionKey(1, 1, 0, 0, base64Key); + + Map mockKeyMap = new HashMap<>(); + mockKeyMap.put(encryptionKey.getId(), encryptionKey); + when(keyProvider.getAll()).thenReturn(mockKeyMap); + when(keyProvider.getKey(1)).thenReturn(mockKeyMap.get(1)); + } + + @Test + void testDecryptionOfEncryptedContent() throws Exception { + // Simulate encrypted content + String secretKey = encryptionKey.getSecret(); + byte[] secretKeyBytes = Base64.getDecoder().decode(secretKey); + byte[] encryptedPayload = AesGcm.encrypt("value1,value2".getBytes(StandardCharsets.UTF_8), secretKeyBytes); + String encryptedPayloadBase64 = Base64.getEncoder().encodeToString(encryptedPayload); + + JsonObject encryptedJson = new JsonObject() + .put("key_id", encryptionKey.getId()) + .put("encrypted_payload", encryptedPayloadBase64); + + String encryptedContent = encryptedJson.encodePrettily(); + + InputStream encryptedInputStream = new ByteArrayInputStream(encryptedContent.getBytes(StandardCharsets.UTF_8)); + + String decryptedContent = decryptInputStream(encryptedInputStream, keyProvider); + + assertThat(decryptedContent).isEqualTo("value1,value2"); + } +}