Cbc UI d2 4571 cloud encryption cleanup

pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.uid2</groupId>
     <artifactId>uid2-shared</artifactId>
-    <version>8.0.9</version>
+    <version>8.0.15-alpha-177-SNAPSHOT</version>
     <name>${project.groupId}:${project.artifactId}</name>
     <description>Library for all the shared uid2 operations</description>
     <url>https://github.com/IABTechLab/uid2docs</url>

src/main/java/com/uid2/shared/store/EncryptedRotatingSaltProvider.java

@@ -0,0 +1,35 @@
+package com.uid2.shared.store;
+
+import com.uid2.shared.Const;
+import com.uid2.shared.cloud.DownloadCloudStorage;
+import com.uid2.shared.model.SaltEntry;
+import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
+import com.uid2.shared.store.scope.StoreScope;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collection;
+
+import static com.uid2.shared.util.CloudEncryptionHelpers.decryptInputStream;
+
+public class EncryptedRotatingSaltProvider extends RotatingSaltProvider {
+    private final RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider;
+
+    public EncryptedRotatingSaltProvider(DownloadCloudStorage fileStreamProvider, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider, StoreScope scope) {
+        super(fileStreamProvider, scope.getMetadataPath().toString());
+        this.cloudEncryptionKeyProvider = cloudEncryptionKeyProvider;
+    }
+
+    @Override
+    protected SaltEntry[] readInputStream(InputStream inputStream, SaltEntryBuilder entryBuilder, Integer size) throws IOException {
+        String decrypted = decryptInputStream(inputStream, cloudEncryptionKeyProvider);
+        SaltEntry[] entries = new SaltEntry[size];
+        int idx = 0;
+        for (String line : decrypted.split("\n")) {
+            final SaltEntry entry = entryBuilder.toEntry(line);
+            entries[idx] = entry;
+            idx++;
+        }
+        return entries;
+    }
+}

src/main/java/com/uid2/shared/store/EncryptedScopedStoreReader.java

@@ -1,22 +1,18 @@
 package com.uid2.shared.store;
 
 import com.uid2.shared.cloud.DownloadCloudStorage;
-import com.uid2.shared.model.CloudEncryptionKey;
 import com.uid2.shared.store.parser.Parser;
 import com.uid2.shared.store.parser.ParsingResult;
 import com.uid2.shared.store.scope.StoreScope;
 import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
-import io.vertx.core.json.JsonObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.*;
 
-import com.uid2.shared.encryption.AesGcm;
-
 import java.nio.charset.StandardCharsets;
-import java.util.Base64;
-import java.util.Map;
+
+import static com.uid2.shared.util.CloudEncryptionHelpers.decryptInputStream;
 
 public class EncryptedScopedStoreReader<T> extends ScopedStoreReader<T> {
     private static final Logger LOGGER = LoggerFactory.getLogger(EncryptedScopedStoreReader.class);
@@ -31,8 +27,7 @@ public EncryptedScopedStoreReader(DownloadCloudStorage fileStreamProvider, Store
     @Override
     protected long loadContent(String path) throws Exception {
         try (InputStream inputStream = this.contentStreamProvider.download(path)) {
-            String encryptedContent = inputStreamToString(inputStream);
-            String decryptedContent = getDecryptedContent(encryptedContent);
+            String decryptedContent = decryptInputStream(inputStream, cloudEncryptionKeyProvider);
             ParsingResult<T> parsed = this.parser.deserialize(new ByteArrayInputStream(decryptedContent.getBytes(StandardCharsets.UTF_8)));
             latestSnapshot.set(parsed.getData());
 
@@ -45,39 +40,4 @@ protected long loadContent(String path) throws Exception {
             throw e;
         }
     }
-
-    protected String getDecryptedContent(String encryptedContent) throws Exception {
-        JsonObject json = new JsonObject(encryptedContent);
-        int keyId = json.getInteger("key_id");
-        String encryptedPayload = json.getString("encrypted_payload");
-        Map<Integer, CloudEncryptionKey> cloudEncryptionKeys = cloudEncryptionKeyProvider.getAll();
-        CloudEncryptionKey decryptionKey = null;
-        for (CloudEncryptionKey key : cloudEncryptionKeys.values()) {
-            if (key.getId() == keyId) {
-                decryptionKey = key;
-                break;
-            }
-        }
-
-        if (decryptionKey == null) {
-            throw new IllegalStateException("No matching S3 key found for decryption for key ID: " + keyId);
-        }
-
-        byte[] secret = Base64.getDecoder().decode(decryptionKey.getSecret());
-        byte[] encryptedBytes = Base64.getDecoder().decode(encryptedPayload);
-        byte[] decryptedBytes = AesGcm.decrypt(encryptedBytes, 0, secret);
-
-        return new String(decryptedBytes, StandardCharsets.UTF_8);
-    }
-
-    public static String inputStreamToString(InputStream inputStream) throws IOException {
-        try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
-            StringBuilder stringBuilder = new StringBuilder();
-            String line;
-            while ((line = reader.readLine()) != null) {
-                stringBuilder.append(line);
-            }
-            return stringBuilder.toString();
-        }
-    }
 }

src/main/java/com/uid2/shared/store/RotatingSaltProvider.java

@@ -13,6 +13,7 @@
 import org.hashids.Hashids;
 
 import java.io.BufferedReader;
+import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
@@ -21,6 +22,7 @@
 import java.util.*;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /*
   1. metadata.json format
@@ -130,20 +132,25 @@ private SaltSnapshot loadSnapshot(JsonObject spec, String firstLevelSalt, SaltEn
         final Instant expires = Instant.ofEpochMilli(spec.getLong("expires", defaultExpires.toEpochMilli()));
 
         final String path = spec.getString("location");
-        int idx = 0;
-        final SaltEntry[] entries = new SaltEntry[spec.getInteger("size")];
-
-        try (InputStream inputStream = this.contentStreamProvider.download(path);
-             InputStreamReader inputStreamReader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
-             BufferedReader reader = new BufferedReader(inputStreamReader)) {
-            for (String l; (l = reader.readLine()) != null; ++idx) {
-                final SaltEntry entry = entryBuilder.toEntry(l);
+        Integer size = spec.getInteger("size");
+        SaltEntry[] entries = readInputStream(this.contentStreamProvider.download(path), entryBuilder, size);
+
+        LOGGER.info("Loaded " + size + " salts");
+        return new SaltSnapshot(effective, expires, entries, firstLevelSalt);
+    }
+
+    protected SaltEntry[] readInputStream(InputStream inputStream, SaltEntryBuilder entryBuilder, Integer size) throws IOException {
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
+            String line;
+            SaltEntry[] entries = new SaltEntry[size];
+            int idx = 0;
+            while ((line = reader.readLine()) != null) {
+                final SaltEntry entry = entryBuilder.toEntry(line);
                 entries[idx] = entry;
+                idx++;
             }
+            return entries;
         }
-
-        LOGGER.info("Loaded " + idx + " salts");
-        return new SaltSnapshot(effective, expires, entries, firstLevelSalt);
     }
 
     public static class SaltSnapshot implements ISaltSnapshot {
@@ -214,7 +221,7 @@ public String encode(long id) {
         }
     }
 
-    static final class SaltEntryBuilder {
+    protected static final class SaltEntryBuilder {
         private final IdHashingScheme idHashingScheme;
 
         public SaltEntryBuilder(IdHashingScheme idHashingScheme) {

src/main/java/com/uid2/shared/store/reader/RotatingClientSideKeypairStore.java

@@ -3,6 +3,7 @@
 import com.uid2.shared.cloud.DownloadCloudStorage;
 import com.uid2.shared.model.ClientSideKeypair;
 import com.uid2.shared.store.CloudPath;
+import com.uid2.shared.store.EncryptedScopedStoreReader;
 import com.uid2.shared.store.IClientSideKeypairStore;
 import com.uid2.shared.store.ScopedStoreReader;
 import com.uid2.shared.store.parser.ClientSideKeypairParser;
@@ -19,6 +20,10 @@ public RotatingClientSideKeypairStore(DownloadCloudStorage fileStreamProvider, S
         this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new ClientSideKeypairParser(), "client_side_keypairs");
     }
 
+    public RotatingClientSideKeypairStore(DownloadCloudStorage fileStreamProvider, StoreScope scope, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) {
+        this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new ClientSideKeypairParser(), "client_side_keypairs", cloudEncryptionKeyProvider);
+    }
+
     @Override
     public long getVersion(JsonObject metadata) {
         return metadata.getLong("version");

src/main/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProvider.java

@@ -24,7 +24,7 @@
 import java.time.Instant;
 
 public class RotatingCloudEncryptionKeyProvider implements StoreReader<Map<Integer, CloudEncryptionKey>> {
-    ScopedStoreReader<Map<Integer, CloudEncryptionKey>> reader;
+    protected ScopedStoreReader<Map<Integer, CloudEncryptionKey>> reader;
 
     private static final Logger LOGGER = LoggerFactory.getLogger(RotatingCloudEncryptionKeyProvider.class);
     public Map<Integer, List<CloudEncryptionKey>> siteToKeysMap = new HashMap<>();
@@ -33,6 +33,11 @@ public RotatingCloudEncryptionKeyProvider(DownloadCloudStorage fileStreamProvide
         this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new CloudEncryptionKeyParser(), "cloud_encryption_keys");
     }
 
+
+    public RotatingCloudEncryptionKeyProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope, ScopedStoreReader<Map<Integer, CloudEncryptionKey>> reader) {
+        this.reader = reader;
+    }
+
     @Override
     public JsonObject getMetadata() throws Exception {
         return reader.getMetadata();
@@ -61,6 +66,15 @@ public Map<Integer, CloudEncryptionKey> getAll() {
         return keys != null ? keys : new HashMap<>();
     }
 
+    public CloudEncryptionKey getKey(int id) {
+        Map<Integer, CloudEncryptionKey> snapshot = reader.getSnapshot();
+        if(snapshot == null) {
+            return null;
+        }
+
+        return snapshot.get(id);
+    }
+
     public void updateSiteToKeysMapping() {
         Map<Integer, CloudEncryptionKey> allKeys = getAll();
         siteToKeysMap.clear();

src/main/java/com/uid2/shared/util/CloudEncryptionHelpers.java

@@ -0,0 +1,53 @@
+package com.uid2.shared.util;
+
+import java.io.InputStream;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.uid2.shared.encryption.AesGcm;
+import com.uid2.shared.model.CloudEncryptionKey;
+
+import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
+import io.vertx.core.json.JsonObject;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+import java.io.*;
+
+public class CloudEncryptionHelpers {
+    public static String decryptInputStream(InputStream inputStream, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) throws IOException {
+        JsonFactory factory = new JsonFactory();
+        JsonParser parser = factory.createParser(inputStream);
+        int keyId = -1;
+        byte[] encryptedPayload = null;
+        parser.nextToken();
+        while (parser.nextToken() != JsonToken.END_OBJECT) {
+                String fieldName = parser.getCurrentName();
+                if(fieldName.equals("key_id")) {
+                    parser.nextToken();
+                    keyId = parser.getIntValue();
+                }
+                if(fieldName.equals("encrypted_payload")) {
+                    parser.nextToken();
+                    encryptedPayload = parser.getBinaryValue();
+                }
+        }
+
+        if(keyId == -1 || encryptedPayload == null) {
+            throw new IllegalStateException("failed to parse json");
+        }
+
+        CloudEncryptionKey decryptionKey = cloudEncryptionKeyProvider.getKey(keyId);
+
+        if (decryptionKey == null) {
+            throw new IllegalStateException("No matching S3 key found for decryption for key ID: " + keyId);
+        }
+
+        byte[] secret = Base64.getDecoder().decode(decryptionKey.getSecret());
+        byte[] encryptedBytes = encryptedPayload;
+        byte[] decryptedBytes = AesGcm.decrypt(encryptedBytes, 0, secret);
+
+        return new String(decryptedBytes, StandardCharsets.UTF_8);
+    }
+}