Merge remote-tracking branch 'origin' into abu-add-encryption-urls

Author: abuabraham-ttd

.trivyignore

@@ -3,4 +3,4 @@
 # for more details
 
 # https://thetradedesk.atlassian.net/browse/UID2-5186
-CVE-2024-8176 exp:2025-03-27
+CVE-2024-8176 exp:2025-04-03

src/main/java/com/uid2/admin/Main.java

@@ -5,15 +5,18 @@
 import com.uid2.admin.auth.OktaAuthProvider;
 import com.uid2.admin.auth.AuthProvider;
 import com.uid2.admin.auth.TokenRefreshHandler;
+import com.uid2.admin.cloudencryption.CloudKeyRotationStrategy;
+import com.uid2.admin.cloudencryption.ExpiredKeyCountRetentionStrategy;
 import com.uid2.admin.job.JobDispatcher;
 import com.uid2.admin.job.jobsync.EncryptedFilesSyncJob;
 import com.uid2.admin.job.jobsync.PrivateSiteDataSyncJob;
 import com.uid2.admin.job.jobsync.keyset.ReplaceSharingTypesWithSitesJob;
 import com.uid2.admin.legacy.LegacyClientKeyStoreWriter;
 import com.uid2.admin.legacy.RotatingLegacyClientKeyProvider;
 import com.uid2.admin.managers.KeysetManager;
+import com.uid2.admin.cloudencryption.CloudSecretGenerator;
 import com.uid2.admin.monitoring.DataStoreMetrics;
-import com.uid2.admin.managers.CloudEncryptionKeyManager;
+import com.uid2.admin.cloudencryption.CloudEncryptionKeyManager;
 import com.uid2.admin.secret.*;
 import com.uid2.admin.store.*;
 import com.uid2.admin.store.reader.RotatingAdminKeysetStore;
@@ -29,7 +32,6 @@
 import com.uid2.admin.vertx.service.*;
 import com.uid2.shared.Const;
 import com.uid2.shared.Utils;
-import com.uid2.shared.secret.IKeyGenerator;
 import com.uid2.shared.secret.KeyHasher;
 import com.uid2.shared.secret.SecureKeyGenerator;
 import com.uid2.shared.auth.EnclaveIdentifierProvider;
@@ -74,7 +76,6 @@ public class Main {
 
     private final Vertx vertx;
     private final JsonObject config;
-
     public Main(Vertx vertx, JsonObject config) {
         this.vertx = vertx;
         this.config = config;
@@ -122,7 +123,7 @@ public void run() {
             try {
                 adminKeysetProvider.loadContent();
             } catch (CloudStorageException e) {
-                if(e.getMessage().contains("The specified key does not exist")){
+                if (e.getMessage().contains("The specified key does not exist")) {
                     adminKeysetStoreWriter.upload(new HashMap<>(), null);
                     adminKeysetProvider.loadContent();
                 } else {
@@ -134,7 +135,7 @@ public void run() {
             GlobalScope keysetKeysGlobalScope = new GlobalScope(keysetKeyMetadataPath);
             RotatingKeysetKeyStore keysetKeysProvider = new RotatingKeysetKeyStore(cloudStorage, keysetKeysGlobalScope);
             KeysetKeyStoreWriter keysetKeyStoreWriter = new KeysetKeyStoreWriter(keysetKeysProvider, fileManager, versionGenerator, clock, keysetKeysGlobalScope, enableKeysets);
-            if(enableKeysets) {
+            if (enableKeysets) {
                 try {
                     keysetKeysProvider.loadContent();
                 } catch (CloudStorageException e) {
@@ -154,7 +155,7 @@ public void run() {
             try {
                 clientSideKeypairProvider.loadContent();
             } catch (CloudStorageException e) {
-                if(e.getMessage().contains("The specified key does not exist")) {
+                if (e.getMessage().contains("The specified key does not exist")) {
                     clientSideKeypairStoreWriter.upload(new HashSet<>(), null);
                     clientSideKeypairProvider.loadContent();
                 } else {
@@ -163,13 +164,13 @@ public void run() {
             }
 
             CloudPath serviceMetadataPath = new CloudPath(config.getString(Const.Config.ServiceMetadataPathProp));
-            GlobalScope serviceGlobalScope= new GlobalScope(serviceMetadataPath);
+            GlobalScope serviceGlobalScope = new GlobalScope(serviceMetadataPath);
             RotatingServiceStore serviceProvider = new RotatingServiceStore(cloudStorage, serviceGlobalScope);
             ServiceStoreWriter serviceStoreWriter = new ServiceStoreWriter(serviceProvider, fileManager, jsonWriter, versionGenerator, clock, serviceGlobalScope);
             try {
                 serviceProvider.loadContent();
             } catch (CloudStorageException e) {
-                if(e.getMessage().contains("The specified key does not exist")) {
+                if (e.getMessage().contains("The specified key does not exist")) {
                     serviceStoreWriter.upload(new HashSet<>(), null);
                     serviceProvider.loadContent();
                 } else {
@@ -178,13 +179,13 @@ public void run() {
             }
 
             CloudPath serviceLinkMetadataPath = new CloudPath(config.getString(Const.Config.ServiceLinkMetadataPathProp));
-            GlobalScope serviceLinkGlobalScope= new GlobalScope(serviceLinkMetadataPath);
+            GlobalScope serviceLinkGlobalScope = new GlobalScope(serviceLinkMetadataPath);
             RotatingServiceLinkStore serviceLinkProvider = new RotatingServiceLinkStore(cloudStorage, serviceLinkGlobalScope);
             ServiceLinkStoreWriter serviceLinkStoreWriter = new ServiceLinkStoreWriter(serviceLinkProvider, fileManager, jsonWriter, versionGenerator, clock, serviceLinkGlobalScope);
             try {
                 serviceLinkProvider.loadContent();
             } catch (CloudStorageException e) {
-                if(e.getMessage().contains("The specified key does not exist")) {
+                if (e.getMessage().contains("The specified key does not exist")) {
                     serviceLinkStoreWriter.upload(new HashSet<>(), null);
                     serviceLinkProvider.loadContent();
                 } else {
@@ -202,8 +203,7 @@ public void run() {
             GlobalScope cloudEncryptionKeyGlobalScope = new GlobalScope(cloudEncryptionKeyMetadataPath);
             RotatingCloudEncryptionKeyProvider rotatingCloudEncryptionKeyProvider = new RotatingCloudEncryptionKeyProvider(cloudStorage, cloudEncryptionKeyGlobalScope);
             CloudEncryptionKeyStoreWriter cloudEncryptionKeyStoreWriter = new CloudEncryptionKeyStoreWriter(rotatingCloudEncryptionKeyProvider, fileManager, jsonWriter, versionGenerator, clock, cloudEncryptionKeyGlobalScope);
-            IKeyGenerator keyGenerator = new SecureKeyGenerator();
-            CloudEncryptionKeyManager cloudEncryptionKeyManager = new CloudEncryptionKeyManager(rotatingCloudEncryptionKeyProvider, cloudEncryptionKeyStoreWriter,keyGenerator);
+            SecureKeyGenerator keyGenerator = new SecureKeyGenerator();
             try {
                 rotatingCloudEncryptionKeyProvider.loadContent();
             } catch (CloudStorageException e) {
@@ -247,6 +247,11 @@ public void run() {
 
             ClientSideKeypairService clientSideKeypairService = new ClientSideKeypairService(config, auth, writeLock, clientSideKeypairStoreWriter, clientSideKeypairProvider, siteProvider, keysetManager, keypairGenerator, clock);
 
+            var cloudEncryptionSecretGenerator = new CloudSecretGenerator(keyGenerator);
+            var cloudEncryptionKeyManager = new CloudEncryptionKeyManager(rotatingCloudEncryptionKeyProvider, cloudEncryptionKeyStoreWriter, cloudEncryptionSecretGenerator);
+            var cloudEncryptionKeyRetentionStrategy = new ExpiredKeyCountRetentionStrategy(clock, 5);
+            var cloudEncryptionKeyRotationStrategy = new CloudKeyRotationStrategy(cloudEncryptionSecretGenerator, clock, cloudEncryptionKeyRetentionStrategy);
+
             IService[] services = {
                     new ClientKeyService(config, auth, writeLock, clientKeyStoreWriter, clientKeyProvider, siteProvider, keysetManager, keyGenerator, keyHasher),
                     new EnclaveIdService(auth, writeLock, enclaveStoreWriter, enclaveIdProvider, clock),
@@ -264,7 +269,7 @@ public void run() {
                     new EncryptedFilesSyncService(auth, jobDispatcher, writeLock, config, rotatingCloudEncryptionKeyProvider),
                     new JobDispatcherService(auth, jobDispatcher),
                     new SearchService(auth, clientKeyProvider, operatorKeyProvider),
-                    new CloudEncryptionKeyService(auth, rotatingCloudEncryptionKeyProvider)
+                    new CloudEncryptionKeyService(auth, rotatingCloudEncryptionKeyProvider, cloudEncryptionKeyStoreWriter, operatorKeyProvider, cloudEncryptionKeyRotationStrategy)
             };
 
 
@@ -280,7 +285,7 @@ public void run() {
             try {
                 keysetProvider.loadContent();
             } catch (CloudStorageException e) {
-                if(e.getMessage().contains("The specified key does not exist")){
+                if (e.getMessage().contains("The specified key does not exist")) {
                     keysetStoreWriter.upload(new HashMap<>(), null);
                     keysetProvider.loadContent();
                 } else {
@@ -306,7 +311,7 @@ public void run() {
             The jobs are executed after because they copy data from these files locations consumed by public and private operators.
             This caused an issue because the files were empty and the job started to fail so the operators got empty files.
              */
-            if(enableKeysets) {
+            if (enableKeysets) {
                 synchronized (writeLock) {
                     //UID2-628 keep keys.json and keyset_keys.json in sync. This function syncs them on start up
                     keysetProvider.loadContent();
@@ -343,7 +348,7 @@ public void run() {
             CompletableFuture<Boolean> privateSiteDataSyncJobFuture = jobDispatcher.executeNextJob();
             privateSiteDataSyncJobFuture.get();
 
-            EncryptedFilesSyncJob encryptedFilesSyncJob = new EncryptedFilesSyncJob(config, writeLock,rotatingCloudEncryptionKeyProvider);
+            EncryptedFilesSyncJob encryptedFilesSyncJob = new EncryptedFilesSyncJob(config, writeLock, rotatingCloudEncryptionKeyProvider);
             jobDispatcher.enqueue(encryptedFilesSyncJob);
             CompletableFuture<Boolean> encryptedFilesSyncJobFuture = jobDispatcher.executeNextJob();
             encryptedFilesSyncJobFuture.get();

src/main/java/com/uid2/admin/cloudencryption/CloudEncryptionKeyGenerator.java

@@ -0,0 +1,40 @@
+package com.uid2.admin.cloudencryption;
+
+import com.uid2.admin.store.Clock;
+import com.uid2.shared.model.CloudEncryptionKey;
+
+import java.util.Collection;
+
+public class CloudEncryptionKeyGenerator {
+    private final Clock clock;
+    private final CloudSecretGenerator secretGenerator;
+    private final KeyIdGenerator idGenerator;
+
+    public CloudEncryptionKeyGenerator(
+            Clock clock,
+            CloudSecretGenerator secretGenerator,
+            Collection<CloudEncryptionKey> existingKeys) {
+        this.clock = clock;
+        this.secretGenerator = secretGenerator;
+        this.idGenerator = new KeyIdGenerator(existingKeys);
+    }
+
+    public CloudEncryptionKey makeNewKey(Integer siteId) {
+        var nowSeconds = clock.getEpochSecond();
+        var keyId = idGenerator.nextId();
+        var secret = secretGenerator.generate();
+        return new CloudEncryptionKey(keyId, siteId, nowSeconds, nowSeconds, secret);
+    }
+
+    private static class KeyIdGenerator {
+        private int lastId;
+
+        public KeyIdGenerator(Collection<CloudEncryptionKey> existingKeys) {
+            this.lastId = existingKeys.stream().map(CloudEncryptionKey::getId).max(Integer::compareTo).orElse(0);
+        }
+
+        public int nextId() {
+            return ++lastId;
+        }
+    }
+}

…/managers/CloudEncryptionKeyManager.java …ncryption/CloudEncryptionKeyManager.javasrc/main/java/com/uid2/admin/managers/CloudEncryptionKeyManager.java renamed to src/main/java/com/uid2/admin/cloudencryption/CloudEncryptionKeyManager.java src/main/java/com/uid2/admin/managers/CloudEncryptionKeyManager.java renamed to src/main/java/com/uid2/admin/cloudencryption/CloudEncryptionKeyManager.java

@@ -1,17 +1,12 @@
-package com.uid2.admin.managers;
+package com.uid2.admin.cloudencryption;
 
 import com.uid2.admin.store.writer.CloudEncryptionKeyStoreWriter;
 import com.uid2.shared.auth.OperatorKey;
 import com.uid2.shared.model.CloudEncryptionKey;
-import com.uid2.shared.secret.IKeyGenerator;
 import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
 import java.time.Instant;
 import java.util.*;
 
@@ -21,16 +16,16 @@ public class CloudEncryptionKeyManager {
 
     private final RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider;
     private final CloudEncryptionKeyStoreWriter cloudEncryptionKeyStoreWriter;
-    private final IKeyGenerator keyGenerator;
+    private final CloudSecretGenerator secretGenerator;
 
     public CloudEncryptionKeyManager(
             RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider,
             CloudEncryptionKeyStoreWriter cloudEncryptionKeyStoreWriter,
-            IKeyGenerator keyGenerator
+            CloudSecretGenerator keyGenerator
     ) {
         this.RotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider;
         this.cloudEncryptionKeyStoreWriter = cloudEncryptionKeyStoreWriter;
-        this.keyGenerator = keyGenerator;
+        this.secretGenerator = keyGenerator;
     }
 
     // Ensures there are `keyCountPerSite` sites for each site corresponding of operatorKeys. If there are less - create new ones.
@@ -89,15 +84,10 @@ private static Set<Integer> uniqueSiteIdsForOperators(Collection<OperatorKey> op
 
     CloudEncryptionKey generateCloudEncryptionKey(int siteId, long activates, long created) throws Exception {
         int newKeyId = getNextKeyId();
-        String secret = generateSecret();
+        String secret = secretGenerator.generate();
         return new CloudEncryptionKey(newKeyId, siteId, activates, created, secret);
     }
 
-    String generateSecret() throws Exception {
-        //Generate a 32-byte key for AesGcm
-        return keyGenerator.generateRandomKeyString(32);
-    }
-
     void addCloudEncryptionKey(CloudEncryptionKey cloudEncryptionKey) throws Exception {
         Map<Integer, CloudEncryptionKey> cloudEncryptionKeys = new HashMap<>(RotatingCloudEncryptionKeyProvider.getAll());
         cloudEncryptionKeys.put(cloudEncryptionKey.getId(), cloudEncryptionKey);
@@ -112,49 +102,6 @@ int getNextKeyId() {
         return cloudEncryptionKeys.keySet().stream().max(Integer::compareTo).orElse(0) + 1;
     }
 
-    // Used in test only
-    // Creates and uploads a CloudEncryptionKey that activates immediately for a specific sites, for emergency rotation
-    CloudEncryptionKey createAndAddImmediateCloudEncryptionKey(int siteId) throws Exception {
-        int newKeyId = getNextKeyId();
-        long created = Instant.now().getEpochSecond();
-        CloudEncryptionKey newKey = new CloudEncryptionKey(newKeyId, siteId, created, created, generateSecret());
-        addCloudEncryptionKey(newKey);
-        return newKey;
-    }
-
-    // Used in test only
-    CloudEncryptionKey getCloudEncryptionKeyByKeyIdentifier(int keyIdentifier) {
-        return RotatingCloudEncryptionKeyProvider.getAll().get(keyIdentifier);
-    }
-
-    // Used in test only
-    Optional<CloudEncryptionKey> getCloudEncryptionKeyBySiteId(int siteId) {
-        return RotatingCloudEncryptionKeyProvider.getAll().values().stream()
-                .filter(key -> key.getSiteId() == siteId)
-                .findFirst();
-    }
-
-    // Used in test only
-    List<CloudEncryptionKey> getAllCloudEncryptionKeysBySiteId(int siteId) {
-        return RotatingCloudEncryptionKeyProvider.getAll().values().stream()
-                .filter(key -> key.getSiteId() == siteId)
-                .collect(Collectors.toList());
-    }
-
-    // Used in test only
-    Map<Integer, CloudEncryptionKey> getAllCloudEncryptionKeys() {
-        return RotatingCloudEncryptionKeyProvider.getAll();
-    }
-
-    // Used in test only
-    boolean doesSiteHaveKeys(int siteId) {
-        Map<Integer, CloudEncryptionKey> allKeys = RotatingCloudEncryptionKeyProvider.getAll();
-        if (allKeys == null) {
-            return false;
-        }
-        return allKeys.values().stream().anyMatch(key -> key.getSiteId() == siteId);
-    }
-
     int countKeysForSite(int siteId) {
         Map<Integer, CloudEncryptionKey> allKeys = RotatingCloudEncryptionKeyProvider.getAll();
         return (int) allKeys.values().stream().filter(key -> key.getSiteId() == siteId).count();

src/main/java/com/uid2/admin/cloudencryption/CloudKeyRetentionStrategy.java

@@ -0,0 +1,9 @@
+package com.uid2.admin.cloudencryption;
+
+import com.uid2.shared.model.CloudEncryptionKey;
+
+import java.util.Set;
+
+public interface CloudKeyRetentionStrategy {
+    Set<CloudEncryptionKey> selectKeysToRetain(Set<CloudEncryptionKey> keysForSite);
+}

src/main/java/com/uid2/admin/cloudencryption/CloudKeyRotationStrategy.java

@@ -0,0 +1,55 @@
+package com.uid2.admin.cloudencryption;
+
+import com.google.common.collect.Streams;
+import com.uid2.admin.store.Clock;
+import com.uid2.shared.auth.OperatorKey;
+import com.uid2.shared.model.CloudEncryptionKey;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class CloudKeyRotationStrategy {
+        private final CloudSecretGenerator secretGenerator;
+        private final Clock clock;
+        private final CloudKeyRetentionStrategy keyRetentionStrategy;
+
+        public CloudKeyRotationStrategy(
+                CloudSecretGenerator secretGenerator,
+                Clock clock,
+                CloudKeyRetentionStrategy keyRetentionStrategy
+        ) {
+                this.secretGenerator = secretGenerator;
+                this.clock = clock;
+                this.keyRetentionStrategy = keyRetentionStrategy;
+        }
+
+        public Set<CloudEncryptionKey> computeDesiredKeys(
+                Collection<CloudEncryptionKey> existingCloudKeys,
+                Collection<OperatorKey> operatorKeys
+        ) {
+                var keyGenerator = new CloudEncryptionKeyGenerator(clock, secretGenerator, existingCloudKeys);
+                Map<Integer, Set<CloudEncryptionKey>> existingKeysBySite = existingCloudKeys
+                        .stream()
+                        .collect(Collectors.groupingBy(CloudEncryptionKey::getSiteId, Collectors.toSet()));
+
+                return operatorKeys
+                        .stream()
+                        .map(OperatorKey::getSiteId)
+                        .distinct()
+                        .flatMap(siteId -> desiredKeysForSite(siteId, keyGenerator, existingKeysBySite.getOrDefault(siteId, Set.of())))
+                        .collect(Collectors.toSet());
+        }
+
+        private Stream<CloudEncryptionKey> desiredKeysForSite(
+                Integer siteId,
+                CloudEncryptionKeyGenerator keyGenerator,
+                Set<CloudEncryptionKey> existingKeys
+        ) {
+                var existingKeysToRetain = keyRetentionStrategy.selectKeysToRetain(existingKeys);
+                var newKey = keyGenerator.makeNewKey(siteId);
+                return Streams.concat(existingKeysToRetain.stream(), Stream.of(newKey));
+        }
+}

src/main/java/com/uid2/admin/cloudencryption/CloudSecretGenerator.java

@@ -0,0 +1,17 @@
+package com.uid2.admin.cloudencryption;
+
+import com.uid2.shared.secret.SecureKeyGenerator;
+
+public class CloudSecretGenerator {
+    private final SecureKeyGenerator keyGenerator;
+
+    // The SecureKeyGenerator is preferable to the IKeyGenerator interface as it doesn't throw Exception
+    public CloudSecretGenerator(SecureKeyGenerator keyGenerator) {
+        this.keyGenerator = keyGenerator;
+    }
+
+    public String generate() {
+        //Generate a 32-byte key for AesGcm
+        return keyGenerator.generateRandomKeyString(32);
+    }
+}