Adding more work on salt encryption

Dockerfile

@@ -26,4 +26,5 @@ CMD java \
     -Djava.security.egd=file:/dev/./urandom \
     -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \
     -Dlogback.configurationFile=${LOGBACK_CONF} \
+    -Xmx4g \
     -jar ${JAR_NAME}-${JAR_VERSION}.jar

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.uid2</groupId>
     <artifactId>uid2-admin</artifactId>
-    <version>5.16.0</version>
+    <version>5.16.8-alpha-103-SNAPSHOT</version>
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -16,7 +16,7 @@
         <!-- check micrometer.version vertx-micrometer-metrics consumes before bumping up -->
         <micrometer.version>1.12.2</micrometer.version>
         <junit-jupiter.version>5.11.2</junit-jupiter.version>
-        <uid2-shared.version>8.0.6</uid2-shared.version>
+        <uid2-shared.version>8.0.25</uid2-shared.version>
         <okta-jwt.version>0.5.10</okta-jwt.version>
         <image.version>${project.version}</image.version>
     </properties>

src/main/java/com/uid2/admin/job/EncryptionJob/ClientKeyEncryptionJob.java

@@ -33,7 +33,7 @@ public String getId() {
     public void execute() throws Exception {
         PrivateSiteDataMap<LegacyClientKey> desiredPrivateState = PrivateSiteUtil.getClientKeys(globalOperators, globalClientKeys);
         multiScopeStoreWriter.uploadPrivateWithEncryption(desiredPrivateState, null);
-        PrivateSiteDataMap<LegacyClientKey> desiredPublicState = PublicSiteUtil.getPublicClients(globalClientKeys,globalOperators);
+        PrivateSiteDataMap<LegacyClientKey> desiredPublicState = PublicSiteUtil.getPublicClients(globalClientKeys, globalOperators);
         multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, null);
     }
 }

src/main/java/com/uid2/admin/job/EncryptionJob/ClientSideKeypairEncryptionJob.java

@@ -0,0 +1,36 @@
+package com.uid2.admin.job.EncryptionJob;
+
+import com.uid2.admin.job.model.Job;
+import com.uid2.admin.model.PrivateSiteDataMap;
+import com.uid2.admin.store.MultiScopeStoreWriter;
+import com.uid2.admin.util.PublicSiteUtil;
+import com.uid2.shared.auth.OperatorKey;
+import com.uid2.shared.model.ClientSideKeypair;
+
+import java.util.Collection;
+
+public class ClientSideKeypairEncryptionJob extends Job {
+    private final Collection<OperatorKey> globalOperators;
+    private final Collection<ClientSideKeypair> globalClientSideKeypairs;
+
+    private final MultiScopeStoreWriter<Collection<ClientSideKeypair>> multiScopeStoreWriter;
+
+    public ClientSideKeypairEncryptionJob(Collection<OperatorKey> globalOperators, Collection<ClientSideKeypair> globalClientSideKeypairs,
+                                          MultiScopeStoreWriter<Collection<ClientSideKeypair>> multiScopeStoreWriter) {
+        this.globalOperators = globalOperators;
+        this.globalClientSideKeypairs = globalClientSideKeypairs;
+        this.multiScopeStoreWriter = multiScopeStoreWriter;
+    }
+
+    @Override
+    public String getId() {
+        return "cloud-encryption-sync-clientside-keypair";
+    }
+
+    @Override
+    public void execute() throws Exception {
+        // Only public operators support clientside keypair
+        PrivateSiteDataMap<ClientSideKeypair> desiredPublicState = PublicSiteUtil.getPublicClientKeypairs(globalClientSideKeypairs, globalOperators);
+        multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, null);
+    }
+}

src/main/java/com/uid2/admin/job/EncryptionJob/SaltEncryptionJob.java

@@ -0,0 +1,41 @@
+package com.uid2.admin.job.EncryptionJob;
+
+import com.uid2.admin.job.model.Job;
+import com.uid2.admin.model.PrivateSiteDataMap;
+import com.uid2.admin.store.MultiScopeStoreWriter;
+import com.uid2.admin.util.PrivateSiteUtil;
+import com.uid2.admin.util.PublicSiteUtil;
+import com.uid2.shared.auth.OperatorKey;
+import com.uid2.shared.model.SaltEntry;
+import com.uid2.shared.store.RotatingSaltProvider;
+
+import java.util.Collection;
+import java.util.List;
+
+public class SaltEncryptionJob extends Job {
+    private final Collection<OperatorKey> globalOperators;
+    private final Collection<RotatingSaltProvider.SaltSnapshot> saltEntries;
+    private final MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> multiScopeStoreWriter;
+
+    public SaltEncryptionJob(Collection<OperatorKey> globalOperators,
+                             Collection<RotatingSaltProvider.SaltSnapshot> saltEntries,
+                             MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> multiScopeStoreWriter) {
+        this.globalOperators = globalOperators;
+        this.saltEntries = saltEntries;
+        this.multiScopeStoreWriter = multiScopeStoreWriter;
+    }
+
+
+    @Override
+    public String getId() {
+        return "cloud-encryption-sync-salts";
+    }
+
+    @Override
+    public void execute() throws Exception {
+        List<Integer> desiredPrivateState = PrivateSiteUtil.getPrivateSaltSites(globalOperators);
+        multiScopeStoreWriter.uploadPrivateWithEncryption(desiredPrivateState, saltEntries, null);
+        List<Integer> desiredPublicState = PublicSiteUtil.getPublicSaltSites(globalOperators);
+        multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, saltEntries, null);
+    }
+}

src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java

@@ -2,7 +2,6 @@
 
 import com.fasterxml.jackson.databind.ObjectWriter;
 import com.uid2.admin.job.EncryptionJob.*;
-import com.uid2.admin.job.EncryptionJob.ClientKeyEncryptionJob;
 import com.uid2.admin.job.model.Job;
 import com.uid2.admin.store.*;
 import com.uid2.admin.store.factory.*;
@@ -17,12 +16,15 @@
 import com.uid2.shared.auth.RotatingOperatorKeyProvider;
 import com.uid2.shared.cloud.CloudUtils;
 import com.uid2.shared.cloud.ICloudStorage;
+import com.uid2.shared.cloud.TaggableCloudStorage;
+import com.uid2.shared.model.ClientSideKeypair;
 import com.uid2.shared.model.EncryptionKey;
 import com.uid2.shared.model.KeysetKey;
 import com.uid2.shared.model.Site;
 import com.uid2.shared.store.CloudPath;
 import com.uid2.admin.legacy.LegacyClientKey;
-import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
+import com.uid2.shared.store.EncryptedRotatingSaltProvider;
+import com.uid2.shared.store.RotatingSaltProvider;
 import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
 import com.uid2.shared.store.scope.GlobalScope;
 import io.vertx.core.json.JsonObject;
@@ -34,12 +36,12 @@
 public class EncryptedFilesSyncJob extends Job {
     private final JsonObject config;
     private final WriteLock writeLock;
-    private final RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider;
+    private final RotatingCloudEncryptionKeyProvider rotatingCloudEncryptionKeyProvider;
 
     public EncryptedFilesSyncJob(JsonObject config, WriteLock writeLock, RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider) {
         this.config = config;
         this.writeLock = writeLock;
-        this.RotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider;
+        this.rotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider;
     }
 
     @Override
@@ -49,20 +51,22 @@ public String getId() {
 
     @Override
     public void execute() throws Exception {
-        ICloudStorage cloudStorage = CloudUtils.createStorage(config.getString(Const.Config.CoreS3BucketProp), config);
+        TaggableCloudStorage cloudStorage = CloudUtils.createStorage(config.getString(Const.Config.CoreS3BucketProp), config);
         FileStorage fileStorage = new TmpFileStorage();
         ObjectWriter jsonWriter = JsonUtil.createJsonWriter();
         Clock clock = new InstantClock();
         VersionGenerator versionGenerator = new EpochVersionGenerator(clock);
         FileManager fileManager = new FileManager(cloudStorage, fileStorage);
 
+        RotatingSaltProvider saltProvider = new RotatingSaltProvider(cloudStorage, config.getString(Const.Config.SaltsMetadataPathProp));
+
         SiteStoreFactory siteStoreFactory = new SiteStoreFactory(
                 cloudStorage,
                 new CloudPath(config.getString(Const.Config.SitesMetadataPathProp)),
                 jsonWriter,
                 versionGenerator,
                 clock,
-                RotatingCloudEncryptionKeyProvider,
+                rotatingCloudEncryptionKeyProvider,
                 fileManager);
 
         ClientKeyStoreFactory clientKeyStoreFactory = new ClientKeyStoreFactory(
@@ -71,15 +75,15 @@ public void execute() throws Exception {
                 jsonWriter,
                 versionGenerator,
                 clock,
-                RotatingCloudEncryptionKeyProvider,
+                rotatingCloudEncryptionKeyProvider,
                 fileManager);
 
         EncryptionKeyStoreFactory encryptionKeyStoreFactory = new EncryptionKeyStoreFactory(
                 cloudStorage,
                 new CloudPath(config.getString(Const.Config.KeysMetadataPathProp)),
                 versionGenerator,
                 clock,
-                RotatingCloudEncryptionKeyProvider,
+                rotatingCloudEncryptionKeyProvider,
                 fileManager);
 
         KeyAclStoreFactory keyAclStoreFactory = new KeyAclStoreFactory(
@@ -88,7 +92,7 @@ public void execute() throws Exception {
                 jsonWriter,
                 versionGenerator,
                 clock,
-                RotatingCloudEncryptionKeyProvider,
+                rotatingCloudEncryptionKeyProvider,
                 fileManager);
 
         KeysetStoreFactory keysetStoreFactory = new KeysetStoreFactory(
@@ -98,7 +102,7 @@ public void execute() throws Exception {
                 versionGenerator,
                 clock,
                 fileManager,
-                RotatingCloudEncryptionKeyProvider,
+                rotatingCloudEncryptionKeyProvider,
                 config.getBoolean(enableKeysetConfigProp));
 
         KeysetKeyStoreFactory keysetKeyStoreFactory = new KeysetKeyStoreFactory(
@@ -107,15 +111,33 @@ public void execute() throws Exception {
                 versionGenerator,
                 clock,
                 fileManager,
-                RotatingCloudEncryptionKeyProvider,
+                rotatingCloudEncryptionKeyProvider,
                 config.getBoolean(enableKeysetConfigProp));
 
+        SaltStoreFactory saltStoreFactory = new SaltStoreFactory(
+                config,
+                new CloudPath(config.getString(Const.Config.SaltsMetadataPathProp)),
+                fileManager,
+                cloudStorage,
+                versionGenerator,
+                rotatingCloudEncryptionKeyProvider
+        );
+
+        ClientSideKeypairStoreFactory clientSideKeypairStoreFactory = new ClientSideKeypairStoreFactory(
+                cloudStorage,
+                new CloudPath(config.getString(Const.Config.ClientSideKeypairsMetadataPathProp)),
+                versionGenerator,
+                clock,
+                rotatingCloudEncryptionKeyProvider,
+                fileManager
+        );
+
         CloudPath operatorMetadataPath = new CloudPath(config.getString(Const.Config.OperatorsMetadataPathProp));
         GlobalScope operatorScope = new GlobalScope(operatorMetadataPath);
         RotatingOperatorKeyProvider operatorKeyProvider = new RotatingOperatorKeyProvider(cloudStorage, cloudStorage, operatorScope);
 
         synchronized (writeLock) {
-            RotatingCloudEncryptionKeyProvider.loadContent();
+            rotatingCloudEncryptionKeyProvider.loadContent();
             operatorKeyProvider.loadContent(operatorKeyProvider.getMetadata());
             siteStoreFactory.getGlobalReader().loadContent(siteStoreFactory.getGlobalReader().getMetadata());
             clientKeyStoreFactory.getGlobalReader().loadContent();
@@ -125,13 +147,18 @@ public void execute() throws Exception {
                 keysetStoreFactory.getGlobalReader().loadContent();
                 keysetKeyStoreFactory.getGlobalReader().loadContent();
             }
+            saltProvider.loadContent();
+            clientSideKeypairStoreFactory.getGlobalReader().loadContent();
         }
+
         Collection<OperatorKey> globalOperators = operatorKeyProvider.getAll();
         Collection<Site> globalSites = siteStoreFactory.getGlobalReader().getAllSites();
         Collection<LegacyClientKey> globalClients = clientKeyStoreFactory.getGlobalReader().getAll();
         Collection<EncryptionKey> globalEncryptionKeys = encryptionKeyStoreFactory.getGlobalReader().getSnapshot().getActiveKeySet();
         Integer globalMaxKeyId = encryptionKeyStoreFactory.getGlobalReader().getMetadata().getInteger("max_key_id");
         Map<Integer, EncryptionKeyAcl> globalKeyAcls = keyAclStoreFactory.getGlobalReader().getSnapshot().getAllAcls();
+        Collection<ClientSideKeypair> globalClientSideKeypair = clientSideKeypairStoreFactory.getGlobalReader().getAll();
+
         MultiScopeStoreWriter<Collection<Site>> siteWriter = new MultiScopeStoreWriter<>(
                 fileManager,
                 siteStoreFactory,
@@ -148,6 +175,14 @@ public void execute() throws Exception {
                 fileManager,
                 keyAclStoreFactory,
                 MultiScopeStoreWriter::areMapsEqual);
+        MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> saltWriter = new MultiScopeStoreWriter<>(
+                fileManager,
+                saltStoreFactory,
+                MultiScopeStoreWriter::areCollectionsEqual);
+        MultiScopeStoreWriter<Collection<ClientSideKeypair>> clientSideKeypairWriter = new MultiScopeStoreWriter<>(
+                fileManager,
+                clientSideKeypairStoreFactory,
+                MultiScopeStoreWriter::areCollectionsEqual);
 
         SiteEncryptionJob siteEncryptionSyncJob = new SiteEncryptionJob(siteWriter, globalSites, globalOperators);
         ClientKeyEncryptionJob clientEncryptionSyncJob = new ClientKeyEncryptionJob(clientWriter, globalClients, globalOperators);
@@ -160,10 +195,15 @@ public void execute() throws Exception {
                 encryptionKeyWriter
         );
         KeyAclEncryptionJob keyAclEncryptionSyncJob = new KeyAclEncryptionJob(keyAclWriter, globalOperators, globalKeyAcls);
+        SaltEncryptionJob saltEncryptionJob = new SaltEncryptionJob(globalOperators, saltProvider.getSnapshots(), saltWriter);
+        ClientSideKeypairEncryptionJob clientSideKeypairEncryptionJob = new ClientSideKeypairEncryptionJob(globalOperators, globalClientSideKeypair, clientSideKeypairWriter);
+
         siteEncryptionSyncJob.execute();
         clientEncryptionSyncJob.execute();
         encryptionKeyEncryptionSyncJob.execute();
         keyAclEncryptionSyncJob.execute();
+        saltEncryptionJob.execute();
+        clientSideKeypairEncryptionJob.execute();
 
         if(config.getBoolean(enableKeysetConfigProp)) {
             Map<Integer, Keyset> globalKeysets = keysetStoreFactory.getGlobalReader().getSnapshot().getAllKeysets();

src/main/java/com/uid2/admin/store/MultiScopeStoreWriter.java

@@ -69,6 +69,13 @@ public void uploadPrivateWithEncryption(Map<Integer, T> desiredState, JsonObject
         }
     }
 
+    public void uploadPrivateWithEncryption(List<Integer> siteIds, T desiredState, JsonObject extraMeta) throws Exception {
+        EncryptedStoreFactory<T> encryptedFactory = (EncryptedStoreFactory<T>) factory;
+        for (Integer siteId : siteIds) {
+            encryptedFactory.getEncryptedWriter(siteId,false).upload(desiredState, extraMeta);
+        }
+    }
+
     public void uploadPublicWithEncryption(Map<Integer, T> desiredPublicState, JsonObject extraMeta) throws Exception {
         EncryptedStoreFactory<T> encryptedFactory = (EncryptedStoreFactory<T>) factory;
         for (Map.Entry<Integer, T> entry : desiredPublicState.entrySet()) {
@@ -77,6 +84,13 @@ public void uploadPublicWithEncryption(Map<Integer, T> desiredPublicState, JsonO
         }
     }
 
+    public void uploadPublicWithEncryption(List<Integer> siteIds, T desiredState, JsonObject extraMeta) throws Exception {
+        EncryptedStoreFactory<T> encryptedFactory = (EncryptedStoreFactory<T>) factory;
+        for (Integer siteId : siteIds) {
+            encryptedFactory.getEncryptedWriter(siteId,true).upload(desiredState, extraMeta);
+        }
+    }
+
     public static <K, V> boolean areMapsEqual(Map<K, V> a, Map<K, V> b) {
         return a.size() == b.size() && a.entrySet().stream().allMatch(b.entrySet()::contains);
     }