Use EncryptedFilesSyncJob

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.uid2</groupId>
     <artifactId>uid2-admin</artifactId>
-    <version>5.18.8-alpha-127-SNAPSHOT</version>
+    <version>5.18.9-alpha-128-SNAPSHOT</version>
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

src/main/java/com/uid2/admin/Main.java

@@ -265,7 +265,8 @@ public void run() {
                     new SaltService(auth, writeLock, saltStoreWriter, saltProvider, saltRotation),
                     new SiteService(auth, writeLock, siteStoreWriter, siteProvider, clientKeyProvider),
                     new PartnerConfigService(auth, writeLock, partnerStoreWriter, partnerConfigProvider),
-                    new PrivateSiteDataRefreshService(auth, jobDispatcher, writeLock, config, rotatingCloudEncryptionKeyProvider),
+                    new PrivateSiteDataRefreshService(auth, jobDispatcher, writeLock, config),
+                    new EncryptedFilesSyncService(auth, jobDispatcher, writeLock, config, rotatingCloudEncryptionKeyProvider),
                     new JobDispatcherService(auth, jobDispatcher),
                     new SearchService(auth, clientKeyProvider, operatorKeyProvider),
                     new CloudEncryptionKeyService(auth, cloudEncryptionKeyManager)

src/main/java/com/uid2/admin/job/EncryptionJob/SaltEncryptionJob.java

@@ -14,14 +14,14 @@
 
 public class SaltEncryptionJob extends Job {
     private final Collection<OperatorKey> globalOperators;
-    private final Collection<RotatingSaltProvider.SaltSnapshot> saltEntries;
+    private final RotatingSaltProvider saltProvider;
     private final MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> multiScopeStoreWriter;
 
     public SaltEncryptionJob(Collection<OperatorKey> globalOperators,
-                             Collection<RotatingSaltProvider.SaltSnapshot> saltEntries,
+                             RotatingSaltProvider saltProvider,
                              MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> multiScopeStoreWriter) {
         this.globalOperators = globalOperators;
-        this.saltEntries = saltEntries;
+        this.saltProvider = saltProvider;
         this.multiScopeStoreWriter = multiScopeStoreWriter;
     }
 
@@ -34,8 +34,8 @@ public String getId() {
     @Override
     public void execute() throws Exception {
         List<Integer> desiredPrivateState = PrivateSiteUtil.getPrivateSaltSites(globalOperators);
-        multiScopeStoreWriter.uploadPrivateWithEncryption(desiredPrivateState, saltEntries, null);
+        multiScopeStoreWriter.uploadPrivateWithEncryption(desiredPrivateState, saltProvider.getSnapshots(), saltProvider.getMetadata());
         List<Integer> desiredPublicState = PublicSiteUtil.getPublicSaltSites(globalOperators);
-        multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, saltEntries, null);
+        multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, saltProvider.getSnapshots(), saltProvider.getMetadata());
     }
 }

src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java

@@ -195,7 +195,7 @@ public void execute() throws Exception {
                 encryptionKeyWriter
         );
         KeyAclEncryptionJob keyAclEncryptionSyncJob = new KeyAclEncryptionJob(keyAclWriter, globalOperators, globalKeyAcls);
-        SaltEncryptionJob saltEncryptionJob = new SaltEncryptionJob(globalOperators, saltProvider.getSnapshots(), saltWriter);
+        SaltEncryptionJob saltEncryptionJob = new SaltEncryptionJob(globalOperators, saltProvider, saltWriter);
         ClientSideKeypairEncryptionJob clientSideKeypairEncryptionJob = new ClientSideKeypairEncryptionJob(globalOperators, globalClientSideKeypair, clientSideKeypairWriter);
 
         siteEncryptionSyncJob.execute();

src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java

@@ -28,8 +28,6 @@ public class EncryptedSaltStoreWriter extends SaltStoreWriter implements StoreWr
     private RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider;
     private Integer siteId;
 
-    private final List<RotatingSaltProvider.SaltSnapshot> previousSeenSnapshots = new ArrayList<>();
-
     private static final Logger LOGGER = LoggerFactory.getLogger(EncryptedSaltStoreWriter.class);
     public EncryptedSaltStoreWriter(JsonObject config, RotatingSaltProvider provider, FileManager fileManager,
                                     TaggableCloudStorage cloudStorage, VersionGenerator versionGenerator, StoreScope scope,
@@ -51,12 +49,6 @@ protected void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, S
             throw new IllegalStateException("Site ID is not set.");
         }
 
-        if (!cloudStorage.list(location).isEmpty()) {
-            // update the tags on the file to ensure it is still marked as current
-            this.setStatusTagToCurrent(location);
-            return;
-        }
-
         StringBuilder stringBuilder = new StringBuilder();
 
         for (SaltEntry entry: snapshot.getAllRotatingSalts()) {
@@ -89,27 +81,12 @@ protected void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, S
 
         this.upload(newSaltsFile.toString(), location);
     }
-
-    @Override
-    protected void refreshProvider() {
-       // we do not need to refresh the provider on encrypted writers
-    }
-
-    @Override
-    protected List<RotatingSaltProvider.SaltSnapshot> getSnapshots(RotatingSaltProvider.SaltSnapshot data){
-    /*
-    Since metadata.json is overwritten during the process, we maintain a history of all snapshots seen so far.
-    On the final write, we append this history to metadata.json to ensure no snapshots are lost.
-    */
-        this.previousSeenSnapshots.add(data);
-        return this.previousSeenSnapshots;
-    }
 
     @Override
     public void upload(Object data, JsonObject extraMeta) throws Exception {
-        for(RotatingSaltProvider.SaltSnapshot saltSnapshot: (Collection<RotatingSaltProvider.SaltSnapshot>) data) {
-            super.upload(saltSnapshot);
-        }
+        @SuppressWarnings("unchecked")
+        List<RotatingSaltProvider.SaltSnapshot> snapshots = new ArrayList<>((Collection<RotatingSaltProvider.SaltSnapshot>) data);
+        this.buildAndUploadMetadata(extraMeta, this.uploadSnapshotsAndGetMetadata(snapshots));
     }
 
     @Override

src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java

@@ -43,7 +43,7 @@ public SaltStoreWriter(JsonObject config, RotatingSaltProvider provider, FileMan
         this.versionGenerator = versionGenerator;
     }
 
-    protected List<RotatingSaltProvider.SaltSnapshot> getSnapshots(RotatingSaltProvider.SaltSnapshot data){
+    private List<RotatingSaltProvider.SaltSnapshot> getSnapshots(RotatingSaltProvider.SaltSnapshot data){
         if (provider.getSnapshots() == null) {
             throw new IllegalStateException("Snapshots cannot be null");
         }
@@ -75,10 +75,7 @@ protected List<RotatingSaltProvider.SaltSnapshot> getSnapshots(RotatingSaltProvi
         return filteredSnapshots;
     }
 
-    public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
-        final Instant now = Instant.now();
-        final long generated = now.getEpochSecond();
-
+    protected JsonObject getMetadata() throws Exception {
         JsonObject metadata = null;
         try {
             metadata = provider.getMetadata();
@@ -89,15 +86,20 @@ public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
                 throw e;
             }
         }
-        // bump up metadata version
+        return metadata;
+    }
+
+    protected void buildAndUploadMetadata(JsonObject metadata, JsonArray snapshotsMetadata ) throws Exception{
+        final Instant now = Instant.now();
+        final long generated = now.getEpochSecond();
         metadata.put("version", versionGenerator.getVersion());
         metadata.put("generated", generated);
-
-        final JsonArray snapshotsMetadata = new JsonArray();
         metadata.put("salts", snapshotsMetadata);
+        fileManager.uploadMetadata(metadata, "salts", new CloudPath(provider.getMetadataPath()));
+    }
 
-        List<RotatingSaltProvider.SaltSnapshot> snapshots = this.getSnapshots(data);
-
+    protected JsonArray uploadSnapshotsAndGetMetadata(List<RotatingSaltProvider.SaltSnapshot> snapshots) throws Exception {
+        final JsonArray snapshotsMetadata = new JsonArray();
         for (RotatingSaltProvider.SaltSnapshot snapshot : snapshots) {
             final String location = getSaltSnapshotLocation(snapshot);
             final JsonObject snapshotMetadata = new JsonObject();
@@ -106,17 +108,19 @@ public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
             snapshotMetadata.put("location", location);
             snapshotMetadata.put("size", snapshot.getAllRotatingSalts().length);
             snapshotsMetadata.add(snapshotMetadata);
-
             uploadSaltsSnapshot(snapshot, location);
         }
+        return snapshotsMetadata;
+    }
 
-        fileManager.uploadMetadata(metadata, "salts", new CloudPath(provider.getMetadataPath()));
-
-        // refresh manually
+    public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
+        JsonObject metadata = this.getMetadata();
+        List<RotatingSaltProvider.SaltSnapshot> snapshots = this.getSnapshots(data);
+        this.buildAndUploadMetadata(metadata, this.uploadSnapshotsAndGetMetadata(snapshots));
         refreshProvider();
     }
 
-    protected void refreshProvider() throws Exception {
+    private void refreshProvider() throws Exception {
         provider.loadContent();
     }
 

src/main/java/com/uid2/admin/vertx/service/EncryptedFilesSyncService.java

@@ -38,14 +38,14 @@ public EncryptedFilesSyncService(
 
     @Override
     public void setupRoutes(Router router) {
-        router.post("/api/encrypted-files/refresh").handler(auth.handle((ctx) -> {
+        router.post("/api/encrypted-files/refresh").blockingHandler(auth.handle((ctx) -> {
                     synchronized (writeLock) {
                         this.handleEncryptedFileSync(ctx);
                     }
                 },
                 Role.MAINTAINER, Role.PRIVATE_OPERATOR_SYNC));
 
-        router.post("/api/encrypted-files/syncNow").handler(auth.handle(
+        router.post("/api/encrypted-files/syncNow").blockingHandler(auth.handle(
                 this::handleEncryptedFileSyncNow,
                 Role.MAINTAINER, Role.PRIVATE_OPERATOR_SYNC));
     }

src/main/java/com/uid2/admin/vertx/service/PrivateSiteDataRefreshService.java

@@ -7,8 +7,6 @@
 import com.uid2.admin.job.jobsync.keyset.ReplaceSharingTypesWithSitesJob;
 import com.uid2.admin.vertx.WriteLock;
 import com.uid2.shared.auth.Role;
-import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
-import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
 import io.vertx.core.json.JsonObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -24,19 +22,16 @@ public class PrivateSiteDataRefreshService implements IService {
     private final JobDispatcher jobDispatcher;
     private final WriteLock writeLock;
     private final JsonObject config;
-    private final RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider;
 
     public PrivateSiteDataRefreshService(
             AdminAuthMiddleware auth,
             JobDispatcher jobDispatcher,
             WriteLock writeLock,
-            JsonObject config,
-            RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider) {
+            JsonObject config) {
         this.auth = auth;
         this.jobDispatcher = jobDispatcher;
         this.writeLock = writeLock;
         this.config = config;
-        this.RotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider;
     }
 
     @Override
@@ -64,9 +59,6 @@ private void handlePrivateSiteDataGenerate(RoutingContext rc) {
             PrivateSiteDataSyncJob job = new PrivateSiteDataSyncJob(config, writeLock);
             jobDispatcher.enqueue(job);
 
-            EncryptedFilesSyncJob encryptedFileSyncJob = new EncryptedFilesSyncJob(config, writeLock, RotatingCloudEncryptionKeyProvider);
-            jobDispatcher.enqueue(encryptedFileSyncJob);
-
             rc.response().end("OK");
         } catch (Exception e) {
             LOGGER.error(e.getMessage(), e);
@@ -86,8 +78,6 @@ private void handlePrivateSiteDataGenerateNow(RoutingContext rc) {
             CompletableFuture<Boolean> privateSiteDataSyncJobFuture = jobDispatcher.executeNextJob();
             privateSiteDataSyncJobFuture.get();
 
-            EncryptedFilesSyncJob encryptedFileSyncJob = new EncryptedFilesSyncJob(config, writeLock, RotatingCloudEncryptionKeyProvider);
-            jobDispatcher.enqueue(encryptedFileSyncJob);
             CompletableFuture<Boolean> encryptedFileSyncJobFuture = jobDispatcher.executeNextJob();
             encryptedFileSyncJobFuture.get();
 

src/test/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriterTest.java

@@ -111,6 +111,12 @@ private void verifyFile(String filelocation, RotatingSaltProvider.SaltSnapshot s
     public void testUploadNew() throws Exception {
         RotatingSaltProvider.SaltSnapshot snapshot = makeSnapshot(Instant.ofEpochMilli(1740607938167L), Instant.ofEpochMilli(Instant.now().toEpochMilli() + 90002), 100);
         RotatingSaltProvider.SaltSnapshot snapshot2 = makeSnapshot(Instant.ofEpochMilli(1740694476392L), Instant.ofEpochMilli(Instant.now().toEpochMilli() + 130000), 10);
+        JsonObject metadata = new JsonObject()
+                .put("version", 1742770328863L)
+                .put("generated", 1742770328)
+                .put("first_level", "FIRST-LEVEL")
+                .put("id_prefix", "a")
+                .put("id_secret", "ID-SECRET");
         when(rotatingSaltProvider.getMetadata()).thenThrow(new CloudStorageException("The specified key does not exist: AmazonS3Exception: test-core-bucket"));
         when(rotatingSaltProvider.getSnapshots()).thenReturn(null);
 
@@ -123,19 +129,16 @@ public void testUploadNew() throws Exception {
         EncryptedSaltStoreWriter encryptedSaltStoreWriter = new EncryptedSaltStoreWriter(config, rotatingSaltProvider,
                 fileManager, taggableCloudStorage, versionGenerator, storeScope, rotatingCloudEncryptionKeyProvider, siteId);
 
-        encryptedSaltStoreWriter.upload(snapshot);
+        encryptedSaltStoreWriter.upload(List.of(snapshot,snapshot2), metadata);
         verify(fileManager).uploadMetadata(metadataCaptor.capture(), nameCaptor.capture(), locationCaptor.capture());
 
         // Capture the metadata
         JsonObject capturedMetadata = metadataCaptor.getValue();
-        assertEquals(1, capturedMetadata.getJsonArray("salts").size(), "The 'salts' array should contain exactly 1 item");
-        encryptedSaltStoreWriter.upload(snapshot2);
 
-        verify(fileManager,times(2)).uploadMetadata(metadataCaptor.capture(), nameCaptor.capture(), locationCaptor.capture());
-        capturedMetadata = metadataCaptor.getValue();
         assertEquals(2, capturedMetadata.getJsonArray("salts").size(), "The 'salts' array should contain 2 items");
-
-        verify(taggableCloudStorage,times(3)).upload(pathCaptor.capture(), cloudPathCaptor.capture(), any());
+        assertEquals(capturedMetadata.getString("first_level"), metadata.getValue("first_level"));
+        assertEquals(capturedMetadata.getString("id_prefix"), metadata.getValue("id_prefix"));
+        verify(taggableCloudStorage,times(2)).upload(pathCaptor.capture(), cloudPathCaptor.capture(), any());
 
         verifyFile(pathCaptor.getValue(), snapshot);
     }
@@ -171,7 +174,14 @@ public void testUnencryptedAndEncryptedBehavesTheSame() throws Exception {
         EncryptedSaltStoreWriter encryptedSaltStoreWriter = new EncryptedSaltStoreWriter(config, rotatingSaltProvider,
                 fileManager, taggableCloudStorage, versionGenerator, storeScope, rotatingCloudEncryptionKeyProvider, siteId);
 
-        encryptedSaltStoreWriter.upload(snapshot2);
+        JsonObject metadata = new JsonObject()
+                .put("version", 1742770328863L)
+                .put("generated", 1742770328)
+                .put("first_level", "FIRST-LEVEL")
+                .put("id_prefix", "a")
+                .put("id_secret", "ID-SECRET");
+
+        encryptedSaltStoreWriter.upload(List.of(snapshot2), metadata);
 
         verify(fileManager,atLeastOnce()).uploadMetadata(metadataCaptor.capture(), nameCaptor.capture(), locationCaptor.capture());