diff --git a/pom.xml b/pom.xml
index 37f83aa4..8cf6c25f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
com.uid2
uid2-admin
- 5.18.8-alpha-127-SNAPSHOT
+ 5.18.9-alpha-128-SNAPSHOT
UTF-8
diff --git a/src/main/java/com/uid2/admin/Main.java b/src/main/java/com/uid2/admin/Main.java
index 070c0f3d..3320114f 100644
--- a/src/main/java/com/uid2/admin/Main.java
+++ b/src/main/java/com/uid2/admin/Main.java
@@ -265,7 +265,8 @@ public void run() {
new SaltService(auth, writeLock, saltStoreWriter, saltProvider, saltRotation),
new SiteService(auth, writeLock, siteStoreWriter, siteProvider, clientKeyProvider),
new PartnerConfigService(auth, writeLock, partnerStoreWriter, partnerConfigProvider),
- new PrivateSiteDataRefreshService(auth, jobDispatcher, writeLock, config, rotatingCloudEncryptionKeyProvider),
+ new PrivateSiteDataRefreshService(auth, jobDispatcher, writeLock, config),
+ new EncryptedFilesSyncService(auth, jobDispatcher, writeLock, config, rotatingCloudEncryptionKeyProvider),
new JobDispatcherService(auth, jobDispatcher),
new SearchService(auth, clientKeyProvider, operatorKeyProvider),
new CloudEncryptionKeyService(auth, cloudEncryptionKeyManager)
diff --git a/src/main/java/com/uid2/admin/job/EncryptionJob/SaltEncryptionJob.java b/src/main/java/com/uid2/admin/job/EncryptionJob/SaltEncryptionJob.java
index 4ffb7b11..8ebae0b2 100644
--- a/src/main/java/com/uid2/admin/job/EncryptionJob/SaltEncryptionJob.java
+++ b/src/main/java/com/uid2/admin/job/EncryptionJob/SaltEncryptionJob.java
@@ -14,14 +14,14 @@
public class SaltEncryptionJob extends Job {
private final Collection globalOperators;
- private final Collection saltEntries;
+ private final RotatingSaltProvider saltProvider;
private final MultiScopeStoreWriter> multiScopeStoreWriter;
public SaltEncryptionJob(Collection globalOperators,
- Collection saltEntries,
+ RotatingSaltProvider saltProvider,
MultiScopeStoreWriter> multiScopeStoreWriter) {
this.globalOperators = globalOperators;
- this.saltEntries = saltEntries;
+ this.saltProvider = saltProvider;
this.multiScopeStoreWriter = multiScopeStoreWriter;
}
@@ -34,8 +34,8 @@ public String getId() {
@Override
public void execute() throws Exception {
List desiredPrivateState = PrivateSiteUtil.getPrivateSaltSites(globalOperators);
- multiScopeStoreWriter.uploadPrivateWithEncryption(desiredPrivateState, saltEntries, null);
+ multiScopeStoreWriter.uploadPrivateWithEncryption(desiredPrivateState, saltProvider.getSnapshots(), saltProvider.getMetadata());
List desiredPublicState = PublicSiteUtil.getPublicSaltSites(globalOperators);
- multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, saltEntries, null);
+ multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, saltProvider.getSnapshots(), saltProvider.getMetadata());
}
}
diff --git a/src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java b/src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java
index 186809d4..69bea884 100644
--- a/src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java
+++ b/src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java
@@ -195,7 +195,7 @@ public void execute() throws Exception {
encryptionKeyWriter
);
KeyAclEncryptionJob keyAclEncryptionSyncJob = new KeyAclEncryptionJob(keyAclWriter, globalOperators, globalKeyAcls);
- SaltEncryptionJob saltEncryptionJob = new SaltEncryptionJob(globalOperators, saltProvider.getSnapshots(), saltWriter);
+ SaltEncryptionJob saltEncryptionJob = new SaltEncryptionJob(globalOperators, saltProvider, saltWriter);
ClientSideKeypairEncryptionJob clientSideKeypairEncryptionJob = new ClientSideKeypairEncryptionJob(globalOperators, globalClientSideKeypair, clientSideKeypairWriter);
siteEncryptionSyncJob.execute();
diff --git a/src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java
index 9ce37db4..0a408d2c 100644
--- a/src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java
+++ b/src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java
@@ -28,8 +28,6 @@ public class EncryptedSaltStoreWriter extends SaltStoreWriter implements StoreWr
private RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider;
private Integer siteId;
- private final List previousSeenSnapshots = new ArrayList<>();
-
private static final Logger LOGGER = LoggerFactory.getLogger(EncryptedSaltStoreWriter.class);
public EncryptedSaltStoreWriter(JsonObject config, RotatingSaltProvider provider, FileManager fileManager,
TaggableCloudStorage cloudStorage, VersionGenerator versionGenerator, StoreScope scope,
@@ -51,12 +49,6 @@ protected void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, S
throw new IllegalStateException("Site ID is not set.");
}
- if (!cloudStorage.list(location).isEmpty()) {
- // update the tags on the file to ensure it is still marked as current
- this.setStatusTagToCurrent(location);
- return;
- }
-
StringBuilder stringBuilder = new StringBuilder();
for (SaltEntry entry: snapshot.getAllRotatingSalts()) {
@@ -89,27 +81,12 @@ protected void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, S
this.upload(newSaltsFile.toString(), location);
}
-
- @Override
- protected void refreshProvider() {
- // we do not need to refresh the provider on encrypted writers
- }
-
- @Override
- protected List getSnapshots(RotatingSaltProvider.SaltSnapshot data){
- /*
- Since metadata.json is overwritten during the process, we maintain a history of all snapshots seen so far.
- On the final write, we append this history to metadata.json to ensure no snapshots are lost.
- */
- this.previousSeenSnapshots.add(data);
- return this.previousSeenSnapshots;
- }
@Override
public void upload(Object data, JsonObject extraMeta) throws Exception {
- for(RotatingSaltProvider.SaltSnapshot saltSnapshot: (Collection) data) {
- super.upload(saltSnapshot);
- }
+ @SuppressWarnings("unchecked")
+ List snapshots = new ArrayList<>((Collection) data);
+ this.buildAndUploadMetadata(extraMeta, this.uploadSnapshotsAndGetMetadata(snapshots));
}
@Override
diff --git a/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java
index b72db1dd..5cc91019 100644
--- a/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java
+++ b/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java
@@ -43,7 +43,7 @@ public SaltStoreWriter(JsonObject config, RotatingSaltProvider provider, FileMan
this.versionGenerator = versionGenerator;
}
- protected List getSnapshots(RotatingSaltProvider.SaltSnapshot data){
+ private List getSnapshots(RotatingSaltProvider.SaltSnapshot data){
if (provider.getSnapshots() == null) {
throw new IllegalStateException("Snapshots cannot be null");
}
@@ -75,10 +75,7 @@ protected List getSnapshots(RotatingSaltProvi
return filteredSnapshots;
}
- public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
- final Instant now = Instant.now();
- final long generated = now.getEpochSecond();
-
+ protected JsonObject getMetadata() throws Exception {
JsonObject metadata = null;
try {
metadata = provider.getMetadata();
@@ -89,15 +86,20 @@ public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
throw e;
}
}
- // bump up metadata version
+ return metadata;
+ }
+
+ protected void buildAndUploadMetadata(JsonObject metadata, JsonArray snapshotsMetadata ) throws Exception{
+ final Instant now = Instant.now();
+ final long generated = now.getEpochSecond();
metadata.put("version", versionGenerator.getVersion());
metadata.put("generated", generated);
-
- final JsonArray snapshotsMetadata = new JsonArray();
metadata.put("salts", snapshotsMetadata);
+ fileManager.uploadMetadata(metadata, "salts", new CloudPath(provider.getMetadataPath()));
+ }
- List snapshots = this.getSnapshots(data);
-
+ protected JsonArray uploadSnapshotsAndGetMetadata(List snapshots) throws Exception {
+ final JsonArray snapshotsMetadata = new JsonArray();
for (RotatingSaltProvider.SaltSnapshot snapshot : snapshots) {
final String location = getSaltSnapshotLocation(snapshot);
final JsonObject snapshotMetadata = new JsonObject();
@@ -106,17 +108,19 @@ public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
snapshotMetadata.put("location", location);
snapshotMetadata.put("size", snapshot.getAllRotatingSalts().length);
snapshotsMetadata.add(snapshotMetadata);
-
uploadSaltsSnapshot(snapshot, location);
}
+ return snapshotsMetadata;
+ }
- fileManager.uploadMetadata(metadata, "salts", new CloudPath(provider.getMetadataPath()));
-
- // refresh manually
+ public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
+ JsonObject metadata = this.getMetadata();
+ List snapshots = this.getSnapshots(data);
+ this.buildAndUploadMetadata(metadata, this.uploadSnapshotsAndGetMetadata(snapshots));
refreshProvider();
}
- protected void refreshProvider() throws Exception {
+ private void refreshProvider() throws Exception {
provider.loadContent();
}
diff --git a/src/main/java/com/uid2/admin/vertx/service/EncryptedFilesSyncService.java b/src/main/java/com/uid2/admin/vertx/service/EncryptedFilesSyncService.java
index e16d091b..1a562b0d 100644
--- a/src/main/java/com/uid2/admin/vertx/service/EncryptedFilesSyncService.java
+++ b/src/main/java/com/uid2/admin/vertx/service/EncryptedFilesSyncService.java
@@ -38,14 +38,14 @@ public EncryptedFilesSyncService(
@Override
public void setupRoutes(Router router) {
- router.post("/api/encrypted-files/refresh").handler(auth.handle((ctx) -> {
+ router.post("/api/encrypted-files/refresh").blockingHandler(auth.handle((ctx) -> {
synchronized (writeLock) {
this.handleEncryptedFileSync(ctx);
}
},
Role.MAINTAINER, Role.PRIVATE_OPERATOR_SYNC));
- router.post("/api/encrypted-files/syncNow").handler(auth.handle(
+ router.post("/api/encrypted-files/syncNow").blockingHandler(auth.handle(
this::handleEncryptedFileSyncNow,
Role.MAINTAINER, Role.PRIVATE_OPERATOR_SYNC));
}
diff --git a/src/main/java/com/uid2/admin/vertx/service/PrivateSiteDataRefreshService.java b/src/main/java/com/uid2/admin/vertx/service/PrivateSiteDataRefreshService.java
index d7e17e5a..cba05894 100644
--- a/src/main/java/com/uid2/admin/vertx/service/PrivateSiteDataRefreshService.java
+++ b/src/main/java/com/uid2/admin/vertx/service/PrivateSiteDataRefreshService.java
@@ -7,8 +7,6 @@
import com.uid2.admin.job.jobsync.keyset.ReplaceSharingTypesWithSitesJob;
import com.uid2.admin.vertx.WriteLock;
import com.uid2.shared.auth.Role;
-import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
-import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
import io.vertx.core.json.JsonObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -24,19 +22,16 @@ public class PrivateSiteDataRefreshService implements IService {
private final JobDispatcher jobDispatcher;
private final WriteLock writeLock;
private final JsonObject config;
- private final RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider;
public PrivateSiteDataRefreshService(
AdminAuthMiddleware auth,
JobDispatcher jobDispatcher,
WriteLock writeLock,
- JsonObject config,
- RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider) {
+ JsonObject config) {
this.auth = auth;
this.jobDispatcher = jobDispatcher;
this.writeLock = writeLock;
this.config = config;
- this.RotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider;
}
@Override
@@ -64,9 +59,6 @@ private void handlePrivateSiteDataGenerate(RoutingContext rc) {
PrivateSiteDataSyncJob job = new PrivateSiteDataSyncJob(config, writeLock);
jobDispatcher.enqueue(job);
- EncryptedFilesSyncJob encryptedFileSyncJob = new EncryptedFilesSyncJob(config, writeLock, RotatingCloudEncryptionKeyProvider);
- jobDispatcher.enqueue(encryptedFileSyncJob);
-
rc.response().end("OK");
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
@@ -86,8 +78,6 @@ private void handlePrivateSiteDataGenerateNow(RoutingContext rc) {
CompletableFuture privateSiteDataSyncJobFuture = jobDispatcher.executeNextJob();
privateSiteDataSyncJobFuture.get();
- EncryptedFilesSyncJob encryptedFileSyncJob = new EncryptedFilesSyncJob(config, writeLock, RotatingCloudEncryptionKeyProvider);
- jobDispatcher.enqueue(encryptedFileSyncJob);
CompletableFuture encryptedFileSyncJobFuture = jobDispatcher.executeNextJob();
encryptedFileSyncJobFuture.get();
diff --git a/src/test/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriterTest.java b/src/test/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriterTest.java
index 142b3903..64bb703b 100644
--- a/src/test/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriterTest.java
+++ b/src/test/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriterTest.java
@@ -111,6 +111,12 @@ private void verifyFile(String filelocation, RotatingSaltProvider.SaltSnapshot s
public void testUploadNew() throws Exception {
RotatingSaltProvider.SaltSnapshot snapshot = makeSnapshot(Instant.ofEpochMilli(1740607938167L), Instant.ofEpochMilli(Instant.now().toEpochMilli() + 90002), 100);
RotatingSaltProvider.SaltSnapshot snapshot2 = makeSnapshot(Instant.ofEpochMilli(1740694476392L), Instant.ofEpochMilli(Instant.now().toEpochMilli() + 130000), 10);
+ JsonObject metadata = new JsonObject()
+ .put("version", 1742770328863L)
+ .put("generated", 1742770328)
+ .put("first_level", "FIRST-LEVEL")
+ .put("id_prefix", "a")
+ .put("id_secret", "ID-SECRET");
when(rotatingSaltProvider.getMetadata()).thenThrow(new CloudStorageException("The specified key does not exist: AmazonS3Exception: test-core-bucket"));
when(rotatingSaltProvider.getSnapshots()).thenReturn(null);
@@ -123,19 +129,16 @@ public void testUploadNew() throws Exception {
EncryptedSaltStoreWriter encryptedSaltStoreWriter = new EncryptedSaltStoreWriter(config, rotatingSaltProvider,
fileManager, taggableCloudStorage, versionGenerator, storeScope, rotatingCloudEncryptionKeyProvider, siteId);
- encryptedSaltStoreWriter.upload(snapshot);
+ encryptedSaltStoreWriter.upload(List.of(snapshot,snapshot2), metadata);
verify(fileManager).uploadMetadata(metadataCaptor.capture(), nameCaptor.capture(), locationCaptor.capture());
// Capture the metadata
JsonObject capturedMetadata = metadataCaptor.getValue();
- assertEquals(1, capturedMetadata.getJsonArray("salts").size(), "The 'salts' array should contain exactly 1 item");
- encryptedSaltStoreWriter.upload(snapshot2);
- verify(fileManager,times(2)).uploadMetadata(metadataCaptor.capture(), nameCaptor.capture(), locationCaptor.capture());
- capturedMetadata = metadataCaptor.getValue();
assertEquals(2, capturedMetadata.getJsonArray("salts").size(), "The 'salts' array should contain 2 items");
-
- verify(taggableCloudStorage,times(3)).upload(pathCaptor.capture(), cloudPathCaptor.capture(), any());
+ assertEquals(capturedMetadata.getString("first_level"), metadata.getValue("first_level"));
+ assertEquals(capturedMetadata.getString("id_prefix"), metadata.getValue("id_prefix"));
+ verify(taggableCloudStorage,times(2)).upload(pathCaptor.capture(), cloudPathCaptor.capture(), any());
verifyFile(pathCaptor.getValue(), snapshot);
}
@@ -171,7 +174,14 @@ public void testUnencryptedAndEncryptedBehavesTheSame() throws Exception {
EncryptedSaltStoreWriter encryptedSaltStoreWriter = new EncryptedSaltStoreWriter(config, rotatingSaltProvider,
fileManager, taggableCloudStorage, versionGenerator, storeScope, rotatingCloudEncryptionKeyProvider, siteId);
- encryptedSaltStoreWriter.upload(snapshot2);
+ JsonObject metadata = new JsonObject()
+ .put("version", 1742770328863L)
+ .put("generated", 1742770328)
+ .put("first_level", "FIRST-LEVEL")
+ .put("id_prefix", "a")
+ .put("id_secret", "ID-SECRET");
+
+ encryptedSaltStoreWriter.upload(List.of(snapshot2), metadata);
verify(fileManager,atLeastOnce()).uploadMetadata(metadataCaptor.capture(), nameCaptor.capture(), locationCaptor.capture());