Skip to content

Commit 098c800

Browse files
abuabraham-ttdabuabraham-ttd
authored
Use EncryptedFilesSyncJob (#400)
* Use EncryptedFilesSyncJob and refactor Encrypted/SaltWriter
1 parent 30845e4 commit 098c800

File tree

9 files changed

+52
-70
lines changed

9 files changed

+52
-70
lines changed

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
<groupId>com.uid2</groupId>
88
<artifactId>uid2-admin</artifactId>
9-
<version>5.18.8-alpha-127-SNAPSHOT</version>
9+
<version>5.18.9-alpha-128-SNAPSHOT</version>
1010

1111
<properties>
1212
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

‎src/main/java/com/uid2/admin/Main.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,8 @@ public void run() {
265265
new SaltService(auth, writeLock, saltStoreWriter, saltProvider, saltRotation),
266266
new SiteService(auth, writeLock, siteStoreWriter, siteProvider, clientKeyProvider),
267267
new PartnerConfigService(auth, writeLock, partnerStoreWriter, partnerConfigProvider),
268-
new PrivateSiteDataRefreshService(auth, jobDispatcher, writeLock, config, rotatingCloudEncryptionKeyProvider),
268+
new PrivateSiteDataRefreshService(auth, jobDispatcher, writeLock, config),
269+
new EncryptedFilesSyncService(auth, jobDispatcher, writeLock, config, rotatingCloudEncryptionKeyProvider),
269270
new JobDispatcherService(auth, jobDispatcher),
270271
new SearchService(auth, clientKeyProvider, operatorKeyProvider),
271272
new CloudEncryptionKeyService(auth, cloudEncryptionKeyManager)

‎src/main/java/com/uid2/admin/job/EncryptionJob/SaltEncryptionJob.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,14 @@
1414

1515
public class SaltEncryptionJob extends Job {
1616
private final Collection<OperatorKey> globalOperators;
17-
private final Collection<RotatingSaltProvider.SaltSnapshot> saltEntries;
17+
private final RotatingSaltProvider saltProvider;
1818
private final MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> multiScopeStoreWriter;
1919

2020
public SaltEncryptionJob(Collection<OperatorKey> globalOperators,
21-
Collection<RotatingSaltProvider.SaltSnapshot> saltEntries,
21+
RotatingSaltProvider saltProvider,
2222
MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> multiScopeStoreWriter) {
2323
this.globalOperators = globalOperators;
24-
this.saltEntries = saltEntries;
24+
this.saltProvider = saltProvider;
2525
this.multiScopeStoreWriter = multiScopeStoreWriter;
2626
}
2727

@@ -34,8 +34,8 @@ public String getId() {
3434
@Override
3535
public void execute() throws Exception {
3636
List<Integer> desiredPrivateState = PrivateSiteUtil.getPrivateSaltSites(globalOperators);
37-
multiScopeStoreWriter.uploadPrivateWithEncryption(desiredPrivateState, saltEntries, null);
37+
multiScopeStoreWriter.uploadPrivateWithEncryption(desiredPrivateState, saltProvider.getSnapshots(), saltProvider.getMetadata());
3838
List<Integer> desiredPublicState = PublicSiteUtil.getPublicSaltSites(globalOperators);
39-
multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, saltEntries, null);
39+
multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, saltProvider.getSnapshots(), saltProvider.getMetadata());
4040
}
4141
}

‎src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,7 @@ public void execute() throws Exception {
195195
encryptionKeyWriter
196196
);
197197
KeyAclEncryptionJob keyAclEncryptionSyncJob = new KeyAclEncryptionJob(keyAclWriter, globalOperators, globalKeyAcls);
198-
SaltEncryptionJob saltEncryptionJob = new SaltEncryptionJob(globalOperators, saltProvider.getSnapshots(), saltWriter);
198+
SaltEncryptionJob saltEncryptionJob = new SaltEncryptionJob(globalOperators, saltProvider, saltWriter);
199199
ClientSideKeypairEncryptionJob clientSideKeypairEncryptionJob = new ClientSideKeypairEncryptionJob(globalOperators, globalClientSideKeypair, clientSideKeypairWriter);
200200

201201
siteEncryptionSyncJob.execute();

‎src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java‎

Lines changed: 3 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,6 @@ public class EncryptedSaltStoreWriter extends SaltStoreWriter implements StoreWr
2828
private RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider;
2929
private Integer siteId;
3030

31-
private final List<RotatingSaltProvider.SaltSnapshot> previousSeenSnapshots = new ArrayList<>();
32-
3331
private static final Logger LOGGER = LoggerFactory.getLogger(EncryptedSaltStoreWriter.class);
3432
public EncryptedSaltStoreWriter(JsonObject config, RotatingSaltProvider provider, FileManager fileManager,
3533
TaggableCloudStorage cloudStorage, VersionGenerator versionGenerator, StoreScope scope,
@@ -51,12 +49,6 @@ protected void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, S
5149
throw new IllegalStateException("Site ID is not set.");
5250
}
5351

54-
if (!cloudStorage.list(location).isEmpty()) {
55-
// update the tags on the file to ensure it is still marked as current
56-
this.setStatusTagToCurrent(location);
57-
return;
58-
}
59-
6052
StringBuilder stringBuilder = new StringBuilder();
6153

6254
for (SaltEntry entry: snapshot.getAllRotatingSalts()) {
@@ -89,27 +81,12 @@ protected void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, S
8981

9082
this.upload(newSaltsFile.toString(), location);
9183
}
92-
93-
@Override
94-
protected void refreshProvider() {
95-
// we do not need to refresh the provider on encrypted writers
96-
}
97-
98-
@Override
99-
protected List<RotatingSaltProvider.SaltSnapshot> getSnapshots(RotatingSaltProvider.SaltSnapshot data){
100-
/*
101-
Since metadata.json is overwritten during the process, we maintain a history of all snapshots seen so far.
102-
On the final write, we append this history to metadata.json to ensure no snapshots are lost.
103-
*/
104-
this.previousSeenSnapshots.add(data);
105-
return this.previousSeenSnapshots;
106-
}
10784

10885
@Override
10986
public void upload(Object data, JsonObject extraMeta) throws Exception {
110-
for(RotatingSaltProvider.SaltSnapshot saltSnapshot: (Collection<RotatingSaltProvider.SaltSnapshot>) data) {
111-
super.upload(saltSnapshot);
112-
}
87+
@SuppressWarnings("unchecked")
88+
List<RotatingSaltProvider.SaltSnapshot> snapshots = new ArrayList<>((Collection<RotatingSaltProvider.SaltSnapshot>) data);
89+
this.buildAndUploadMetadata(extraMeta, this.uploadSnapshotsAndGetMetadata(snapshots));
11390
}
11491

11592
@Override

‎src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java‎

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ public SaltStoreWriter(JsonObject config, RotatingSaltProvider provider, FileMan
4343
this.versionGenerator = versionGenerator;
4444
}
4545

46-
protected List<RotatingSaltProvider.SaltSnapshot> getSnapshots(RotatingSaltProvider.SaltSnapshot data){
46+
private List<RotatingSaltProvider.SaltSnapshot> getSnapshots(RotatingSaltProvider.SaltSnapshot data){
4747
if (provider.getSnapshots() == null) {
4848
throw new IllegalStateException("Snapshots cannot be null");
4949
}
@@ -75,10 +75,7 @@ protected List<RotatingSaltProvider.SaltSnapshot> getSnapshots(RotatingSaltProvi
7575
return filteredSnapshots;
7676
}
7777

78-
public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
79-
final Instant now = Instant.now();
80-
final long generated = now.getEpochSecond();
81-
78+
protected JsonObject getMetadata() throws Exception {
8279
JsonObject metadata = null;
8380
try {
8481
metadata = provider.getMetadata();
@@ -89,15 +86,20 @@ public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
8986
throw e;
9087
}
9188
}
92-
// bump up metadata version
89+
return metadata;
90+
}
91+
92+
protected void buildAndUploadMetadata(JsonObject metadata, JsonArray snapshotsMetadata ) throws Exception{
93+
final Instant now = Instant.now();
94+
final long generated = now.getEpochSecond();
9395
metadata.put("version", versionGenerator.getVersion());
9496
metadata.put("generated", generated);
95-
96-
final JsonArray snapshotsMetadata = new JsonArray();
9797
metadata.put("salts", snapshotsMetadata);
98+
fileManager.uploadMetadata(metadata, "salts", new CloudPath(provider.getMetadataPath()));
99+
}
98100

99-
List<RotatingSaltProvider.SaltSnapshot> snapshots = this.getSnapshots(data);
100-
101+
protected JsonArray uploadSnapshotsAndGetMetadata(List<RotatingSaltProvider.SaltSnapshot> snapshots) throws Exception {
102+
final JsonArray snapshotsMetadata = new JsonArray();
101103
for (RotatingSaltProvider.SaltSnapshot snapshot : snapshots) {
102104
final String location = getSaltSnapshotLocation(snapshot);
103105
final JsonObject snapshotMetadata = new JsonObject();
@@ -106,17 +108,19 @@ public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
106108
snapshotMetadata.put("location", location);
107109
snapshotMetadata.put("size", snapshot.getAllRotatingSalts().length);
108110
snapshotsMetadata.add(snapshotMetadata);
109-
110111
uploadSaltsSnapshot(snapshot, location);
111112
}
113+
return snapshotsMetadata;
114+
}
112115

113-
fileManager.uploadMetadata(metadata, "salts", new CloudPath(provider.getMetadataPath()));
114-
115-
// refresh manually
116+
public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
117+
JsonObject metadata = this.getMetadata();
118+
List<RotatingSaltProvider.SaltSnapshot> snapshots = this.getSnapshots(data);
119+
this.buildAndUploadMetadata(metadata, this.uploadSnapshotsAndGetMetadata(snapshots));
116120
refreshProvider();
117121
}
118122

119-
protected void refreshProvider() throws Exception {
123+
private void refreshProvider() throws Exception {
120124
provider.loadContent();
121125
}
122126

‎src/main/java/com/uid2/admin/vertx/service/EncryptedFilesSyncService.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,14 +38,14 @@ public EncryptedFilesSyncService(
3838

3939
@Override
4040
public void setupRoutes(Router router) {
41-
router.post("/api/encrypted-files/refresh").handler(auth.handle((ctx) -> {
41+
router.post("/api/encrypted-files/refresh").blockingHandler(auth.handle((ctx) -> {
4242
synchronized (writeLock) {
4343
this.handleEncryptedFileSync(ctx);
4444
}
4545
},
4646
Role.MAINTAINER, Role.PRIVATE_OPERATOR_SYNC));
4747

48-
router.post("/api/encrypted-files/syncNow").handler(auth.handle(
48+
router.post("/api/encrypted-files/syncNow").blockingHandler(auth.handle(
4949
this::handleEncryptedFileSyncNow,
5050
Role.MAINTAINER, Role.PRIVATE_OPERATOR_SYNC));
5151
}

‎src/main/java/com/uid2/admin/vertx/service/PrivateSiteDataRefreshService.java‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,6 @@
77
import com.uid2.admin.job.jobsync.keyset.ReplaceSharingTypesWithSitesJob;
88
import com.uid2.admin.vertx.WriteLock;
99
import com.uid2.shared.auth.Role;
10-
import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
11-
import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
1210
import io.vertx.core.json.JsonObject;
1311
import org.slf4j.Logger;
1412
import org.slf4j.LoggerFactory;
@@ -24,19 +22,16 @@ public class PrivateSiteDataRefreshService implements IService {
2422
private final JobDispatcher jobDispatcher;
2523
private final WriteLock writeLock;
2624
private final JsonObject config;
27-
private final RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider;
2825

2926
public PrivateSiteDataRefreshService(
3027
AdminAuthMiddleware auth,
3128
JobDispatcher jobDispatcher,
3229
WriteLock writeLock,
33-
JsonObject config,
34-
RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider) {
30+
JsonObject config) {
3531
this.auth = auth;
3632
this.jobDispatcher = jobDispatcher;
3733
this.writeLock = writeLock;
3834
this.config = config;
39-
this.RotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider;
4035
}
4136

4237
@Override
@@ -64,9 +59,6 @@ private void handlePrivateSiteDataGenerate(RoutingContext rc) {
6459
PrivateSiteDataSyncJob job = new PrivateSiteDataSyncJob(config, writeLock);
6560
jobDispatcher.enqueue(job);
6661

67-
EncryptedFilesSyncJob encryptedFileSyncJob = new EncryptedFilesSyncJob(config, writeLock, RotatingCloudEncryptionKeyProvider);
68-
jobDispatcher.enqueue(encryptedFileSyncJob);
69-
7062
rc.response().end("OK");
7163
} catch (Exception e) {
7264
LOGGER.error(e.getMessage(), e);
@@ -86,8 +78,6 @@ private void handlePrivateSiteDataGenerateNow(RoutingContext rc) {
8678
CompletableFuture<Boolean> privateSiteDataSyncJobFuture = jobDispatcher.executeNextJob();
8779
privateSiteDataSyncJobFuture.get();
8880

89-
EncryptedFilesSyncJob encryptedFileSyncJob = new EncryptedFilesSyncJob(config, writeLock, RotatingCloudEncryptionKeyProvider);
90-
jobDispatcher.enqueue(encryptedFileSyncJob);
9181
CompletableFuture<Boolean> encryptedFileSyncJobFuture = jobDispatcher.executeNextJob();
9282
encryptedFileSyncJobFuture.get();
9383

‎src/test/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriterTest.java‎

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,12 @@ private void verifyFile(String filelocation, RotatingSaltProvider.SaltSnapshot s
111111
public void testUploadNew() throws Exception {
112112
RotatingSaltProvider.SaltSnapshot snapshot = makeSnapshot(Instant.ofEpochMilli(1740607938167L), Instant.ofEpochMilli(Instant.now().toEpochMilli() + 90002), 100);
113113
RotatingSaltProvider.SaltSnapshot snapshot2 = makeSnapshot(Instant.ofEpochMilli(1740694476392L), Instant.ofEpochMilli(Instant.now().toEpochMilli() + 130000), 10);
114+
JsonObject metadata = new JsonObject()
115+
.put("version", 1742770328863L)
116+
.put("generated", 1742770328)
117+
.put("first_level", "FIRST-LEVEL")
118+
.put("id_prefix", "a")
119+
.put("id_secret", "ID-SECRET");
114120
when(rotatingSaltProvider.getMetadata()).thenThrow(new CloudStorageException("The specified key does not exist: AmazonS3Exception: test-core-bucket"));
115121
when(rotatingSaltProvider.getSnapshots()).thenReturn(null);
116122

@@ -123,19 +129,16 @@ public void testUploadNew() throws Exception {
123129
EncryptedSaltStoreWriter encryptedSaltStoreWriter = new EncryptedSaltStoreWriter(config, rotatingSaltProvider,
124130
fileManager, taggableCloudStorage, versionGenerator, storeScope, rotatingCloudEncryptionKeyProvider, siteId);
125131

126-
encryptedSaltStoreWriter.upload(snapshot);
132+
encryptedSaltStoreWriter.upload(List.of(snapshot,snapshot2), metadata);
127133
verify(fileManager).uploadMetadata(metadataCaptor.capture(), nameCaptor.capture(), locationCaptor.capture());
128134

129135
// Capture the metadata
130136
JsonObject capturedMetadata = metadataCaptor.getValue();
131-
assertEquals(1, capturedMetadata.getJsonArray("salts").size(), "The 'salts' array should contain exactly 1 item");
132-
encryptedSaltStoreWriter.upload(snapshot2);
133137

134-
verify(fileManager,times(2)).uploadMetadata(metadataCaptor.capture(), nameCaptor.capture(), locationCaptor.capture());
135-
capturedMetadata = metadataCaptor.getValue();
136138
assertEquals(2, capturedMetadata.getJsonArray("salts").size(), "The 'salts' array should contain 2 items");
137-
138-
verify(taggableCloudStorage,times(3)).upload(pathCaptor.capture(), cloudPathCaptor.capture(), any());
139+
assertEquals(capturedMetadata.getString("first_level"), metadata.getValue("first_level"));
140+
assertEquals(capturedMetadata.getString("id_prefix"), metadata.getValue("id_prefix"));
141+
verify(taggableCloudStorage,times(2)).upload(pathCaptor.capture(), cloudPathCaptor.capture(), any());
139142

140143
verifyFile(pathCaptor.getValue(), snapshot);
141144
}
@@ -171,7 +174,14 @@ public void testUnencryptedAndEncryptedBehavesTheSame() throws Exception {
171174
EncryptedSaltStoreWriter encryptedSaltStoreWriter = new EncryptedSaltStoreWriter(config, rotatingSaltProvider,
172175
fileManager, taggableCloudStorage, versionGenerator, storeScope, rotatingCloudEncryptionKeyProvider, siteId);
173176

174-
encryptedSaltStoreWriter.upload(snapshot2);
177+
JsonObject metadata = new JsonObject()
178+
.put("version", 1742770328863L)
179+
.put("generated", 1742770328)
180+
.put("first_level", "FIRST-LEVEL")
181+
.put("id_prefix", "a")
182+
.put("id_secret", "ID-SECRET");
183+
184+
encryptedSaltStoreWriter.upload(List.of(snapshot2), metadata);
175185

176186
verify(fileManager,atLeastOnce()).uploadMetadata(metadataCaptor.capture(), nameCaptor.capture(), locationCaptor.capture());
177187

0 commit comments

Comments
 (0)