Skip to content

Commit fc62e0e

Browse files
cody-constine-ttdcody-constine-ttd
committed
Got the encryption job working on salts
1 parent 36235b0 commit fc62e0e

File tree

7 files changed

+93
-38
lines changed

7 files changed

+93
-38
lines changed

‎src/main/java/com/uid2/admin/job/EncryptionJob/SaltEncryptionJob.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,17 +7,18 @@
77
import com.uid2.admin.util.PublicSiteUtil;
88
import com.uid2.shared.auth.OperatorKey;
99
import com.uid2.shared.model.SaltEntry;
10+
import com.uid2.shared.store.RotatingSaltProvider;
1011

1112
import java.util.Collection;
1213

1314
public class SaltEncryptionJob extends Job {
1415
private final Collection<OperatorKey> globalOperators;
15-
private final Collection<SaltEntry> saltEntries;
16-
private final MultiScopeStoreWriter<Collection<SaltEntry>> multiScopeStoreWriter;
16+
private final Collection<RotatingSaltProvider.SaltSnapshot> saltEntries;
17+
private final MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> multiScopeStoreWriter;
1718

1819
public SaltEncryptionJob(Collection<OperatorKey> globalOperators,
19-
Collection<SaltEntry> saltEntries,
20-
MultiScopeStoreWriter<Collection<SaltEntry>> multiScopeStoreWriter) {
20+
Collection<RotatingSaltProvider.SaltSnapshot> saltEntries,
21+
MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> multiScopeStoreWriter) {
2122
this.globalOperators = globalOperators;
2223
this.saltEntries = saltEntries;
2324
this.multiScopeStoreWriter = multiScopeStoreWriter;
@@ -31,9 +32,9 @@ public String getId() {
3132

3233
@Override
3334
public void execute() throws Exception {
34-
PrivateSiteDataMap<SaltEntry> desiredPrivateState = PrivateSiteUtil.getPrivateSaltEntries(saltEntries, globalOperators);
35+
PrivateSiteDataMap<RotatingSaltProvider.SaltSnapshot> desiredPrivateState = PrivateSiteUtil.getPrivateSaltEntries(saltEntries, globalOperators);
3536
multiScopeStoreWriter.uploadPrivateWithEncryption(desiredPrivateState, null);
36-
PrivateSiteDataMap<SaltEntry> desiredPublicState = PublicSiteUtil.getPublicSaltEntries(saltEntries, globalOperators);
37+
PrivateSiteDataMap<RotatingSaltProvider.SaltSnapshot> desiredPublicState = PublicSiteUtil.getPublicSaltEntries(saltEntries, globalOperators);
3738
multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, null);
3839
}
3940
}

‎src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java‎

Lines changed: 33 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,14 @@
1616
import com.uid2.shared.auth.RotatingOperatorKeyProvider;
1717
import com.uid2.shared.cloud.CloudUtils;
1818
import com.uid2.shared.cloud.ICloudStorage;
19+
import com.uid2.shared.cloud.TaggableCloudStorage;
1920
import com.uid2.shared.model.EncryptionKey;
2021
import com.uid2.shared.model.KeysetKey;
2122
import com.uid2.shared.model.Site;
2223
import com.uid2.shared.store.CloudPath;
2324
import com.uid2.admin.legacy.LegacyClientKey;
25+
import com.uid2.shared.store.RotatingEncryptedSaltProvider;
26+
import com.uid2.shared.store.RotatingSaltProvider;
2427
import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
2528
import com.uid2.shared.store.scope.GlobalScope;
2629
import io.vertx.core.json.JsonObject;
@@ -32,12 +35,12 @@
3235
public class EncryptedFilesSyncJob extends Job {
3336
private final JsonObject config;
3437
private final WriteLock writeLock;
35-
private final RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider;
38+
private final RotatingCloudEncryptionKeyProvider rotatingCloudEncryptionKeyProvider;
3639

3740
public EncryptedFilesSyncJob(JsonObject config, WriteLock writeLock, RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider) {
3841
this.config = config;
3942
this.writeLock = writeLock;
40-
this.RotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider;
43+
this.rotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider;
4144
}
4245

4346
@Override
@@ -47,20 +50,22 @@ public String getId() {
4750

4851
@Override
4952
public void execute() throws Exception {
50-
ICloudStorage cloudStorage = CloudUtils.createStorage(config.getString(Const.Config.CoreS3BucketProp), config);
53+
TaggableCloudStorage cloudStorage = CloudUtils.createStorage(config.getString(Const.Config.CoreS3BucketProp), config);
5154
FileStorage fileStorage = new TmpFileStorage();
5255
ObjectWriter jsonWriter = JsonUtil.createJsonWriter();
5356
Clock clock = new InstantClock();
5457
VersionGenerator versionGenerator = new EpochVersionGenerator(clock);
5558
FileManager fileManager = new FileManager(cloudStorage, fileStorage);
5659

60+
RotatingSaltProvider saltProvider = new RotatingSaltProvider(cloudStorage, config.getString(Const.Config.SaltsMetadataPathProp));
61+
5762
SiteStoreFactory siteStoreFactory = new SiteStoreFactory(
5863
cloudStorage,
5964
new CloudPath(config.getString(Const.Config.SitesMetadataPathProp)),
6065
jsonWriter,
6166
versionGenerator,
6267
clock,
63-
RotatingCloudEncryptionKeyProvider,
68+
rotatingCloudEncryptionKeyProvider,
6469
fileManager);
6570

6671
ClientKeyStoreFactory clientKeyStoreFactory = new ClientKeyStoreFactory(
@@ -69,15 +74,15 @@ public void execute() throws Exception {
6974
jsonWriter,
7075
versionGenerator,
7176
clock,
72-
RotatingCloudEncryptionKeyProvider,
77+
rotatingCloudEncryptionKeyProvider,
7378
fileManager);
7479

7580
EncryptionKeyStoreFactory encryptionKeyStoreFactory = new EncryptionKeyStoreFactory(
7681
cloudStorage,
7782
new CloudPath(config.getString(Const.Config.KeysMetadataPathProp)),
7883
versionGenerator,
7984
clock,
80-
RotatingCloudEncryptionKeyProvider,
85+
rotatingCloudEncryptionKeyProvider,
8186
fileManager);
8287

8388
KeyAclStoreFactory keyAclStoreFactory = new KeyAclStoreFactory(
@@ -86,7 +91,7 @@ public void execute() throws Exception {
8691
jsonWriter,
8792
versionGenerator,
8893
clock,
89-
RotatingCloudEncryptionKeyProvider,
94+
rotatingCloudEncryptionKeyProvider,
9095
fileManager);
9196

9297
KeysetStoreFactory keysetStoreFactory = new KeysetStoreFactory(
@@ -96,7 +101,7 @@ public void execute() throws Exception {
96101
versionGenerator,
97102
clock,
98103
fileManager,
99-
RotatingCloudEncryptionKeyProvider,
104+
rotatingCloudEncryptionKeyProvider,
100105
config.getBoolean(enableKeysetConfigProp));
101106

102107
KeysetKeyStoreFactory keysetKeyStoreFactory = new KeysetKeyStoreFactory(
@@ -105,15 +110,24 @@ public void execute() throws Exception {
105110
versionGenerator,
106111
clock,
107112
fileManager,
108-
RotatingCloudEncryptionKeyProvider,
113+
rotatingCloudEncryptionKeyProvider,
109114
config.getBoolean(enableKeysetConfigProp));
110115

116+
SaltStoreFactory saltStoreFactory = new SaltStoreFactory(
117+
config,
118+
new CloudPath(config.getString(Const.Config.SaltsMetadataPathProp)),
119+
fileManager,
120+
cloudStorage,
121+
versionGenerator,
122+
rotatingCloudEncryptionKeyProvider
123+
);
124+
111125
CloudPath operatorMetadataPath = new CloudPath(config.getString(Const.Config.OperatorsMetadataPathProp));
112126
GlobalScope operatorScope = new GlobalScope(operatorMetadataPath);
113127
RotatingOperatorKeyProvider operatorKeyProvider = new RotatingOperatorKeyProvider(cloudStorage, cloudStorage, operatorScope);
114128

115129
synchronized (writeLock) {
116-
RotatingCloudEncryptionKeyProvider.loadContent();
130+
rotatingCloudEncryptionKeyProvider.loadContent();
117131
operatorKeyProvider.loadContent(operatorKeyProvider.getMetadata());
118132
siteStoreFactory.getGlobalReader().loadContent(siteStoreFactory.getGlobalReader().getMetadata());
119133
clientKeyStoreFactory.getGlobalReader().loadContent();
@@ -123,7 +137,9 @@ public void execute() throws Exception {
123137
keysetStoreFactory.getGlobalReader().loadContent();
124138
keysetKeyStoreFactory.getGlobalReader().loadContent();
125139
}
140+
saltProvider.loadContent();
126141
}
142+
127143
Collection<OperatorKey> globalOperators = operatorKeyProvider.getAll();
128144
Collection<Site> globalSites = siteStoreFactory.getGlobalReader().getAllSites();
129145
Collection<LegacyClientKey> globalClients = clientKeyStoreFactory.getGlobalReader().getAll();
@@ -146,6 +162,10 @@ public void execute() throws Exception {
146162
fileManager,
147163
keyAclStoreFactory,
148164
MultiScopeStoreWriter::areMapsEqual);
165+
MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> saltWriter = new MultiScopeStoreWriter<>(
166+
fileManager,
167+
saltStoreFactory,
168+
MultiScopeStoreWriter::areCollectionsEqual);
149169

150170
SiteEncryptionJob siteEncryptionSyncJob = new SiteEncryptionJob(siteWriter, globalSites, globalOperators);
151171
ClientKeyEncryptionJob clientEncryptionSyncJob = new ClientKeyEncryptionJob(clientWriter, globalClients, globalOperators);
@@ -158,10 +178,13 @@ public void execute() throws Exception {
158178
encryptionKeyWriter
159179
);
160180
KeyAclEncryptionJob keyAclEncryptionSyncJob = new KeyAclEncryptionJob(keyAclWriter, globalOperators, globalKeyAcls);
181+
SaltEncryptionJob saltEncryptionJob = new SaltEncryptionJob(globalOperators, saltProvider.getSnapshots(), saltWriter);
182+
161183
siteEncryptionSyncJob.execute();
162184
clientEncryptionSyncJob.execute();
163185
encryptionKeyEncryptionSyncJob.execute();
164186
keyAclEncryptionSyncJob.execute();
187+
saltEncryptionJob.execute();
165188

166189
if(config.getBoolean(enableKeysetConfigProp)) {
167190
Map<Integer, Keyset> globalKeysets = keysetStoreFactory.getGlobalReader().getSnapshot().getAllKeysets();

‎src/main/java/com/uid2/admin/store/factory/SaltStoreFactory.java‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import com.uid2.admin.store.version.VersionGenerator;
55
import com.uid2.admin.store.writer.EncyptedSaltStoreWriter;
66
import com.uid2.admin.store.writer.StoreWriter;
7+
import com.uid2.shared.Const;
78
import com.uid2.shared.cloud.TaggableCloudStorage;
89
import com.uid2.shared.store.CloudPath;
910
import com.uid2.shared.store.RotatingEncryptedSaltProvider;
@@ -18,18 +19,16 @@
1819
public class SaltStoreFactory implements EncryptedStoreFactory<Collection<RotatingSaltProvider.SaltSnapshot>> {
1920
JsonObject config;
2021
CloudPath rootMetadatapath;
21-
RotatingSaltProvider saltProvider;
2222
FileManager fileManager;
2323
TaggableCloudStorage taggableCloudStorage;
2424
VersionGenerator versionGenerator;
2525
RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider;
2626

27-
public SaltStoreFactory(JsonObject config, CloudPath rootMetadataPath, RotatingSaltProvider saltProvider, FileManager fileManager,
27+
public SaltStoreFactory(JsonObject config, CloudPath rootMetadataPath, FileManager fileManager,
2828
TaggableCloudStorage taggableCloudStorage, VersionGenerator versionGenerator,
2929
RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) {
3030
this.config = config;
3131
this.rootMetadatapath = rootMetadataPath;
32-
this.saltProvider = saltProvider;
3332
this.fileManager = fileManager;
3433
this.taggableCloudStorage = taggableCloudStorage;
3534
this.versionGenerator = versionGenerator;
@@ -38,15 +37,15 @@ public SaltStoreFactory(JsonObject config, CloudPath rootMetadataPath, RotatingS
3837

3938
@Override
4039
public StoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> getEncryptedWriter(Integer siteId, boolean isPublic) {
41-
return new EncyptedSaltStoreWriter(config, saltProvider, fileManager, taggableCloudStorage, versionGenerator,
42-
new EncryptedScope(rootMetadatapath, siteId, isPublic), cloudEncryptionKeyProvider, siteId);
40+
EncryptedScope scope = new EncryptedScope(rootMetadatapath, siteId, isPublic);
41+
RotatingEncryptedSaltProvider saltProvider = new RotatingEncryptedSaltProvider(taggableCloudStorage,
42+
scope.resolve(new CloudPath(config.getString(Const.Config.SaltsMetadataPathProp))).toString(), cloudEncryptionKeyProvider );
43+
return new EncyptedSaltStoreWriter(config, saltProvider, fileManager, taggableCloudStorage, versionGenerator, scope, cloudEncryptionKeyProvider, siteId);
4344
}
4445

4546
@Override
4647
public StoreReader<Collection<RotatingSaltProvider.SaltSnapshot>> getEncryptedReader(Integer siteId, boolean isPublic) {
47-
return new RotatingEncryptedSaltProvider(taggableCloudStorage,
48-
new EncryptedScope(rootMetadatapath, siteId, isPublic).getMetadataPath().toString(),
49-
cloudEncryptionKeyProvider);
48+
return null;
5049
}
5150

5251
@Override

‎src/main/java/com/uid2/admin/store/writer/EncyptedSaltStoreWriter.java‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import com.uid2.shared.cloud.TaggableCloudStorage;
66
import com.uid2.shared.encryption.AesGcm;
77
import com.uid2.shared.model.CloudEncryptionKey;
8+
import com.uid2.shared.model.SaltEntry;
89
import com.uid2.shared.store.CloudPath;
910
import com.uid2.shared.store.RotatingSaltProvider;
1011
import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
@@ -13,8 +14,12 @@
1314
import org.slf4j.LoggerFactory;
1415
import io.vertx.core.json.JsonObject;
1516

17+
import java.io.BufferedWriter;
1618
import java.nio.charset.StandardCharsets;
19+
import java.nio.file.Files;
20+
import java.nio.file.Path;
1721
import java.util.Base64;
22+
import java.util.Collection;
1823

1924
public class EncyptedSaltStoreWriter extends SaltStoreWriter implements StoreWriter {
2025
private StoreScope scope;
@@ -37,11 +42,19 @@ protected java.lang.String getSaltSnapshotLocation(RotatingSaltProvider.SaltSnap
3742
}
3843

3944
@Override
40-
protected void upload(String data, String location) throws Exception {
45+
protected void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, String location) throws Exception {
4146
if (siteId == null) {
4247
throw new IllegalStateException("Site ID is not set.");
4348
}
4449

50+
StringBuilder stringBuilder = new StringBuilder();
51+
52+
for (SaltEntry entry: snapshot.getAllRotatingSalts()) {
53+
stringBuilder.append(entry.getId()).append(",").append(entry.getLastUpdated()).append(",").append(entry.getSalt()).append("\n");
54+
}
55+
56+
String data = stringBuilder.toString();
57+
4558
CloudEncryptionKey encryptionKey = null;
4659
try {
4760
encryptionKey = cloudEncryptionKeyProvider.getEncryptionKeyForSite(siteId);
@@ -59,13 +72,19 @@ protected void upload(String data, String location) throws Exception {
5972
throw new IllegalStateException("No Cloud Encryption keys available for encryption for site ID: " + siteId);
6073
}
6174

75+
final Path newSaltsFile = Files.createTempFile("salts", ".txt");
76+
try (BufferedWriter w = Files.newBufferedWriter(newSaltsFile)) {
77+
w.write(encryptedJson.encodePrettily());
78+
}
6279

63-
super.upload(encryptedJson.encodePrettily(), location);
80+
this.upload(newSaltsFile.toString(), location);
6481
}
6582

6683
@Override
6784
public void upload(Object data, JsonObject extraMeta) throws Exception {
68-
super.upload((RotatingSaltProvider.SaltSnapshot) data);
85+
for(RotatingSaltProvider.SaltSnapshot saltSnapshot: (Collection<RotatingSaltProvider.SaltSnapshot>) data) {
86+
super.upload(saltSnapshot);
87+
}
6988
}
7089

7190
@Override

‎src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import java.nio.file.Files;
1717
import java.nio.file.Path;
1818
import java.time.Instant;
19+
import java.util.Collections;
1920
import java.util.Comparator;
2021
import java.util.List;
2122
import java.util.Map;
@@ -46,17 +47,27 @@ public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
4647
final Instant now = Instant.now();
4748
final long generated = now.getEpochSecond();
4849

49-
final JsonObject metadata = provider.getMetadata();
50+
JsonObject metadata = null;
51+
try {
52+
metadata = provider.getMetadata();
53+
} catch (Exception e) {
54+
metadata = new JsonObject();
55+
}
5056
// bump up metadata version
5157
metadata.put("version", versionGenerator.getVersion());
5258
metadata.put("generated", generated);
5359

5460
final JsonArray snapshotsMetadata = new JsonArray();
5561
metadata.put("salts", snapshotsMetadata);
5662

57-
final List<RotatingSaltProvider.SaltSnapshot> snapshots = Stream.concat(provider.getSnapshots().stream(), Stream.of(data))
58-
.sorted(Comparator.comparing(RotatingSaltProvider.SaltSnapshot::getEffective))
59-
.collect(Collectors.toList());
63+
List<RotatingSaltProvider.SaltSnapshot> snapshots = null;
64+
try {
65+
snapshots = Stream.concat(provider.getSnapshots().stream(), Stream.of(data))
66+
.sorted(Comparator.comparing(RotatingSaltProvider.SaltSnapshot::getEffective))
67+
.collect(Collectors.toList());
68+
} catch (Exception e) {
69+
snapshots = List.of(data);
70+
}
6071
// of the currently effective snapshots keep only the most recent one
6172
RotatingSaltProvider.SaltSnapshot newestEffectiveSnapshot = snapshots.stream()
6273
.filter(snapshot -> snapshot.isEffective(now))
@@ -115,7 +126,7 @@ protected String getSaltSnapshotLocation(RotatingSaltProvider.SaltSnapshot snaps
115126
return saltSnapshotLocationPrefix + snapshot.getEffective().toEpochMilli();
116127
}
117128

118-
private void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, String location) throws Exception {
129+
protected void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, String location) throws Exception {
119130
// do not overwrite existing files
120131
if (!cloudStorage.list(location).isEmpty()) {
121132
// update the tags on the file to ensure it is still marked as current

‎src/main/java/com/uid2/admin/util/PrivateSiteUtil.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import com.uid2.shared.model.KeysetKey;
99
import com.uid2.shared.model.SaltEntry;
1010
import com.uid2.shared.model.Site;
11+
import com.uid2.shared.store.RotatingSaltProvider;
1112
import org.slf4j.Logger;
1213
import org.slf4j.LoggerFactory;
1314

@@ -312,10 +313,10 @@ public static PrivateSiteDataMap<KeysetKey> getKeysetKeys(Collection<OperatorKey
312313
return result;
313314
}
314315

315-
public static PrivateSiteDataMap<SaltEntry> getPrivateSaltEntries(
316-
Collection<SaltEntry> globalSaltEntries,
316+
public static PrivateSiteDataMap<RotatingSaltProvider.SaltSnapshot> getPrivateSaltEntries(
317+
Collection<RotatingSaltProvider.SaltSnapshot> globalSaltEntries,
317318
Collection<OperatorKey> operators) {
318-
final PrivateSiteDataMap<SaltEntry> result = getPrivateSites(operators);
319+
final PrivateSiteDataMap<RotatingSaltProvider.SaltSnapshot> result = getPrivateSites(operators);
319320

320321
globalSaltEntries.forEach(saltEntry -> {
321322
result.forEach((publicSiteId, publicSiteData) -> {

‎src/main/java/com/uid2/admin/util/PublicSiteUtil.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import com.uid2.shared.model.KeysetKey;
1111
import com.uid2.shared.model.SaltEntry;
1212
import com.uid2.shared.model.Site;
13+
import com.uid2.shared.store.RotatingSaltProvider;
1314
import org.slf4j.Logger;
1415
import org.slf4j.LoggerFactory;
1516

@@ -130,10 +131,10 @@ public static PrivateSiteDataMap<KeysetKey> getPublicKeysetKeys(
130131
return result;
131132
}
132133

133-
public static PrivateSiteDataMap<SaltEntry> getPublicSaltEntries(
134-
Collection<SaltEntry> globalSaltEntries,
134+
public static PrivateSiteDataMap<RotatingSaltProvider.SaltSnapshot> getPublicSaltEntries(
135+
Collection<RotatingSaltProvider.SaltSnapshot> globalSaltEntries,
135136
Collection<OperatorKey> operators) {
136-
final PrivateSiteDataMap<SaltEntry> result = getPublicSitesMap(operators);
137+
final PrivateSiteDataMap<RotatingSaltProvider.SaltSnapshot> result = getPublicSitesMap(operators);
137138

138139
globalSaltEntries.forEach(saltEntry -> {
139140
result.forEach((publicSiteId, publicSiteData) -> {

0 commit comments

Comments
 (0)