Skip to content

Commit 08b94bb

Browse files
cody-constine-ttdcody-constine-ttd
authored
Merge pull request #353 from IABTechLab/cbc-UID2-4571-cloud-encryption-cleanup
Adding more work on salt encryption
2 parents 6885a54 + 6c1bbbe commit 08b94bb

17 files changed

+612
-33
lines changed

‎Dockerfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,5 @@ CMD java \
2626
-Djava.security.egd=file:/dev/./urandom \
2727
-Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \
2828
-Dlogback.configurationFile=${LOGBACK_CONF} \
29+
-Xmx4g \
2930
-jar ${JAR_NAME}-${JAR_VERSION}.jar

‎pom.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
<groupId>com.uid2</groupId>
88
<artifactId>uid2-admin</artifactId>
9-
<version>5.16.0</version>
9+
<version>5.16.8-alpha-103-SNAPSHOT</version>
1010

1111
<properties>
1212
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -16,7 +16,7 @@
1616
<!-- check micrometer.version vertx-micrometer-metrics consumes before bumping up -->
1717
<micrometer.version>1.12.2</micrometer.version>
1818
<junit-jupiter.version>5.11.2</junit-jupiter.version>
19-
<uid2-shared.version>8.0.6</uid2-shared.version>
19+
<uid2-shared.version>8.0.25</uid2-shared.version>
2020
<okta-jwt.version>0.5.10</okta-jwt.version>
2121
<image.version>${project.version}</image.version>
2222
</properties>

‎src/main/java/com/uid2/admin/job/EncryptionJob/ClientKeyEncryptionJob.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ public String getId() {
3333
public void execute() throws Exception {
3434
PrivateSiteDataMap<LegacyClientKey> desiredPrivateState = PrivateSiteUtil.getClientKeys(globalOperators, globalClientKeys);
3535
multiScopeStoreWriter.uploadPrivateWithEncryption(desiredPrivateState, null);
36-
PrivateSiteDataMap<LegacyClientKey> desiredPublicState = PublicSiteUtil.getPublicClients(globalClientKeys,globalOperators);
36+
PrivateSiteDataMap<LegacyClientKey> desiredPublicState = PublicSiteUtil.getPublicClients(globalClientKeys, globalOperators);
3737
multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, null);
3838
}
3939
}

‎src/main/java/com/uid2/admin/job/EncryptionJob/ClientSideKeypairEncryptionJob.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package com.uid2.admin.job.EncryptionJob;
2+
3+
import com.uid2.admin.job.model.Job;
4+
import com.uid2.admin.model.PrivateSiteDataMap;
5+
import com.uid2.admin.store.MultiScopeStoreWriter;
6+
import com.uid2.admin.util.PublicSiteUtil;
7+
import com.uid2.shared.auth.OperatorKey;
8+
import com.uid2.shared.model.ClientSideKeypair;
9+
10+
import java.util.Collection;
11+
12+
public class ClientSideKeypairEncryptionJob extends Job {
13+
private final Collection<OperatorKey> globalOperators;
14+
private final Collection<ClientSideKeypair> globalClientSideKeypairs;
15+
16+
private final MultiScopeStoreWriter<Collection<ClientSideKeypair>> multiScopeStoreWriter;
17+
18+
public ClientSideKeypairEncryptionJob(Collection<OperatorKey> globalOperators, Collection<ClientSideKeypair> globalClientSideKeypairs,
19+
MultiScopeStoreWriter<Collection<ClientSideKeypair>> multiScopeStoreWriter) {
20+
this.globalOperators = globalOperators;
21+
this.globalClientSideKeypairs = globalClientSideKeypairs;
22+
this.multiScopeStoreWriter = multiScopeStoreWriter;
23+
}
24+
25+
@Override
26+
public String getId() {
27+
return "cloud-encryption-sync-clientside-keypair";
28+
}
29+
30+
@Override
31+
public void execute() throws Exception {
32+
// Only public operators support clientside keypair
33+
PrivateSiteDataMap<ClientSideKeypair> desiredPublicState = PublicSiteUtil.getPublicClientKeypairs(globalClientSideKeypairs, globalOperators);
34+
multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, null);
35+
}
36+
}

‎src/main/java/com/uid2/admin/job/EncryptionJob/SaltEncryptionJob.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
package com.uid2.admin.job.EncryptionJob;
2+
3+
import com.uid2.admin.job.model.Job;
4+
import com.uid2.admin.model.PrivateSiteDataMap;
5+
import com.uid2.admin.store.MultiScopeStoreWriter;
6+
import com.uid2.admin.util.PrivateSiteUtil;
7+
import com.uid2.admin.util.PublicSiteUtil;
8+
import com.uid2.shared.auth.OperatorKey;
9+
import com.uid2.shared.model.SaltEntry;
10+
import com.uid2.shared.store.RotatingSaltProvider;
11+
12+
import java.util.Collection;
13+
import java.util.List;
14+
15+
public class SaltEncryptionJob extends Job {
16+
private final Collection<OperatorKey> globalOperators;
17+
private final Collection<RotatingSaltProvider.SaltSnapshot> saltEntries;
18+
private final MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> multiScopeStoreWriter;
19+
20+
public SaltEncryptionJob(Collection<OperatorKey> globalOperators,
21+
Collection<RotatingSaltProvider.SaltSnapshot> saltEntries,
22+
MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> multiScopeStoreWriter) {
23+
this.globalOperators = globalOperators;
24+
this.saltEntries = saltEntries;
25+
this.multiScopeStoreWriter = multiScopeStoreWriter;
26+
}
27+
28+
29+
@Override
30+
public String getId() {
31+
return "cloud-encryption-sync-salts";
32+
}
33+
34+
@Override
35+
public void execute() throws Exception {
36+
List<Integer> desiredPrivateState = PrivateSiteUtil.getPrivateSaltSites(globalOperators);
37+
multiScopeStoreWriter.uploadPrivateWithEncryption(desiredPrivateState, saltEntries, null);
38+
List<Integer> desiredPublicState = PublicSiteUtil.getPublicSaltSites(globalOperators);
39+
multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, saltEntries, null);
40+
}
41+
}

‎src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java‎

Lines changed: 52 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import com.fasterxml.jackson.databind.ObjectWriter;
44
import com.uid2.admin.job.EncryptionJob.*;
5-
import com.uid2.admin.job.EncryptionJob.ClientKeyEncryptionJob;
65
import com.uid2.admin.job.model.Job;
76
import com.uid2.admin.store.*;
87
import com.uid2.admin.store.factory.*;
@@ -17,12 +16,15 @@
1716
import com.uid2.shared.auth.RotatingOperatorKeyProvider;
1817
import com.uid2.shared.cloud.CloudUtils;
1918
import com.uid2.shared.cloud.ICloudStorage;
19+
import com.uid2.shared.cloud.TaggableCloudStorage;
20+
import com.uid2.shared.model.ClientSideKeypair;
2021
import com.uid2.shared.model.EncryptionKey;
2122
import com.uid2.shared.model.KeysetKey;
2223
import com.uid2.shared.model.Site;
2324
import com.uid2.shared.store.CloudPath;
2425
import com.uid2.admin.legacy.LegacyClientKey;
25-
import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
26+
import com.uid2.shared.store.EncryptedRotatingSaltProvider;
27+
import com.uid2.shared.store.RotatingSaltProvider;
2628
import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
2729
import com.uid2.shared.store.scope.GlobalScope;
2830
import io.vertx.core.json.JsonObject;
@@ -34,12 +36,12 @@
3436
public class EncryptedFilesSyncJob extends Job {
3537
private final JsonObject config;
3638
private final WriteLock writeLock;
37-
private final RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider;
39+
private final RotatingCloudEncryptionKeyProvider rotatingCloudEncryptionKeyProvider;
3840

3941
public EncryptedFilesSyncJob(JsonObject config, WriteLock writeLock, RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider) {
4042
this.config = config;
4143
this.writeLock = writeLock;
42-
this.RotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider;
44+
this.rotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider;
4345
}
4446

4547
@Override
@@ -49,20 +51,22 @@ public String getId() {
4951

5052
@Override
5153
public void execute() throws Exception {
52-
ICloudStorage cloudStorage = CloudUtils.createStorage(config.getString(Const.Config.CoreS3BucketProp), config);
54+
TaggableCloudStorage cloudStorage = CloudUtils.createStorage(config.getString(Const.Config.CoreS3BucketProp), config);
5355
FileStorage fileStorage = new TmpFileStorage();
5456
ObjectWriter jsonWriter = JsonUtil.createJsonWriter();
5557
Clock clock = new InstantClock();
5658
VersionGenerator versionGenerator = new EpochVersionGenerator(clock);
5759
FileManager fileManager = new FileManager(cloudStorage, fileStorage);
5860

61+
RotatingSaltProvider saltProvider = new RotatingSaltProvider(cloudStorage, config.getString(Const.Config.SaltsMetadataPathProp));
62+
5963
SiteStoreFactory siteStoreFactory = new SiteStoreFactory(
6064
cloudStorage,
6165
new CloudPath(config.getString(Const.Config.SitesMetadataPathProp)),
6266
jsonWriter,
6367
versionGenerator,
6468
clock,
65-
RotatingCloudEncryptionKeyProvider,
69+
rotatingCloudEncryptionKeyProvider,
6670
fileManager);
6771

6872
ClientKeyStoreFactory clientKeyStoreFactory = new ClientKeyStoreFactory(
@@ -71,15 +75,15 @@ public void execute() throws Exception {
7175
jsonWriter,
7276
versionGenerator,
7377
clock,
74-
RotatingCloudEncryptionKeyProvider,
78+
rotatingCloudEncryptionKeyProvider,
7579
fileManager);
7680

7781
EncryptionKeyStoreFactory encryptionKeyStoreFactory = new EncryptionKeyStoreFactory(
7882
cloudStorage,
7983
new CloudPath(config.getString(Const.Config.KeysMetadataPathProp)),
8084
versionGenerator,
8185
clock,
82-
RotatingCloudEncryptionKeyProvider,
86+
rotatingCloudEncryptionKeyProvider,
8387
fileManager);
8488

8589
KeyAclStoreFactory keyAclStoreFactory = new KeyAclStoreFactory(
@@ -88,7 +92,7 @@ public void execute() throws Exception {
8892
jsonWriter,
8993
versionGenerator,
9094
clock,
91-
RotatingCloudEncryptionKeyProvider,
95+
rotatingCloudEncryptionKeyProvider,
9296
fileManager);
9397

9498
KeysetStoreFactory keysetStoreFactory = new KeysetStoreFactory(
@@ -98,7 +102,7 @@ public void execute() throws Exception {
98102
versionGenerator,
99103
clock,
100104
fileManager,
101-
RotatingCloudEncryptionKeyProvider,
105+
rotatingCloudEncryptionKeyProvider,
102106
config.getBoolean(enableKeysetConfigProp));
103107

104108
KeysetKeyStoreFactory keysetKeyStoreFactory = new KeysetKeyStoreFactory(
@@ -107,15 +111,33 @@ public void execute() throws Exception {
107111
versionGenerator,
108112
clock,
109113
fileManager,
110-
RotatingCloudEncryptionKeyProvider,
114+
rotatingCloudEncryptionKeyProvider,
111115
config.getBoolean(enableKeysetConfigProp));
112116

117+
SaltStoreFactory saltStoreFactory = new SaltStoreFactory(
118+
config,
119+
new CloudPath(config.getString(Const.Config.SaltsMetadataPathProp)),
120+
fileManager,
121+
cloudStorage,
122+
versionGenerator,
123+
rotatingCloudEncryptionKeyProvider
124+
);
125+
126+
ClientSideKeypairStoreFactory clientSideKeypairStoreFactory = new ClientSideKeypairStoreFactory(
127+
cloudStorage,
128+
new CloudPath(config.getString(Const.Config.ClientSideKeypairsMetadataPathProp)),
129+
versionGenerator,
130+
clock,
131+
rotatingCloudEncryptionKeyProvider,
132+
fileManager
133+
);
134+
113135
CloudPath operatorMetadataPath = new CloudPath(config.getString(Const.Config.OperatorsMetadataPathProp));
114136
GlobalScope operatorScope = new GlobalScope(operatorMetadataPath);
115137
RotatingOperatorKeyProvider operatorKeyProvider = new RotatingOperatorKeyProvider(cloudStorage, cloudStorage, operatorScope);
116138

117139
synchronized (writeLock) {
118-
RotatingCloudEncryptionKeyProvider.loadContent();
140+
rotatingCloudEncryptionKeyProvider.loadContent();
119141
operatorKeyProvider.loadContent(operatorKeyProvider.getMetadata());
120142
siteStoreFactory.getGlobalReader().loadContent(siteStoreFactory.getGlobalReader().getMetadata());
121143
clientKeyStoreFactory.getGlobalReader().loadContent();
@@ -125,13 +147,18 @@ public void execute() throws Exception {
125147
keysetStoreFactory.getGlobalReader().loadContent();
126148
keysetKeyStoreFactory.getGlobalReader().loadContent();
127149
}
150+
saltProvider.loadContent();
151+
clientSideKeypairStoreFactory.getGlobalReader().loadContent();
128152
}
153+
129154
Collection<OperatorKey> globalOperators = operatorKeyProvider.getAll();
130155
Collection<Site> globalSites = siteStoreFactory.getGlobalReader().getAllSites();
131156
Collection<LegacyClientKey> globalClients = clientKeyStoreFactory.getGlobalReader().getAll();
132157
Collection<EncryptionKey> globalEncryptionKeys = encryptionKeyStoreFactory.getGlobalReader().getSnapshot().getActiveKeySet();
133158
Integer globalMaxKeyId = encryptionKeyStoreFactory.getGlobalReader().getMetadata().getInteger("max_key_id");
134159
Map<Integer, EncryptionKeyAcl> globalKeyAcls = keyAclStoreFactory.getGlobalReader().getSnapshot().getAllAcls();
160+
Collection<ClientSideKeypair> globalClientSideKeypair = clientSideKeypairStoreFactory.getGlobalReader().getAll();
161+
135162
MultiScopeStoreWriter<Collection<Site>> siteWriter = new MultiScopeStoreWriter<>(
136163
fileManager,
137164
siteStoreFactory,
@@ -148,6 +175,14 @@ public void execute() throws Exception {
148175
fileManager,
149176
keyAclStoreFactory,
150177
MultiScopeStoreWriter::areMapsEqual);
178+
MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> saltWriter = new MultiScopeStoreWriter<>(
179+
fileManager,
180+
saltStoreFactory,
181+
MultiScopeStoreWriter::areCollectionsEqual);
182+
MultiScopeStoreWriter<Collection<ClientSideKeypair>> clientSideKeypairWriter = new MultiScopeStoreWriter<>(
183+
fileManager,
184+
clientSideKeypairStoreFactory,
185+
MultiScopeStoreWriter::areCollectionsEqual);
151186

152187
SiteEncryptionJob siteEncryptionSyncJob = new SiteEncryptionJob(siteWriter, globalSites, globalOperators);
153188
ClientKeyEncryptionJob clientEncryptionSyncJob = new ClientKeyEncryptionJob(clientWriter, globalClients, globalOperators);
@@ -160,10 +195,15 @@ public void execute() throws Exception {
160195
encryptionKeyWriter
161196
);
162197
KeyAclEncryptionJob keyAclEncryptionSyncJob = new KeyAclEncryptionJob(keyAclWriter, globalOperators, globalKeyAcls);
198+
SaltEncryptionJob saltEncryptionJob = new SaltEncryptionJob(globalOperators, saltProvider.getSnapshots(), saltWriter);
199+
ClientSideKeypairEncryptionJob clientSideKeypairEncryptionJob = new ClientSideKeypairEncryptionJob(globalOperators, globalClientSideKeypair, clientSideKeypairWriter);
200+
163201
siteEncryptionSyncJob.execute();
164202
clientEncryptionSyncJob.execute();
165203
encryptionKeyEncryptionSyncJob.execute();
166204
keyAclEncryptionSyncJob.execute();
205+
saltEncryptionJob.execute();
206+
clientSideKeypairEncryptionJob.execute();
167207

168208
if(config.getBoolean(enableKeysetConfigProp)) {
169209
Map<Integer, Keyset> globalKeysets = keysetStoreFactory.getGlobalReader().getSnapshot().getAllKeysets();

‎src/main/java/com/uid2/admin/store/MultiScopeStoreWriter.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,13 @@ public void uploadPrivateWithEncryption(Map<Integer, T> desiredState, JsonObject
6969
}
7070
}
7171

72+
public void uploadPrivateWithEncryption(List<Integer> siteIds, T desiredState, JsonObject extraMeta) throws Exception {
73+
EncryptedStoreFactory<T> encryptedFactory = (EncryptedStoreFactory<T>) factory;
74+
for (Integer siteId : siteIds) {
75+
encryptedFactory.getEncryptedWriter(siteId,false).upload(desiredState, extraMeta);
76+
}
77+
}
78+
7279
public void uploadPublicWithEncryption(Map<Integer, T> desiredPublicState, JsonObject extraMeta) throws Exception {
7380
EncryptedStoreFactory<T> encryptedFactory = (EncryptedStoreFactory<T>) factory;
7481
for (Map.Entry<Integer, T> entry : desiredPublicState.entrySet()) {
@@ -77,6 +84,13 @@ public void uploadPublicWithEncryption(Map<Integer, T> desiredPublicState, JsonO
7784
}
7885
}
7986

87+
public void uploadPublicWithEncryption(List<Integer> siteIds, T desiredState, JsonObject extraMeta) throws Exception {
88+
EncryptedStoreFactory<T> encryptedFactory = (EncryptedStoreFactory<T>) factory;
89+
for (Integer siteId : siteIds) {
90+
encryptedFactory.getEncryptedWriter(siteId,true).upload(desiredState, extraMeta);
91+
}
92+
}
93+
8094
public static <K, V> boolean areMapsEqual(Map<K, V> a, Map<K, V> b) {
8195
return a.size() == b.size() && a.entrySet().stream().allMatch(b.entrySet()::contains);
8296
}

0 commit comments

Comments
 (0)