Skip to content

Commit 31bd637

Browse files
aulmeaulme
authored
Merge pull request #399 from IABTechLab/aul-UID2-5173-cloud-encryption-key-rotation-api
Key rotation API with retention of 5 last expired keys
2 parents 2fcf504 + d657f5c commit 31bd637

File tree

12 files changed

+449
-45
lines changed

12 files changed

+449
-45
lines changed

‎src/main/java/com/uid2/admin/Main.java‎

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,16 @@
55
import com.uid2.admin.auth.OktaAuthProvider;
66
import com.uid2.admin.auth.AuthProvider;
77
import com.uid2.admin.auth.TokenRefreshHandler;
8+
import com.uid2.admin.cloudEncryption.CloudKeyRotationStrategy;
9+
import com.uid2.admin.cloudEncryption.ExpiredKeyCountRetentionStrategy;
810
import com.uid2.admin.job.JobDispatcher;
911
import com.uid2.admin.job.jobsync.EncryptedFilesSyncJob;
1012
import com.uid2.admin.job.jobsync.PrivateSiteDataSyncJob;
1113
import com.uid2.admin.job.jobsync.keyset.ReplaceSharingTypesWithSitesJob;
1214
import com.uid2.admin.legacy.LegacyClientKeyStoreWriter;
1315
import com.uid2.admin.legacy.RotatingLegacyClientKeyProvider;
1416
import com.uid2.admin.managers.KeysetManager;
17+
import com.uid2.admin.cloudEncryption.CloudSecretGenerator;
1518
import com.uid2.admin.monitoring.DataStoreMetrics;
1619
import com.uid2.admin.managers.CloudEncryptionKeyManager;
1720
import com.uid2.admin.secret.*;
@@ -29,7 +32,6 @@
2932
import com.uid2.admin.vertx.service.*;
3033
import com.uid2.shared.Const;
3134
import com.uid2.shared.Utils;
32-
import com.uid2.shared.secret.IKeyGenerator;
3335
import com.uid2.shared.secret.KeyHasher;
3436
import com.uid2.shared.secret.SecureKeyGenerator;
3537
import com.uid2.shared.auth.EnclaveIdentifierProvider;
@@ -74,7 +76,6 @@ public class Main {
7476

7577
private final Vertx vertx;
7678
private final JsonObject config;
77-
7879
public Main(Vertx vertx, JsonObject config) {
7980
this.vertx = vertx;
8081
this.config = config;
@@ -122,7 +123,7 @@ public void run() {
122123
try {
123124
adminKeysetProvider.loadContent();
124125
} catch (CloudStorageException e) {
125-
if(e.getMessage().contains("The specified key does not exist")){
126+
if (e.getMessage().contains("The specified key does not exist")) {
126127
adminKeysetStoreWriter.upload(new HashMap<>(), null);
127128
adminKeysetProvider.loadContent();
128129
} else {
@@ -134,7 +135,7 @@ public void run() {
134135
GlobalScope keysetKeysGlobalScope = new GlobalScope(keysetKeyMetadataPath);
135136
RotatingKeysetKeyStore keysetKeysProvider = new RotatingKeysetKeyStore(cloudStorage, keysetKeysGlobalScope);
136137
KeysetKeyStoreWriter keysetKeyStoreWriter = new KeysetKeyStoreWriter(keysetKeysProvider, fileManager, versionGenerator, clock, keysetKeysGlobalScope, enableKeysets);
137-
if(enableKeysets) {
138+
if (enableKeysets) {
138139
try {
139140
keysetKeysProvider.loadContent();
140141
} catch (CloudStorageException e) {
@@ -154,7 +155,7 @@ public void run() {
154155
try {
155156
clientSideKeypairProvider.loadContent();
156157
} catch (CloudStorageException e) {
157-
if(e.getMessage().contains("The specified key does not exist")) {
158+
if (e.getMessage().contains("The specified key does not exist")) {
158159
clientSideKeypairStoreWriter.upload(new HashSet<>(), null);
159160
clientSideKeypairProvider.loadContent();
160161
} else {
@@ -163,13 +164,13 @@ public void run() {
163164
}
164165

165166
CloudPath serviceMetadataPath = new CloudPath(config.getString(Const.Config.ServiceMetadataPathProp));
166-
GlobalScope serviceGlobalScope= new GlobalScope(serviceMetadataPath);
167+
GlobalScope serviceGlobalScope = new GlobalScope(serviceMetadataPath);
167168
RotatingServiceStore serviceProvider = new RotatingServiceStore(cloudStorage, serviceGlobalScope);
168169
ServiceStoreWriter serviceStoreWriter = new ServiceStoreWriter(serviceProvider, fileManager, jsonWriter, versionGenerator, clock, serviceGlobalScope);
169170
try {
170171
serviceProvider.loadContent();
171172
} catch (CloudStorageException e) {
172-
if(e.getMessage().contains("The specified key does not exist")) {
173+
if (e.getMessage().contains("The specified key does not exist")) {
173174
serviceStoreWriter.upload(new HashSet<>(), null);
174175
serviceProvider.loadContent();
175176
} else {
@@ -178,13 +179,13 @@ public void run() {
178179
}
179180

180181
CloudPath serviceLinkMetadataPath = new CloudPath(config.getString(Const.Config.ServiceLinkMetadataPathProp));
181-
GlobalScope serviceLinkGlobalScope= new GlobalScope(serviceLinkMetadataPath);
182+
GlobalScope serviceLinkGlobalScope = new GlobalScope(serviceLinkMetadataPath);
182183
RotatingServiceLinkStore serviceLinkProvider = new RotatingServiceLinkStore(cloudStorage, serviceLinkGlobalScope);
183184
ServiceLinkStoreWriter serviceLinkStoreWriter = new ServiceLinkStoreWriter(serviceLinkProvider, fileManager, jsonWriter, versionGenerator, clock, serviceLinkGlobalScope);
184185
try {
185186
serviceLinkProvider.loadContent();
186187
} catch (CloudStorageException e) {
187-
if(e.getMessage().contains("The specified key does not exist")) {
188+
if (e.getMessage().contains("The specified key does not exist")) {
188189
serviceLinkStoreWriter.upload(new HashSet<>(), null);
189190
serviceLinkProvider.loadContent();
190191
} else {
@@ -202,8 +203,7 @@ public void run() {
202203
GlobalScope cloudEncryptionKeyGlobalScope = new GlobalScope(cloudEncryptionKeyMetadataPath);
203204
RotatingCloudEncryptionKeyProvider rotatingCloudEncryptionKeyProvider = new RotatingCloudEncryptionKeyProvider(cloudStorage, cloudEncryptionKeyGlobalScope);
204205
CloudEncryptionKeyStoreWriter cloudEncryptionKeyStoreWriter = new CloudEncryptionKeyStoreWriter(rotatingCloudEncryptionKeyProvider, fileManager, jsonWriter, versionGenerator, clock, cloudEncryptionKeyGlobalScope);
205-
IKeyGenerator keyGenerator = new SecureKeyGenerator();
206-
CloudEncryptionKeyManager cloudEncryptionKeyManager = new CloudEncryptionKeyManager(rotatingCloudEncryptionKeyProvider, cloudEncryptionKeyStoreWriter,keyGenerator);
206+
SecureKeyGenerator keyGenerator = new SecureKeyGenerator();
207207
try {
208208
rotatingCloudEncryptionKeyProvider.loadContent();
209209
} catch (CloudStorageException e) {
@@ -247,6 +247,11 @@ public void run() {
247247

248248
ClientSideKeypairService clientSideKeypairService = new ClientSideKeypairService(config, auth, writeLock, clientSideKeypairStoreWriter, clientSideKeypairProvider, siteProvider, keysetManager, keypairGenerator, clock);
249249

250+
var cloudEncryptionSecretGenerator = new CloudSecretGenerator(keyGenerator);
251+
var cloudEncryptionKeyManager = new CloudEncryptionKeyManager(rotatingCloudEncryptionKeyProvider, cloudEncryptionKeyStoreWriter, cloudEncryptionSecretGenerator);
252+
var cloudEncryptionKeyRetentionStrategy = new ExpiredKeyCountRetentionStrategy(clock, 5);
253+
var cloudEncryptionKeyRotationStrategy = new CloudKeyRotationStrategy(cloudEncryptionSecretGenerator, clock, cloudEncryptionKeyRetentionStrategy);
254+
250255
IService[] services = {
251256
new ClientKeyService(config, auth, writeLock, clientKeyStoreWriter, clientKeyProvider, siteProvider, keysetManager, keyGenerator, keyHasher),
252257
new EnclaveIdService(auth, writeLock, enclaveStoreWriter, enclaveIdProvider, clock),
@@ -263,7 +268,7 @@ public void run() {
263268
new PrivateSiteDataRefreshService(auth, jobDispatcher, writeLock, config, rotatingCloudEncryptionKeyProvider),
264269
new JobDispatcherService(auth, jobDispatcher),
265270
new SearchService(auth, clientKeyProvider, operatorKeyProvider),
266-
new CloudEncryptionKeyService(auth, rotatingCloudEncryptionKeyProvider)
271+
new CloudEncryptionKeyService(auth, rotatingCloudEncryptionKeyProvider, cloudEncryptionKeyStoreWriter, siteProvider, cloudEncryptionKeyRotationStrategy)
267272
};
268273

269274

@@ -279,7 +284,7 @@ public void run() {
279284
try {
280285
keysetProvider.loadContent();
281286
} catch (CloudStorageException e) {
282-
if(e.getMessage().contains("The specified key does not exist")){
287+
if (e.getMessage().contains("The specified key does not exist")) {
283288
keysetStoreWriter.upload(new HashMap<>(), null);
284289
keysetProvider.loadContent();
285290
} else {
@@ -305,7 +310,7 @@ public void run() {
305310
The jobs are executed after because they copy data from these files locations consumed by public and private operators.
306311
This caused an issue because the files were empty and the job started to fail so the operators got empty files.
307312
*/
308-
if(enableKeysets) {
313+
if (enableKeysets) {
309314
synchronized (writeLock) {
310315
//UID2-628 keep keys.json and keyset_keys.json in sync. This function syncs them on start up
311316
keysetProvider.loadContent();
@@ -342,7 +347,7 @@ public void run() {
342347
CompletableFuture<Boolean> privateSiteDataSyncJobFuture = jobDispatcher.executeNextJob();
343348
privateSiteDataSyncJobFuture.get();
344349

345-
EncryptedFilesSyncJob encryptedFilesSyncJob = new EncryptedFilesSyncJob(config, writeLock,rotatingCloudEncryptionKeyProvider);
350+
EncryptedFilesSyncJob encryptedFilesSyncJob = new EncryptedFilesSyncJob(config, writeLock, rotatingCloudEncryptionKeyProvider);
346351
jobDispatcher.enqueue(encryptedFilesSyncJob);
347352
CompletableFuture<Boolean> encryptedFilesSyncJobFuture = jobDispatcher.executeNextJob();
348353
encryptedFilesSyncJobFuture.get();

‎src/main/java/com/uid2/admin/cloudEncryption/CloudKeyRetentionStrategy.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package com.uid2.admin.cloudEncryption;
2+
3+
import com.uid2.shared.model.CloudEncryptionKey;
4+
5+
import java.util.Set;
6+
7+
public interface CloudKeyRetentionStrategy {
8+
Set<CloudEncryptionKey> selectKeysToRetain(Set<CloudEncryptionKey> keysForSite);
9+
}

‎src/main/java/com/uid2/admin/cloudEncryption/CloudKeyRotationStrategy.java‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
package com.uid2.admin.cloudEncryption;
2+
3+
import com.google.common.collect.Streams;
4+
import com.uid2.admin.store.Clock;
5+
import com.uid2.shared.model.CloudEncryptionKey;
6+
import com.uid2.shared.model.Site;
7+
8+
import java.util.Collection;
9+
import java.util.Map;
10+
import java.util.Set;
11+
import java.util.stream.Collectors;
12+
import java.util.stream.Stream;
13+
14+
public class CloudKeyRotationStrategy {
15+
private final CloudSecretGenerator secretGenerator;
16+
private final Clock clock;
17+
private final CloudKeyRetentionStrategy keyRetentionStrategy;
18+
19+
public CloudKeyRotationStrategy(
20+
CloudSecretGenerator secretGenerator,
21+
Clock clock,
22+
CloudKeyRetentionStrategy keyRetentionStrategy
23+
) {
24+
this.secretGenerator = secretGenerator;
25+
this.clock = clock;
26+
this.keyRetentionStrategy = keyRetentionStrategy;
27+
}
28+
29+
public Set<CloudEncryptionKey> computeDesiredKeys(
30+
Collection<CloudEncryptionKey> existingKeys,
31+
Collection<Site> sites
32+
) {
33+
var idGenerator = new KeyIdGenerator(existingKeys);
34+
Map<Integer, Set<CloudEncryptionKey>> existingKeysBySite = existingKeys
35+
.stream()
36+
.collect(Collectors.groupingBy(CloudEncryptionKey::getSiteId, Collectors.toSet()));
37+
38+
return sites
39+
.stream()
40+
.map(Site::getId)
41+
.distinct()
42+
.flatMap(siteId -> desiredKeysForSite(siteId, idGenerator, existingKeysBySite.getOrDefault(siteId, Set.of())))
43+
.collect(Collectors.toSet());
44+
}
45+
46+
private Stream<CloudEncryptionKey> desiredKeysForSite(
47+
Integer siteId,
48+
KeyIdGenerator idGenerator,
49+
Set<CloudEncryptionKey> existingKeys
50+
) {
51+
var existingKeysToRetain = keyRetentionStrategy.selectKeysToRetain(existingKeys);
52+
var newKey = makeNewKey(siteId, idGenerator);
53+
return Streams.concat(existingKeysToRetain.stream(), Stream.of(newKey));
54+
}
55+
56+
private CloudEncryptionKey makeNewKey(Integer siteId, KeyIdGenerator idGenerator) {
57+
var nowSeconds = clock.getEpochSecond();
58+
var keyId = idGenerator.nextId();
59+
var secret = secretGenerator.generate();
60+
return new CloudEncryptionKey(keyId, siteId, nowSeconds, nowSeconds, secret);
61+
}
62+
63+
private static class KeyIdGenerator {
64+
private int lastId;
65+
66+
public KeyIdGenerator(Collection<CloudEncryptionKey> existingKeys) {
67+
this.lastId = existingKeys.stream().map(CloudEncryptionKey::getId).max(Integer::compareTo).orElse(0);
68+
}
69+
70+
public int nextId() {
71+
return ++lastId;
72+
}
73+
}
74+
}

‎src/main/java/com/uid2/admin/cloudEncryption/CloudSecretGenerator.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package com.uid2.admin.cloudEncryption;
2+
3+
import com.uid2.shared.secret.SecureKeyGenerator;
4+
5+
public class CloudSecretGenerator {
6+
private final SecureKeyGenerator keyGenerator;
7+
8+
// The SecureKeyGenerator is preferable to the IKeyGenerator interface as it doesn't throw Exception
9+
public CloudSecretGenerator(SecureKeyGenerator keyGenerator) {
10+
this.keyGenerator = keyGenerator;
11+
}
12+
13+
public String generate() {
14+
//Generate a 32-byte key for AesGcm
15+
return keyGenerator.generateRandomKeyString(32);
16+
}
17+
}

‎src/main/java/com/uid2/admin/cloudEncryption/ExpiredKeyCountRetentionStrategy.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package com.uid2.admin.cloudEncryption;
2+
3+
import com.uid2.admin.store.Clock;
4+
import com.uid2.shared.model.CloudEncryptionKey;
5+
6+
import java.util.Comparator;
7+
import java.util.Set;
8+
import java.util.stream.Collectors;
9+
10+
// Only keep keys past activation time - no future keys allowed
11+
// Keep up to `expiredKeysToRetain` most recent keys
12+
// Considers all keys passed, doesn't do grouping by site (handled upstream)
13+
public class ExpiredKeyCountRetentionStrategy implements CloudKeyRetentionStrategy {
14+
private final Clock clock;
15+
private final int expiredKeysToRetain;
16+
17+
public ExpiredKeyCountRetentionStrategy(Clock clock, int expiredKeysToRetain) {
18+
this.clock = clock;
19+
this.expiredKeysToRetain = expiredKeysToRetain;
20+
}
21+
22+
@Override
23+
public Set<CloudEncryptionKey> selectKeysToRetain(Set<CloudEncryptionKey> keysForSite) {
24+
return keysForSite.stream()
25+
.filter(key -> key.getActivates() <= clock.getEpochSecond())
26+
.sorted(Comparator.comparingLong(CloudEncryptionKey::getActivates).reversed())
27+
.limit(expiredKeysToRetain)
28+
.collect(Collectors.toSet());
29+
}
30+
}

‎src/main/java/com/uid2/admin/managers/CloudEncryptionKeyManager.java‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
package com.uid2.admin.managers;
22

3+
import com.uid2.admin.cloudEncryption.CloudSecretGenerator;
34
import com.uid2.admin.store.writer.CloudEncryptionKeyStoreWriter;
45
import com.uid2.shared.auth.OperatorKey;
56
import com.uid2.shared.model.CloudEncryptionKey;
6-
import com.uid2.shared.secret.IKeyGenerator;
77
import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
88
import org.slf4j.Logger;
99
import org.slf4j.LoggerFactory;
@@ -21,16 +21,16 @@ public class CloudEncryptionKeyManager {
2121

2222
private final RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider;
2323
private final CloudEncryptionKeyStoreWriter cloudEncryptionKeyStoreWriter;
24-
private final IKeyGenerator keyGenerator;
24+
private final CloudSecretGenerator secretGenerator;
2525

2626
public CloudEncryptionKeyManager(
2727
RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider,
2828
CloudEncryptionKeyStoreWriter cloudEncryptionKeyStoreWriter,
29-
IKeyGenerator keyGenerator
29+
CloudSecretGenerator keyGenerator
3030
) {
3131
this.RotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider;
3232
this.cloudEncryptionKeyStoreWriter = cloudEncryptionKeyStoreWriter;
33-
this.keyGenerator = keyGenerator;
33+
this.secretGenerator = keyGenerator;
3434
}
3535

3636
// Ensures there are `keyCountPerSite` sites for each site corresponding of operatorKeys. If there are less - create new ones.
@@ -89,15 +89,10 @@ private static Set<Integer> uniqueSiteIdsForOperators(Collection<OperatorKey> op
8989

9090
CloudEncryptionKey generateCloudEncryptionKey(int siteId, long activates, long created) throws Exception {
9191
int newKeyId = getNextKeyId();
92-
String secret = generateSecret();
92+
String secret = secretGenerator.generate();
9393
return new CloudEncryptionKey(newKeyId, siteId, activates, created, secret);
9494
}
9595

96-
String generateSecret() throws Exception {
97-
//Generate a 32-byte key for AesGcm
98-
return keyGenerator.generateRandomKeyString(32);
99-
}
100-
10196
void addCloudEncryptionKey(CloudEncryptionKey cloudEncryptionKey) throws Exception {
10297
Map<Integer, CloudEncryptionKey> cloudEncryptionKeys = new HashMap<>(RotatingCloudEncryptionKeyProvider.getAll());
10398
cloudEncryptionKeys.put(cloudEncryptionKey.getId(), cloudEncryptionKey);
@@ -117,7 +112,7 @@ int getNextKeyId() {
117112
CloudEncryptionKey createAndAddImmediateCloudEncryptionKey(int siteId) throws Exception {
118113
int newKeyId = getNextKeyId();
119114
long created = Instant.now().getEpochSecond();
120-
CloudEncryptionKey newKey = new CloudEncryptionKey(newKeyId, siteId, created, created, generateSecret());
115+
CloudEncryptionKey newKey = new CloudEncryptionKey(newKeyId, siteId, created, created, secretGenerator.generate());
121116
addCloudEncryptionKey(newKey);
122117
return newKey;
123118
}

‎src/main/java/com/uid2/admin/vertx/Endpoints.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ public enum Endpoints {
9999
API_SITE_UPDATE("/api/site/update"),
100100

101101
CLOUD_ENCRYPTION_KEY_LIST("/api/cloud-encryption-key/list"),
102+
CLOUD_ENCRYPTION_KEY_ROTATE("/api/cloud-encryption-key/rotate"),
102103

103104
LOGIN("/login"),
104105
LOGOUT("/logout"),

0 commit comments

Comments
 (0)