Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 20 additions & 15 deletions src/main/java/com/uid2/admin/Main.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@
import com.uid2.admin.auth.OktaAuthProvider;
import com.uid2.admin.auth.AuthProvider;
import com.uid2.admin.auth.TokenRefreshHandler;
import com.uid2.admin.cloudEncryption.CloudKeyRotationStrategy;
import com.uid2.admin.cloudEncryption.ExpiredKeyCountRetentionStrategy;
import com.uid2.admin.job.JobDispatcher;
import com.uid2.admin.job.jobsync.EncryptedFilesSyncJob;
import com.uid2.admin.job.jobsync.PrivateSiteDataSyncJob;
import com.uid2.admin.job.jobsync.keyset.ReplaceSharingTypesWithSitesJob;
import com.uid2.admin.legacy.LegacyClientKeyStoreWriter;
import com.uid2.admin.legacy.RotatingLegacyClientKeyProvider;
import com.uid2.admin.managers.KeysetManager;
import com.uid2.admin.cloudEncryption.CloudSecretGenerator;
import com.uid2.admin.monitoring.DataStoreMetrics;
import com.uid2.admin.managers.CloudEncryptionKeyManager;
import com.uid2.admin.secret.*;
Expand All @@ -29,7 +32,6 @@
import com.uid2.admin.vertx.service.*;
import com.uid2.shared.Const;
import com.uid2.shared.Utils;
import com.uid2.shared.secret.IKeyGenerator;
import com.uid2.shared.secret.KeyHasher;
import com.uid2.shared.secret.SecureKeyGenerator;
import com.uid2.shared.auth.EnclaveIdentifierProvider;
Expand Down Expand Up @@ -74,7 +76,6 @@ public class Main {

private final Vertx vertx;
private final JsonObject config;

public Main(Vertx vertx, JsonObject config) {
this.vertx = vertx;
this.config = config;
Expand Down Expand Up @@ -122,7 +123,7 @@ public void run() {
try {
adminKeysetProvider.loadContent();
} catch (CloudStorageException e) {
if(e.getMessage().contains("The specified key does not exist")){
if (e.getMessage().contains("The specified key does not exist")) {
adminKeysetStoreWriter.upload(new HashMap<>(), null);
adminKeysetProvider.loadContent();
} else {
Expand All @@ -134,7 +135,7 @@ public void run() {
GlobalScope keysetKeysGlobalScope = new GlobalScope(keysetKeyMetadataPath);
RotatingKeysetKeyStore keysetKeysProvider = new RotatingKeysetKeyStore(cloudStorage, keysetKeysGlobalScope);
KeysetKeyStoreWriter keysetKeyStoreWriter = new KeysetKeyStoreWriter(keysetKeysProvider, fileManager, versionGenerator, clock, keysetKeysGlobalScope, enableKeysets);
if(enableKeysets) {
if (enableKeysets) {
try {
keysetKeysProvider.loadContent();
} catch (CloudStorageException e) {
Expand All @@ -154,7 +155,7 @@ public void run() {
try {
clientSideKeypairProvider.loadContent();
} catch (CloudStorageException e) {
if(e.getMessage().contains("The specified key does not exist")) {
if (e.getMessage().contains("The specified key does not exist")) {
clientSideKeypairStoreWriter.upload(new HashSet<>(), null);
clientSideKeypairProvider.loadContent();
} else {
Expand All @@ -163,13 +164,13 @@ public void run() {
}

CloudPath serviceMetadataPath = new CloudPath(config.getString(Const.Config.ServiceMetadataPathProp));
GlobalScope serviceGlobalScope= new GlobalScope(serviceMetadataPath);
GlobalScope serviceGlobalScope = new GlobalScope(serviceMetadataPath);
RotatingServiceStore serviceProvider = new RotatingServiceStore(cloudStorage, serviceGlobalScope);
ServiceStoreWriter serviceStoreWriter = new ServiceStoreWriter(serviceProvider, fileManager, jsonWriter, versionGenerator, clock, serviceGlobalScope);
try {
serviceProvider.loadContent();
} catch (CloudStorageException e) {
if(e.getMessage().contains("The specified key does not exist")) {
if (e.getMessage().contains("The specified key does not exist")) {
serviceStoreWriter.upload(new HashSet<>(), null);
serviceProvider.loadContent();
} else {
Expand All @@ -178,13 +179,13 @@ public void run() {
}

CloudPath serviceLinkMetadataPath = new CloudPath(config.getString(Const.Config.ServiceLinkMetadataPathProp));
GlobalScope serviceLinkGlobalScope= new GlobalScope(serviceLinkMetadataPath);
GlobalScope serviceLinkGlobalScope = new GlobalScope(serviceLinkMetadataPath);
RotatingServiceLinkStore serviceLinkProvider = new RotatingServiceLinkStore(cloudStorage, serviceLinkGlobalScope);
ServiceLinkStoreWriter serviceLinkStoreWriter = new ServiceLinkStoreWriter(serviceLinkProvider, fileManager, jsonWriter, versionGenerator, clock, serviceLinkGlobalScope);
try {
serviceLinkProvider.loadContent();
} catch (CloudStorageException e) {
if(e.getMessage().contains("The specified key does not exist")) {
if (e.getMessage().contains("The specified key does not exist")) {
serviceLinkStoreWriter.upload(new HashSet<>(), null);
serviceLinkProvider.loadContent();
} else {
Expand All @@ -202,8 +203,7 @@ public void run() {
GlobalScope cloudEncryptionKeyGlobalScope = new GlobalScope(cloudEncryptionKeyMetadataPath);
RotatingCloudEncryptionKeyProvider rotatingCloudEncryptionKeyProvider = new RotatingCloudEncryptionKeyProvider(cloudStorage, cloudEncryptionKeyGlobalScope);
CloudEncryptionKeyStoreWriter cloudEncryptionKeyStoreWriter = new CloudEncryptionKeyStoreWriter(rotatingCloudEncryptionKeyProvider, fileManager, jsonWriter, versionGenerator, clock, cloudEncryptionKeyGlobalScope);
IKeyGenerator keyGenerator = new SecureKeyGenerator();
CloudEncryptionKeyManager cloudEncryptionKeyManager = new CloudEncryptionKeyManager(rotatingCloudEncryptionKeyProvider, cloudEncryptionKeyStoreWriter,keyGenerator);
SecureKeyGenerator keyGenerator = new SecureKeyGenerator();
try {
rotatingCloudEncryptionKeyProvider.loadContent();
} catch (CloudStorageException e) {
Expand Down Expand Up @@ -247,6 +247,11 @@ public void run() {

ClientSideKeypairService clientSideKeypairService = new ClientSideKeypairService(config, auth, writeLock, clientSideKeypairStoreWriter, clientSideKeypairProvider, siteProvider, keysetManager, keypairGenerator, clock);

var cloudEncryptionSecretGenerator = new CloudSecretGenerator(keyGenerator);
var cloudEncryptionKeyManager = new CloudEncryptionKeyManager(rotatingCloudEncryptionKeyProvider, cloudEncryptionKeyStoreWriter, cloudEncryptionSecretGenerator);
var cloudEncryptionKeyRetentionStrategy = new ExpiredKeyCountRetentionStrategy(clock, 5);
var cloudEncryptionKeyRotationStrategy = new CloudKeyRotationStrategy(cloudEncryptionSecretGenerator, clock, cloudEncryptionKeyRetentionStrategy);

IService[] services = {
new ClientKeyService(config, auth, writeLock, clientKeyStoreWriter, clientKeyProvider, siteProvider, keysetManager, keyGenerator, keyHasher),
new EnclaveIdService(auth, writeLock, enclaveStoreWriter, enclaveIdProvider, clock),
Expand All @@ -263,7 +268,7 @@ public void run() {
new PrivateSiteDataRefreshService(auth, jobDispatcher, writeLock, config, rotatingCloudEncryptionKeyProvider),
new JobDispatcherService(auth, jobDispatcher),
new SearchService(auth, clientKeyProvider, operatorKeyProvider),
new CloudEncryptionKeyService(auth, rotatingCloudEncryptionKeyProvider)
new CloudEncryptionKeyService(auth, rotatingCloudEncryptionKeyProvider, cloudEncryptionKeyStoreWriter, siteProvider, cloudEncryptionKeyRotationStrategy)
};


Expand All @@ -279,7 +284,7 @@ public void run() {
try {
keysetProvider.loadContent();
} catch (CloudStorageException e) {
if(e.getMessage().contains("The specified key does not exist")){
if (e.getMessage().contains("The specified key does not exist")) {
keysetStoreWriter.upload(new HashMap<>(), null);
keysetProvider.loadContent();
} else {
Expand All @@ -305,7 +310,7 @@ public void run() {
The jobs are executed after because they copy data from these files locations consumed by public and private operators.
This caused an issue because the files were empty and the job started to fail so the operators got empty files.
*/
if(enableKeysets) {
if (enableKeysets) {
synchronized (writeLock) {
//UID2-628 keep keys.json and keyset_keys.json in sync. This function syncs them on start up
keysetProvider.loadContent();
Expand Down Expand Up @@ -342,7 +347,7 @@ public void run() {
CompletableFuture<Boolean> privateSiteDataSyncJobFuture = jobDispatcher.executeNextJob();
privateSiteDataSyncJobFuture.get();

EncryptedFilesSyncJob encryptedFilesSyncJob = new EncryptedFilesSyncJob(config, writeLock,rotatingCloudEncryptionKeyProvider);
EncryptedFilesSyncJob encryptedFilesSyncJob = new EncryptedFilesSyncJob(config, writeLock, rotatingCloudEncryptionKeyProvider);
jobDispatcher.enqueue(encryptedFilesSyncJob);
CompletableFuture<Boolean> encryptedFilesSyncJobFuture = jobDispatcher.executeNextJob();
encryptedFilesSyncJobFuture.get();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package com.uid2.admin.cloudEncryption;

import com.uid2.shared.model.CloudEncryptionKey;

import java.util.Set;

public interface CloudKeyRetentionStrategy {
Set<CloudEncryptionKey> selectKeysToRetain(Set<CloudEncryptionKey> keysForSite);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
package com.uid2.admin.cloudEncryption;

import com.google.common.collect.Streams;
import com.uid2.admin.store.Clock;
import com.uid2.shared.model.CloudEncryptionKey;
import com.uid2.shared.model.Site;

import java.util.Collection;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;

public class CloudKeyRotationStrategy {
private final CloudSecretGenerator secretGenerator;
private final Clock clock;
private final CloudKeyRetentionStrategy keyRetentionStrategy;

public CloudKeyRotationStrategy(
CloudSecretGenerator secretGenerator,
Clock clock,
CloudKeyRetentionStrategy keyRetentionStrategy
) {
this.secretGenerator = secretGenerator;
this.clock = clock;
this.keyRetentionStrategy = keyRetentionStrategy;
}

public Set<CloudEncryptionKey> computeDesiredKeys(
Collection<CloudEncryptionKey> existingKeys,
Collection<Site> sites
) {
var idGenerator = new KeyIdGenerator(existingKeys);
Map<Integer, Set<CloudEncryptionKey>> existingKeysBySite = existingKeys
.stream()
.collect(Collectors.groupingBy(CloudEncryptionKey::getSiteId, Collectors.toSet()));

return sites
.stream()
.map(Site::getId)
.distinct()
.flatMap(siteId -> desiredKeysForSite(siteId, idGenerator, existingKeysBySite.getOrDefault(siteId, Set.of())))
.collect(Collectors.toSet());
}

private Stream<CloudEncryptionKey> desiredKeysForSite(
Integer siteId,
KeyIdGenerator idGenerator,
Set<CloudEncryptionKey> existingKeys
) {
var existingKeysToRetain = keyRetentionStrategy.selectKeysToRetain(existingKeys);
var newKey = makeNewKey(siteId, idGenerator);
return Streams.concat(existingKeysToRetain.stream(), Stream.of(newKey));
}

private CloudEncryptionKey makeNewKey(Integer siteId, KeyIdGenerator idGenerator) {
var nowSeconds = clock.getEpochSecond();
var keyId = idGenerator.nextId();
var secret = secretGenerator.generate();
return new CloudEncryptionKey(keyId, siteId, nowSeconds, nowSeconds, secret);
}

private static class KeyIdGenerator {
private int lastId;

public KeyIdGenerator(Collection<CloudEncryptionKey> existingKeys) {
this.lastId = existingKeys.stream().map(CloudEncryptionKey::getId).max(Integer::compareTo).orElse(0);
}

public int nextId() {
return ++lastId;
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package com.uid2.admin.cloudEncryption;

import com.uid2.shared.secret.SecureKeyGenerator;

public class CloudSecretGenerator {
private final SecureKeyGenerator keyGenerator;

// The SecureKeyGenerator is preferable to the IKeyGenerator interface as it doesn't throw Exception
public CloudSecretGenerator(SecureKeyGenerator keyGenerator) {
this.keyGenerator = keyGenerator;
}

public String generate() {
//Generate a 32-byte key for AesGcm
return keyGenerator.generateRandomKeyString(32);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package com.uid2.admin.cloudEncryption;

import com.uid2.admin.store.Clock;
import com.uid2.shared.model.CloudEncryptionKey;

import java.util.Comparator;
import java.util.Set;
import java.util.stream.Collectors;

// Only keep keys past activation time - no future keys allowed
// Keep up to `expiredKeysToRetain` most recent keys
// Considers all keys passed, doesn't do grouping by site (handled upstream)
public class ExpiredKeyCountRetentionStrategy implements CloudKeyRetentionStrategy {
private final Clock clock;
private final int expiredKeysToRetain;

public ExpiredKeyCountRetentionStrategy(Clock clock, int expiredKeysToRetain) {
this.clock = clock;
this.expiredKeysToRetain = expiredKeysToRetain;
}

@Override
public Set<CloudEncryptionKey> selectKeysToRetain(Set<CloudEncryptionKey> keysForSite) {
return keysForSite.stream()
.filter(key -> key.getActivates() <= clock.getEpochSecond())
.sorted(Comparator.comparingLong(CloudEncryptionKey::getActivates).reversed())
.limit(expiredKeysToRetain)
.collect(Collectors.toSet());
}
}
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package com.uid2.admin.managers;

import com.uid2.admin.cloudEncryption.CloudSecretGenerator;
import com.uid2.admin.store.writer.CloudEncryptionKeyStoreWriter;
import com.uid2.shared.auth.OperatorKey;
import com.uid2.shared.model.CloudEncryptionKey;
import com.uid2.shared.secret.IKeyGenerator;
import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -21,16 +21,16 @@ public class CloudEncryptionKeyManager {

private final RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider;
private final CloudEncryptionKeyStoreWriter cloudEncryptionKeyStoreWriter;
private final IKeyGenerator keyGenerator;
private final CloudSecretGenerator secretGenerator;

public CloudEncryptionKeyManager(
RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider,
CloudEncryptionKeyStoreWriter cloudEncryptionKeyStoreWriter,
IKeyGenerator keyGenerator
CloudSecretGenerator keyGenerator
) {
this.RotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider;
this.cloudEncryptionKeyStoreWriter = cloudEncryptionKeyStoreWriter;
this.keyGenerator = keyGenerator;
this.secretGenerator = keyGenerator;
}

// Ensures there are `keyCountPerSite` sites for each site corresponding of operatorKeys. If there are less - create new ones.
Expand Down Expand Up @@ -89,15 +89,10 @@ private static Set<Integer> uniqueSiteIdsForOperators(Collection<OperatorKey> op

CloudEncryptionKey generateCloudEncryptionKey(int siteId, long activates, long created) throws Exception {
int newKeyId = getNextKeyId();
String secret = generateSecret();
String secret = secretGenerator.generate();
return new CloudEncryptionKey(newKeyId, siteId, activates, created, secret);
}

String generateSecret() throws Exception {
//Generate a 32-byte key for AesGcm
return keyGenerator.generateRandomKeyString(32);
}

void addCloudEncryptionKey(CloudEncryptionKey cloudEncryptionKey) throws Exception {
Map<Integer, CloudEncryptionKey> cloudEncryptionKeys = new HashMap<>(RotatingCloudEncryptionKeyProvider.getAll());
cloudEncryptionKeys.put(cloudEncryptionKey.getId(), cloudEncryptionKey);
Expand All @@ -117,7 +112,7 @@ int getNextKeyId() {
CloudEncryptionKey createAndAddImmediateCloudEncryptionKey(int siteId) throws Exception {
int newKeyId = getNextKeyId();
long created = Instant.now().getEpochSecond();
CloudEncryptionKey newKey = new CloudEncryptionKey(newKeyId, siteId, created, created, generateSecret());
CloudEncryptionKey newKey = new CloudEncryptionKey(newKeyId, siteId, created, created, secretGenerator.generate());
addCloudEncryptionKey(newKey);
return newKey;
}
Expand Down
1 change: 1 addition & 0 deletions src/main/java/com/uid2/admin/vertx/Endpoints.java
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ public enum Endpoints {
API_SITE_UPDATE("/api/site/update"),

CLOUD_ENCRYPTION_KEY_LIST("/api/cloud-encryption-key/list"),
CLOUD_ENCRYPTION_KEY_ROTATE("/api/cloud-encryption-key/rotate"),

LOGIN("/login"),
LOGOUT("/logout"),
Expand Down
Loading