From 2f59930f470b1dd93bd23c9fb0d0c9857d192429 Mon Sep 17 00:00:00 2001 From: way zheng Date: Wed, 1 Oct 2025 17:14:42 -0700 Subject: [PATCH 01/28] fail fast keysetkey out of sync. Currently still using fixed interval --- conf/default-config.json | 1 + src/main/java/com/uid2/operator/Main.java | 5 +-- .../com/uid2/operator/model/KeyManager.java | 7 ++++- .../vertx/OperatorShutdownHandler.java | 31 +++++++++++++++++-- 4 files changed, 39 insertions(+), 5 deletions(-) diff --git a/conf/default-config.json b/conf/default-config.json index c5de0d87b..688f23db5 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -37,6 +37,7 @@ "optout_inmem_cache": false, "enclave_platform": null, "failure_shutdown_wait_hours": 120, + "keyset_key_shutdown_hours": 2, "sharing_token_expiry_seconds": 2592000, "operator_type": "public", "enable_remote_config": true, diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index c4fe54c7e..0318680e7 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -114,7 +114,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.clientSideTokenGenerate = config.getBoolean(Const.Config.EnableClientSideTokenGenerate, false); this.validateServiceLinks = config.getBoolean(Const.Config.ValidateServiceLinks, false); this.encryptedCloudFilesEnabled = config.getBoolean(Const.Config.EncryptedFiles, false); - this.shutdownHandler = new OperatorShutdownHandler(Duration.ofHours(12), Duration.ofHours(config.getInteger(Const.Config.SaltsExpiredShutdownHours, 12)), Clock.systemUTC(), new ShutdownService()); + this.shutdownHandler = new OperatorShutdownHandler(Duration.ofHours(12), Duration.ofHours(config.getInteger(Const.Config.SaltsExpiredShutdownHours, 12)), Duration.ofHours(config.getInteger(Const.Config.KeysetKeyShutdownHours, 2)), Clock.systemUTC(), new ShutdownService()); this.uidInstanceIdProvider = new UidInstanceIdProvider(config); String coreAttestUrl = this.config.getString(Const.Config.CoreAttestUrlProp); @@ -243,7 +243,8 @@ public Main(Vertx vertx, JsonObject config) throws Exception { } private KeyManager getKeyManager() { - return new KeyManager(this.keysetKeyStore, this.keysetProvider); + return new KeyManager(this.keysetKeyStore, this.keysetProvider, + hasKeys -> shutdownHandler.handleKeysetKeyRefreshResponse(hasKeys)); } public static void recordStartupComplete() { diff --git a/src/main/java/com/uid2/operator/model/KeyManager.java b/src/main/java/com/uid2/operator/model/KeyManager.java index 19bae8d07..aa7842ab7 100644 --- a/src/main/java/com/uid2/operator/model/KeyManager.java +++ b/src/main/java/com/uid2/operator/model/KeyManager.java @@ -13,16 +13,19 @@ import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import java.util.stream.Collectors; public class KeyManager { private static final Logger LOGGER = LoggerFactory.getLogger(UIDOperatorVerticle.class); private final IKeysetKeyStore keysetKeyStore; private final RotatingKeysetProvider keysetProvider; + private final Consumer keyAvailabilityHandler; - public KeyManager(IKeysetKeyStore keysetKeyStore, RotatingKeysetProvider keysetProvider) { + public KeyManager(IKeysetKeyStore keysetKeyStore, RotatingKeysetProvider keysetProvider, Consumer keyAvailabilityHandler) { this.keysetKeyStore = keysetKeyStore; this.keysetProvider = keysetProvider; + this.keyAvailabilityHandler = keyAvailabilityHandler; } public KeyManagerSnapshot getKeyManagerSnapshot(int siteId) { @@ -107,6 +110,7 @@ public KeysetKey getMasterKey() { public KeysetKey getMasterKey(Instant asOf) { KeysetKey key = this.keysetKeyStore.getSnapshot().getActiveKey(Const.Data.MasterKeysetId, asOf); if (key == null) { + if (keyAvailabilityHandler != null) keyAvailabilityHandler.accept(false); throw new NoActiveKeyException(String.format("Cannot get a master key with keyset ID %d.", Const.Data.MasterKeysetId)); } return key; @@ -119,6 +123,7 @@ public KeysetKey getRefreshKey() { public KeysetKey getRefreshKey(Instant asOf) { KeysetKey key = this.keysetKeyStore.getSnapshot().getActiveKey(Const.Data.RefreshKeysetId, asOf); if (key == null) { + if (keyAvailabilityHandler != null) keyAvailabilityHandler.accept(false); throw new NoActiveKeyException(String.format("Cannot get a refresh key with keyset ID %d.", Const.Data.RefreshKeysetId)); } return key; diff --git a/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java b/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java index 84075fb03..154ef9afa 100644 --- a/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java +++ b/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java @@ -2,7 +2,6 @@ import com.uid2.operator.service.ShutdownService; import com.uid2.shared.attest.AttestationResponseCode; -import lombok.extern.java.Log; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.utils.Pair; @@ -16,17 +15,22 @@ public class OperatorShutdownHandler { private static final Logger LOGGER = LoggerFactory.getLogger(OperatorShutdownHandler.class); private static final int SALT_FAILURE_LOG_INTERVAL_MINUTES = 10; + private static final int KEYSET_KEY_FAILURE_LOG_INTERVAL_MINUTES = 10; private final Duration attestShutdownWaitTime; private final Duration saltShutdownWaitTime; + private final Duration keysetKeyShutdownWaitTime; private final AtomicReference attestFailureStartTime = new AtomicReference<>(null); private final AtomicReference saltFailureStartTime = new AtomicReference<>(null); + private final AtomicReference keysetKeyFailureStartTime = new AtomicReference<>(null); private final AtomicReference lastSaltFailureLogTime = new AtomicReference<>(null); + private final AtomicReference lastKeysetKeyFailureLogTime = new AtomicReference<>(null); private final Clock clock; private final ShutdownService shutdownService; - public OperatorShutdownHandler(Duration attestShutdownWaitTime, Duration saltShutdownWaitTime, Clock clock, ShutdownService shutdownService) { + public OperatorShutdownHandler(Duration attestShutdownWaitTime, Duration saltShutdownWaitTime, Duration keysetKeyShutdownWaitTime, Clock clock, ShutdownService shutdownService) { this.attestShutdownWaitTime = attestShutdownWaitTime; this.saltShutdownWaitTime = saltShutdownWaitTime; + this.keysetKeyShutdownWaitTime = keysetKeyShutdownWaitTime; this.clock = clock; this.shutdownService = shutdownService; } @@ -54,6 +58,29 @@ public void logSaltFailureAtInterval() { } } + public void handleKeysetKeyRefreshResponse(Boolean success) { + if (success) { + keysetKeyFailureStartTime.set(null); + } else { + logKeysetKeyFailureAtInterval(); + Instant t = keysetKeyFailureStartTime.get(); + if (t == null) { + keysetKeyFailureStartTime.set(clock.instant()); + } else if (Duration.between(t, clock.instant()).compareTo(this.keysetKeyShutdownWaitTime) > 0) { + LOGGER.error("keyset keys have been failing to sync for too long. shutting down operator"); + this.shutdownService.Shutdown(1); + } + } + } + + public void logKeysetKeyFailureAtInterval() { + Instant t = lastKeysetKeyFailureLogTime.get(); + if (t == null || clock.instant().isAfter(t.plus(KEYSET_KEY_FAILURE_LOG_INTERVAL_MINUTES, ChronoUnit.MINUTES))) { + LOGGER.error("keyset keys sync failing"); + lastKeysetKeyFailureLogTime.set(Instant.now()); + } + } + public void handleAttestResponse(Pair response) { if (response.left() == AttestationResponseCode.AttestationFailure) { LOGGER.error("core attestation failed with AttestationFailure, shutting down operator, core response: {}", response.right()); From 8e3d1000396f585a905b03481a716813c3322a57 Mon Sep 17 00:00:00 2001 From: way zheng Date: Wed, 1 Oct 2025 17:24:04 -0700 Subject: [PATCH 02/28] hard coded for standslone test --- conf/default-config.json | 1 - src/main/java/com/uid2/operator/Main.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/conf/default-config.json b/conf/default-config.json index 688f23db5..c5de0d87b 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -37,7 +37,6 @@ "optout_inmem_cache": false, "enclave_platform": null, "failure_shutdown_wait_hours": 120, - "keyset_key_shutdown_hours": 2, "sharing_token_expiry_seconds": 2592000, "operator_type": "public", "enable_remote_config": true, diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 0318680e7..ab68bfc6c 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -114,7 +114,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.clientSideTokenGenerate = config.getBoolean(Const.Config.EnableClientSideTokenGenerate, false); this.validateServiceLinks = config.getBoolean(Const.Config.ValidateServiceLinks, false); this.encryptedCloudFilesEnabled = config.getBoolean(Const.Config.EncryptedFiles, false); - this.shutdownHandler = new OperatorShutdownHandler(Duration.ofHours(12), Duration.ofHours(config.getInteger(Const.Config.SaltsExpiredShutdownHours, 12)), Duration.ofHours(config.getInteger(Const.Config.KeysetKeyShutdownHours, 2)), Clock.systemUTC(), new ShutdownService()); + this.shutdownHandler = new OperatorShutdownHandler(Duration.ofHours(12), Duration.ofHours(config.getInteger(Const.Config.SaltsExpiredShutdownHours, 12)), Duration.ofHours(2), Clock.systemUTC(), new ShutdownService()); this.uidInstanceIdProvider = new UidInstanceIdProvider(config); String coreAttestUrl = this.config.getString(Const.Config.CoreAttestUrlProp); From f584169f31c3a8281a8634329cbc6baa26c91f16 Mon Sep 17 00:00:00 2001 From: way zheng Date: Wed, 1 Oct 2025 17:28:45 -0700 Subject: [PATCH 03/28] tests need backwards constructors --- src/main/java/com/uid2/operator/model/KeyManager.java | 5 +++++ .../com/uid2/operator/vertx/OperatorShutdownHandler.java | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/main/java/com/uid2/operator/model/KeyManager.java b/src/main/java/com/uid2/operator/model/KeyManager.java index aa7842ab7..ccb98f094 100644 --- a/src/main/java/com/uid2/operator/model/KeyManager.java +++ b/src/main/java/com/uid2/operator/model/KeyManager.java @@ -22,6 +22,11 @@ public class KeyManager { private final RotatingKeysetProvider keysetProvider; private final Consumer keyAvailabilityHandler; + // Backward compatible constructor for tests + public KeyManager(IKeysetKeyStore keysetKeyStore, RotatingKeysetProvider keysetProvider) { + this(keysetKeyStore, keysetProvider, null); + } + public KeyManager(IKeysetKeyStore keysetKeyStore, RotatingKeysetProvider keysetProvider, Consumer keyAvailabilityHandler) { this.keysetKeyStore = keysetKeyStore; this.keysetProvider = keysetProvider; diff --git a/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java b/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java index 154ef9afa..7597879d0 100644 --- a/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java +++ b/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java @@ -27,6 +27,11 @@ public class OperatorShutdownHandler { private final Clock clock; private final ShutdownService shutdownService; + // Backward compatible constructor for tests (default 2 hours for keyset key shutdown) + public OperatorShutdownHandler(Duration attestShutdownWaitTime, Duration saltShutdownWaitTime, Clock clock, ShutdownService shutdownService) { + this(attestShutdownWaitTime, saltShutdownWaitTime, Duration.ofHours(2), clock, shutdownService); + } + public OperatorShutdownHandler(Duration attestShutdownWaitTime, Duration saltShutdownWaitTime, Duration keysetKeyShutdownWaitTime, Clock clock, ShutdownService shutdownService) { this.attestShutdownWaitTime = attestShutdownWaitTime; this.saltShutdownWaitTime = saltShutdownWaitTime; From 4c005bf1fc27e9d555cf92cc2da028f19ef1a434 Mon Sep 17 00:00:00 2001 From: way zheng Date: Wed, 1 Oct 2025 17:33:51 -0700 Subject: [PATCH 04/28] unit tests added --- conf/default-config.json | 1 + .../operator/OperatorShutdownHandlerTest.java | 96 +++++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/conf/default-config.json b/conf/default-config.json index c5de0d87b..688f23db5 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -37,6 +37,7 @@ "optout_inmem_cache": false, "enclave_platform": null, "failure_shutdown_wait_hours": 120, + "keyset_key_shutdown_hours": 2, "sharing_token_expiry_seconds": 2592000, "operator_type": "public", "enable_remote_config": true, diff --git a/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java b/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java index 10a00b813..99d41f02f 100644 --- a/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java +++ b/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java @@ -166,4 +166,100 @@ void saltsLogErrorAtInterval(VertxTestContext testContext) { testContext.completeNow(); } + + // ===== Keyset Key Tests ===== + + @Test + void shutdownOnKeysetKeyFailedTooLong(VertxTestContext testContext) { + ListAppender logWatcher = new ListAppender<>(); + logWatcher.start(); + ((Logger) LoggerFactory.getLogger(OperatorShutdownHandler.class)).addAppender(logWatcher); + + // First failure - starts the timer + this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(false); + Assertions.assertTrue(logWatcher.list.get(0).getFormattedMessage().contains("keyset keys sync failing")); + + // Advance time beyond 2 hour threshold + when(clock.instant()).thenAnswer(i -> Instant.now().plus(2, ChronoUnit.HOURS).plusSeconds(60)); + + // Second failure after timeout should trigger shutdown + Assertions.assertThrows(RuntimeException.class, () -> { + this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(false); + }); + + Assertions.assertAll("Keyset Key Failure Log Messages", + () -> verify(shutdownService).Shutdown(1), + () -> Assertions.assertTrue(logWatcher.list.get(1).getFormattedMessage().contains("keyset keys sync failing")), + () -> Assertions.assertTrue(logWatcher.list.get(2).getFormattedMessage().contains("keyset keys have been in failed state for too long. shutting down operator")), + () -> Assertions.assertEquals(3, logWatcher.list.size())); + + testContext.completeNow(); + } + + @Test + void keysetKeyRecoverOnSuccess(VertxTestContext testContext) { + ListAppender logWatcher = new ListAppender<>(); + logWatcher.start(); + ((Logger) LoggerFactory.getLogger(OperatorShutdownHandler.class)).addAppender(logWatcher); + + // Start with failure + this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(false); + Assertions.assertTrue(logWatcher.list.get(0).getFormattedMessage().contains("keyset keys sync failing")); + + // Advance time but then recover + when(clock.instant()).thenAnswer(i -> Instant.now().plus(1, ChronoUnit.HOURS)); + this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(true); + + // Advance time beyond original threshold - should not shutdown because we recovered + when(clock.instant()).thenAnswer(i -> Instant.now().plus(3, ChronoUnit.HOURS)); + assertDoesNotThrow(() -> { + this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(false); + }); + + verify(shutdownService, never()).Shutdown(anyInt()); + testContext.completeNow(); + } + + @Test + void keysetKeyLogErrorAtInterval(VertxTestContext testContext) { + ListAppender logWatcher = new ListAppender<>(); + logWatcher.start(); + ((Logger) LoggerFactory.getLogger(OperatorShutdownHandler.class)).addAppender(logWatcher); + + // First failure logs immediately + this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(false); + Assertions.assertTrue(logWatcher.list.get(0).getFormattedMessage().contains("keyset keys sync failing")); + + // After 9 minutes, should not log again (interval is 10 minutes) + when(clock.instant()).thenAnswer(i -> Instant.now().plus(9, ChronoUnit.MINUTES)); + this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(false); + Assertions.assertEquals(1, logWatcher.list.size()); + + // After 11 minutes, should log again + when(clock.instant()).thenAnswer(i -> Instant.now().plus(11, ChronoUnit.MINUTES)); + this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(false); + Assertions.assertTrue(logWatcher.list.get(1).getFormattedMessage().contains("keyset keys sync failing")); + Assertions.assertEquals(2, logWatcher.list.size()); + + testContext.completeNow(); + } + + @Test + void keysetKeyNoShutdownWhenAlwaysSuccessful(VertxTestContext testContext) { + ListAppender logWatcher = new ListAppender<>(); + logWatcher.start(); + ((Logger) LoggerFactory.getLogger(OperatorShutdownHandler.class)).addAppender(logWatcher); + + // Only successful refreshes + this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(true); + when(clock.instant()).thenAnswer(i -> Instant.now().plus(1, ChronoUnit.HOURS)); + this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(true); + when(clock.instant()).thenAnswer(i -> Instant.now().plus(3, ChronoUnit.HOURS)); + this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(true); + + // No logs, no shutdown + Assertions.assertEquals(0, logWatcher.list.size()); + verify(shutdownService, never()).Shutdown(anyInt()); + testContext.completeNow(); + } } From 5d52808140ef07127c9397b16ac1430b01069b0f Mon Sep 17 00:00:00 2001 From: way zheng Date: Wed, 1 Oct 2025 23:22:41 -0700 Subject: [PATCH 05/28] unit test typo fix --- .../java/com/uid2/operator/OperatorShutdownHandlerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java b/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java index 99d41f02f..6d1383b36 100644 --- a/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java +++ b/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java @@ -190,7 +190,7 @@ void shutdownOnKeysetKeyFailedTooLong(VertxTestContext testContext) { Assertions.assertAll("Keyset Key Failure Log Messages", () -> verify(shutdownService).Shutdown(1), () -> Assertions.assertTrue(logWatcher.list.get(1).getFormattedMessage().contains("keyset keys sync failing")), - () -> Assertions.assertTrue(logWatcher.list.get(2).getFormattedMessage().contains("keyset keys have been in failed state for too long. shutting down operator")), + () -> Assertions.assertTrue(logWatcher.list.get(2).getFormattedMessage().contains("keyset keys have been failing to sync for too long. shutting down operator")), () -> Assertions.assertEquals(3, logWatcher.list.size())); testContext.completeNow(); From 6b188d0b1731d8b9b693fd48b36925ebbc796187 Mon Sep 17 00:00:00 2001 From: way zheng Date: Thu, 2 Oct 2025 10:52:06 -0700 Subject: [PATCH 06/28] unit test typo fix --- .../uid2/operator/OperatorShutdownHandlerTest.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java b/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java index 6d1383b36..4350de394 100644 --- a/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java +++ b/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java @@ -167,22 +167,17 @@ void saltsLogErrorAtInterval(VertxTestContext testContext) { testContext.completeNow(); } - // ===== Keyset Key Tests ===== - @Test void shutdownOnKeysetKeyFailedTooLong(VertxTestContext testContext) { ListAppender logWatcher = new ListAppender<>(); logWatcher.start(); ((Logger) LoggerFactory.getLogger(OperatorShutdownHandler.class)).addAppender(logWatcher); - // First failure - starts the timer this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(false); Assertions.assertTrue(logWatcher.list.get(0).getFormattedMessage().contains("keyset keys sync failing")); - // Advance time beyond 2 hour threshold when(clock.instant()).thenAnswer(i -> Instant.now().plus(2, ChronoUnit.HOURS).plusSeconds(60)); - // Second failure after timeout should trigger shutdown Assertions.assertThrows(RuntimeException.class, () -> { this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(false); }); @@ -202,15 +197,12 @@ void keysetKeyRecoverOnSuccess(VertxTestContext testContext) { logWatcher.start(); ((Logger) LoggerFactory.getLogger(OperatorShutdownHandler.class)).addAppender(logWatcher); - // Start with failure this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(false); Assertions.assertTrue(logWatcher.list.get(0).getFormattedMessage().contains("keyset keys sync failing")); - // Advance time but then recover when(clock.instant()).thenAnswer(i -> Instant.now().plus(1, ChronoUnit.HOURS)); this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(true); - // Advance time beyond original threshold - should not shutdown because we recovered when(clock.instant()).thenAnswer(i -> Instant.now().plus(3, ChronoUnit.HOURS)); assertDoesNotThrow(() -> { this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(false); @@ -226,16 +218,13 @@ void keysetKeyLogErrorAtInterval(VertxTestContext testContext) { logWatcher.start(); ((Logger) LoggerFactory.getLogger(OperatorShutdownHandler.class)).addAppender(logWatcher); - // First failure logs immediately this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(false); Assertions.assertTrue(logWatcher.list.get(0).getFormattedMessage().contains("keyset keys sync failing")); - // After 9 minutes, should not log again (interval is 10 minutes) when(clock.instant()).thenAnswer(i -> Instant.now().plus(9, ChronoUnit.MINUTES)); this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(false); Assertions.assertEquals(1, logWatcher.list.size()); - // After 11 minutes, should log again when(clock.instant()).thenAnswer(i -> Instant.now().plus(11, ChronoUnit.MINUTES)); this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(false); Assertions.assertTrue(logWatcher.list.get(1).getFormattedMessage().contains("keyset keys sync failing")); @@ -250,14 +239,12 @@ void keysetKeyNoShutdownWhenAlwaysSuccessful(VertxTestContext testContext) { logWatcher.start(); ((Logger) LoggerFactory.getLogger(OperatorShutdownHandler.class)).addAppender(logWatcher); - // Only successful refreshes this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(true); when(clock.instant()).thenAnswer(i -> Instant.now().plus(1, ChronoUnit.HOURS)); this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(true); when(clock.instant()).thenAnswer(i -> Instant.now().plus(3, ChronoUnit.HOURS)); this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(true); - // No logs, no shutdown Assertions.assertEquals(0, logWatcher.list.size()); verify(shutdownService, never()).Shutdown(anyInt()); testContext.completeNow(); From f4611939391b4eb5356ceab4ec72f4c0974cb9e2 Mon Sep 17 00:00:00 2001 From: way zheng Date: Thu, 2 Oct 2025 11:04:20 -0700 Subject: [PATCH 07/28] beautify --- src/main/java/com/uid2/operator/Main.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index ab68bfc6c..043412dd3 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -243,8 +243,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception { } private KeyManager getKeyManager() { - return new KeyManager(this.keysetKeyStore, this.keysetProvider, - hasKeys -> shutdownHandler.handleKeysetKeyRefreshResponse(hasKeys)); + return new KeyManager(this.keysetKeyStore, this.keysetProvider, this.shutdownHandler::handleKeysetKeyRefreshResponse); } public static void recordStartupComplete() { From 6d18db5bc1d61812d180773116226c1feafdfd3f Mon Sep 17 00:00:00 2001 From: way zheng Date: Thu, 2 Oct 2025 12:53:33 -0700 Subject: [PATCH 08/28] beautify --- src/main/java/com/uid2/operator/model/KeyManager.java | 1 - .../java/com/uid2/operator/vertx/OperatorShutdownHandler.java | 1 - 2 files changed, 2 deletions(-) diff --git a/src/main/java/com/uid2/operator/model/KeyManager.java b/src/main/java/com/uid2/operator/model/KeyManager.java index ccb98f094..e939656bc 100644 --- a/src/main/java/com/uid2/operator/model/KeyManager.java +++ b/src/main/java/com/uid2/operator/model/KeyManager.java @@ -22,7 +22,6 @@ public class KeyManager { private final RotatingKeysetProvider keysetProvider; private final Consumer keyAvailabilityHandler; - // Backward compatible constructor for tests public KeyManager(IKeysetKeyStore keysetKeyStore, RotatingKeysetProvider keysetProvider) { this(keysetKeyStore, keysetProvider, null); } diff --git a/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java b/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java index 7597879d0..928064192 100644 --- a/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java +++ b/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java @@ -27,7 +27,6 @@ public class OperatorShutdownHandler { private final Clock clock; private final ShutdownService shutdownService; - // Backward compatible constructor for tests (default 2 hours for keyset key shutdown) public OperatorShutdownHandler(Duration attestShutdownWaitTime, Duration saltShutdownWaitTime, Clock clock, ShutdownService shutdownService) { this(attestShutdownWaitTime, saltShutdownWaitTime, Duration.ofHours(2), clock, shutdownService); } From 063a9b5facd069e431aff680b31fa1b49a74aae1 Mon Sep 17 00:00:00 2001 From: way zheng Date: Thu, 2 Oct 2025 12:54:12 -0700 Subject: [PATCH 09/28] beautify --- .../java/com/uid2/operator/vertx/OperatorShutdownHandler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java b/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java index 928064192..f9961d4c7 100644 --- a/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java +++ b/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java @@ -2,6 +2,7 @@ import com.uid2.operator.service.ShutdownService; import com.uid2.shared.attest.AttestationResponseCode; +import lombok.extern.java.Log; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.utils.Pair; From 2012ab2050900dd7805fcb7977d10c7429c75c6d Mon Sep 17 00:00:00 2001 From: way zheng Date: Thu, 2 Oct 2025 13:01:58 -0700 Subject: [PATCH 10/28] notify success --- src/main/java/com/uid2/operator/model/KeyManager.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/uid2/operator/model/KeyManager.java b/src/main/java/com/uid2/operator/model/KeyManager.java index e939656bc..95e73b61d 100644 --- a/src/main/java/com/uid2/operator/model/KeyManager.java +++ b/src/main/java/com/uid2/operator/model/KeyManager.java @@ -117,6 +117,7 @@ public KeysetKey getMasterKey(Instant asOf) { if (keyAvailabilityHandler != null) keyAvailabilityHandler.accept(false); throw new NoActiveKeyException(String.format("Cannot get a master key with keyset ID %d.", Const.Data.MasterKeysetId)); } + if (keyAvailabilityHandler != null) keyAvailabilityHandler.accept(true); return key; } @@ -130,6 +131,7 @@ public KeysetKey getRefreshKey(Instant asOf) { if (keyAvailabilityHandler != null) keyAvailabilityHandler.accept(false); throw new NoActiveKeyException(String.format("Cannot get a refresh key with keyset ID %d.", Const.Data.RefreshKeysetId)); } + if (keyAvailabilityHandler != null) keyAvailabilityHandler.accept(true); return key; } From b64e24bb84e6644ac781ccbb0e240093aa438a5f Mon Sep 17 00:00:00 2001 From: way zheng Date: Thu, 2 Oct 2025 13:47:19 -0700 Subject: [PATCH 11/28] 5 minutes for testing --- src/main/java/com/uid2/operator/Main.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 043412dd3..ad887c436 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -114,7 +114,8 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.clientSideTokenGenerate = config.getBoolean(Const.Config.EnableClientSideTokenGenerate, false); this.validateServiceLinks = config.getBoolean(Const.Config.ValidateServiceLinks, false); this.encryptedCloudFilesEnabled = config.getBoolean(Const.Config.EncryptedFiles, false); - this.shutdownHandler = new OperatorShutdownHandler(Duration.ofHours(12), Duration.ofHours(config.getInteger(Const.Config.SaltsExpiredShutdownHours, 12)), Duration.ofHours(2), Clock.systemUTC(), new ShutdownService()); + //todo: change to a config values after testing + this.shutdownHandler = new OperatorShutdownHandler(Duration.ofHours(12), Duration.ofHours(config.getInteger(Const.Config.SaltsExpiredShutdownHours, 12)), Duration.ofMinutes(5), Clock.systemUTC(), new ShutdownService()); this.uidInstanceIdProvider = new UidInstanceIdProvider(config); String coreAttestUrl = this.config.getString(Const.Config.CoreAttestUrlProp); From 050afa8c20897ce4a12612c748f7cbb5560aae71 Mon Sep 17 00:00:00 2001 From: way zheng Date: Thu, 2 Oct 2025 13:56:50 -0700 Subject: [PATCH 12/28] testing, disable the refresh key call, see if we can reproduce the same issue --- src/main/java/com/uid2/operator/model/KeyManager.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/model/KeyManager.java b/src/main/java/com/uid2/operator/model/KeyManager.java index 95e73b61d..a9796757f 100644 --- a/src/main/java/com/uid2/operator/model/KeyManager.java +++ b/src/main/java/com/uid2/operator/model/KeyManager.java @@ -126,7 +126,10 @@ public KeysetKey getRefreshKey() { } public KeysetKey getRefreshKey(Instant asOf) { - KeysetKey key = this.keysetKeyStore.getSnapshot().getActiveKey(Const.Data.RefreshKeysetId, asOf); + // TEMPORARY: Simulate keyset key unavailability to reproduce Univision issue + KeysetKey key = null; // Force key to be null to trigger exception + + // KeysetKey key = this.keysetKeyStore.getSnapshot().getActiveKey(Const.Data.RefreshKeysetId, asOf); if (key == null) { if (keyAvailabilityHandler != null) keyAvailabilityHandler.accept(false); throw new NoActiveKeyException(String.format("Cannot get a refresh key with keyset ID %d.", Const.Data.RefreshKeysetId)); From 9198423f0cadec4413488e4d4ee69f10c613f8af Mon Sep 17 00:00:00 2001 From: way zheng Date: Fri, 10 Oct 2025 11:33:26 -0700 Subject: [PATCH 13/28] keep track of refresh failure instead of NoActiveKey, which is too late --- src/main/java/com/uid2/operator/Main.java | 234 +++++++++++------- .../com/uid2/operator/model/KeyManager.java | 39 +-- .../vertx/OperatorShutdownHandler.java | 32 ++- .../operator/OperatorShutdownHandlerTest.java | 67 +++-- 4 files changed, 227 insertions(+), 145 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index ad887c436..b6eaf0056 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -61,6 +61,7 @@ import java.time.Duration; import java.time.Instant; import java.util.*; +import java.util.function.Consumer; import java.util.function.Supplier; import static com.uid2.operator.Const.Config.EnableRemoteConfigProp; @@ -100,7 +101,8 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.config = config; this.appVersion = ApplicationVersion.load("uid2-operator", "uid2-shared", "uid2-attestation-api"); - HealthManager.instance.registerGenericComponent(new PodTerminationMonitor(config.getInteger(Const.Config.PodTerminationCheckInterval, 3000))); + HealthManager.instance.registerGenericComponent( + new PodTerminationMonitor(config.getInteger(Const.Config.PodTerminationCheckInterval, 3000))); // allow to switch between in-mem optout file cache and on-disk file cache if (config.getBoolean(Const.Config.OptOutInMemCacheProp)) { @@ -114,8 +116,9 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.clientSideTokenGenerate = config.getBoolean(Const.Config.EnableClientSideTokenGenerate, false); this.validateServiceLinks = config.getBoolean(Const.Config.ValidateServiceLinks, false); this.encryptedCloudFilesEnabled = config.getBoolean(Const.Config.EncryptedFiles, false); - //todo: change to a config values after testing - this.shutdownHandler = new OperatorShutdownHandler(Duration.ofHours(12), Duration.ofHours(config.getInteger(Const.Config.SaltsExpiredShutdownHours, 12)), Duration.ofMinutes(5), Clock.systemUTC(), new ShutdownService()); + this.shutdownHandler = new OperatorShutdownHandler(Duration.ofHours(12), + Duration.ofHours(config.getInteger(Const.Config.SaltsExpiredShutdownHours, 12)), Duration.ofDays(7), + Clock.systemUTC(), new ShutdownService()); this.uidInstanceIdProvider = new UidInstanceIdProvider(config); String coreAttestUrl = this.config.getString(Const.Config.CoreAttestUrlProp); @@ -126,7 +129,8 @@ public Main(Vertx vertx, JsonObject config) throws Exception { DownloadCloudStorage fsStores; if (coreAttestUrl != null) { - var clients = createUidClients(this.vertx, coreAttestUrl, operatorKey, this.shutdownHandler::handleAttestResponse); + var clients = createUidClients(this.vertx, coreAttestUrl, operatorKey, + this.shutdownHandler::handleAttestResponse); UidCoreClient coreClient = clients.getKey(); UidOptOutClient optOutClient = clients.getValue(); fsStores = coreClient; @@ -159,7 +163,8 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.clientSideKeypairProvider = new RotatingClientSideKeypairStore(fsStores, new GlobalScope(new CloudPath(keypairMdPath)), cloudEncryptionKeyProvider); String clientsMdPath = this.config.getString(Const.Config.ClientsMetadataPathProp); - this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, new GlobalScope(new CloudPath(clientsMdPath)), + this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, + new GlobalScope(new CloudPath(clientsMdPath)), cloudEncryptionKeyProvider); String keysetKeysMdPath = this.config.getString(Const.Config.KeysetKeysMetadataPathProp); this.keysetKeyStore = new RotatingKeysetKeyStore(fsStores, new GlobalScope(new CloudPath(keysetKeysMdPath)), @@ -173,25 +178,30 @@ public Main(Vertx vertx, JsonObject config) throws Exception { String sitesMdPath = this.config.getString(Const.Config.SitesMetadataPathProp); this.siteProvider = clientSideTokenGenerate ? new RotatingSiteStore(fsStores, new GlobalScope(new CloudPath(sitesMdPath)), - cloudEncryptionKeyProvider) + cloudEncryptionKeyProvider) : null; } else { String keypairMdPath = this.config.getString(Const.Config.ClientSideKeypairsMetadataPathProp); - this.clientSideKeypairProvider = new RotatingClientSideKeypairStore(fsStores, new GlobalScope(new CloudPath(keypairMdPath))); + this.clientSideKeypairProvider = new RotatingClientSideKeypairStore(fsStores, + new GlobalScope(new CloudPath(keypairMdPath))); String clientsMdPath = this.config.getString(Const.Config.ClientsMetadataPathProp); - this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, new GlobalScope(new CloudPath(clientsMdPath))); + this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, + new GlobalScope(new CloudPath(clientsMdPath))); String keysetKeysMdPath = this.config.getString(Const.Config.KeysetKeysMetadataPathProp); - this.keysetKeyStore = new RotatingKeysetKeyStore(fsStores, new GlobalScope(new CloudPath(keysetKeysMdPath))); + this.keysetKeyStore = new RotatingKeysetKeyStore(fsStores, + new GlobalScope(new CloudPath(keysetKeysMdPath))); String keysetMdPath = this.config.getString(Const.Config.KeysetsMetadataPathProp); this.keysetProvider = new RotatingKeysetProvider(fsStores, new GlobalScope(new CloudPath(keysetMdPath))); String saltsMdPath = this.config.getString(Const.Config.SaltsMetadataPathProp); this.saltProvider = new RotatingSaltProvider(fsStores, saltsMdPath); String sitesMdPath = this.config.getString(Const.Config.SitesMetadataPathProp); - this.siteProvider = clientSideTokenGenerate ? new RotatingSiteStore(fsStores, new GlobalScope(new CloudPath(sitesMdPath))) : null; + this.siteProvider = clientSideTokenGenerate + ? new RotatingSiteStore(fsStores, new GlobalScope(new CloudPath(sitesMdPath))) + : null; } this.optOutStore = new CloudSyncOptOutStore(vertx, fsLocal, this.config, operatorKey, Clock.systemUTC()); - + if (useRemoteConfig) { String configMdPath = this.config.getString(Const.Config.RuntimeConfigMetadataPathProp); this.configStore = new RuntimeConfigStore(fsStores, configMdPath); @@ -203,7 +213,8 @@ public Main(Vertx vertx, JsonObject config) throws Exception { String serviceMdPath = this.config.getString(Const.Config.ServiceMetadataPathProp); this.serviceProvider = new RotatingServiceStore(fsStores, new GlobalScope(new CloudPath(serviceMdPath))); String serviceLinkMdPath = this.config.getString(Const.Config.ServiceLinkMetadataPathProp); - this.serviceLinkProvider = new RotatingServiceLinkStore(fsStores, new GlobalScope(new CloudPath(serviceLinkMdPath))); + this.serviceLinkProvider = new RotatingServiceLinkStore(fsStores, + new GlobalScope(new CloudPath(serviceLinkMdPath))); } if (useStorageMock && coreAttestUrl == null) { @@ -244,11 +255,12 @@ public Main(Vertx vertx, JsonObject config) throws Exception { } private KeyManager getKeyManager() { - return new KeyManager(this.keysetKeyStore, this.keysetProvider, this.shutdownHandler::handleKeysetKeyRefreshResponse); + return new KeyManager(this.keysetKeyStore, this.keysetProvider); } public static void recordStartupComplete() { - if (startupBeginTime == null) return; + if (startupBeginTime == null) + return; final Duration d = Duration.between(startupBeginTime, Instant.now()); Timer.builder("uid2_operator_startup_duration").register(globalRegistry).record(d); LOGGER.info("Startup in {} ms", d.toMillis()); @@ -257,13 +269,12 @@ public static void recordStartupComplete() { public static void main(String[] args) throws Exception { startupBeginTime = Instant.now(); - java.security.Security.setProperty("networkaddress.cache.ttl" , "60"); + java.security.Security.setProperty("networkaddress.cache.ttl", "60"); final String vertxConfigPath = System.getProperty(Const.Config.VERTX_CONFIG_PATH_PROP); if (vertxConfigPath != null) { LOGGER.info("Running CUSTOM CONFIG mode, config: {}", vertxConfigPath); - } - else if (!Utils.isProductionEnvironment()) { + } else if (!Utils.isProductionEnvironment()) { LOGGER.info("Running LOCAL DEBUG mode, config: {}", Const.Config.LOCAL_CONFIG_PATH); System.setProperty(Const.Config.VERTX_CONFIG_PATH_PROP, Const.Config.LOCAL_CONFIG_PATH); } else { @@ -284,7 +295,7 @@ else if (!Utils.isProductionEnvironment()) { app.run(); } catch (Exception e) { LOGGER.error("Error: " + e.getMessage(), e); - ((LoggerContext)org.slf4j.LoggerFactory.getILoggerFactory()).stop(); // flush logs before shutdown + ((LoggerContext) org.slf4j.LoggerFactory.getILoggerFactory()).stop(); // flush logs before shutdown vertx.close(); System.exit(1); } @@ -308,27 +319,27 @@ private ICloudStorage configureCloudOptOutStore() { private ICloudStorage configureAttestedOptOutStore(UidOptOutClient optOutClient, String coreAttestUrl) { String optOutMdPath = this.config.getString(Const.Config.OptOutMetadataPathProp); LOGGER.info("OptOut stores- Using uid2-core attestation endpoint: " + coreAttestUrl); - return this.wrapCloudStorageForOptOut(new OptOutCloudStorage(optOutClient, optOutMdPath, CloudUtils.defaultProxy)); + return this + .wrapCloudStorageForOptOut(new OptOutCloudStorage(optOutClient, optOutMdPath, CloudUtils.defaultProxy)); } private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { if (config.getBoolean(Const.Config.OptOutS3PathCompatProp)) { LOGGER.warn("Using S3 Path Compatibility Conversion: log -> delta, snapshot -> partition"); return new PathConversionWrapper( - cloudStorage, - in -> { - String out = in.replace("log", "delta") - .replace("snapshot", "partition"); - LOGGER.debug("S3 path forward convert: " + in + " -> " + out); - return out; - }, - in -> { - String out = in.replace("delta", "log") - .replace("partition", "snapshot"); - LOGGER.debug("S3 path backward convert: " + in + " -> " + out); - return out; - } - ); + cloudStorage, + in -> { + String out = in.replace("log", "delta") + .replace("snapshot", "partition"); + LOGGER.debug("S3 path forward convert: " + in + " -> " + out); + return out; + }, + in -> { + String out = in.replace("delta", "log") + .replace("partition", "snapshot"); + LOGGER.debug("S3 path backward convert: " + in + " -> " + out); + return out; + }); } else { return cloudStorage; } @@ -339,7 +350,11 @@ private void run() throws Exception { this.createVertxEventLoopsMetric(); Supplier operatorVerticleSupplier = () -> { - UIDOperatorVerticle verticle = new UIDOperatorVerticle(configStore, config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse, this.uidInstanceIdProvider); + UIDOperatorVerticle verticle = new UIDOperatorVerticle(configStore, config, this.clientSideTokenGenerate, + siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, + optOutStore, Clock.systemUTC(), _statsCollectorQueue, + new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), + this.shutdownHandler::handleSaltRetrievalResponse, this.uidInstanceIdProvider); return verticle; }; @@ -353,8 +368,10 @@ private void run() throws Exception { fs.add(createStoreVerticles()); CompositeFuture.all(fs).onComplete(ar -> { - if (ar.failed()) compositePromise.fail(new Exception(ar.cause())); - else compositePromise.complete(); + if (ar.failed()) + compositePromise.fail(new Exception(ar.cause())); + else + compositePromise.complete(); }); compositePromise.future() @@ -404,51 +421,66 @@ private Future createStoreVerticles() throws Exception { if (clientSideTokenGenerate) { fs.add(createAndDeployRotatingStoreVerticle("site", siteProvider, "site_refresh_ms")); - fs.add(createAndDeployRotatingStoreVerticle("client_side_keypairs", clientSideKeypairProvider, "client_side_keypairs_refresh_ms")); + fs.add(createAndDeployRotatingStoreVerticle("client_side_keypairs", clientSideKeypairProvider, + "client_side_keypairs_refresh_ms")); } if (validateServiceLinks) { fs.add(createAndDeployRotatingStoreVerticle("service", serviceProvider, "service_refresh_ms")); - fs.add(createAndDeployRotatingStoreVerticle("service_link", serviceLinkProvider, "service_link_refresh_ms")); + fs.add(createAndDeployRotatingStoreVerticle("service_link", serviceLinkProvider, + "service_link_refresh_ms")); } if (encryptedCloudFilesEnabled) { - fs.add(createAndDeployRotatingStoreVerticle("cloud_encryption_keys", cloudEncryptionKeyProvider, "cloud_encryption_keys_refresh_ms")); + fs.add(createAndDeployRotatingStoreVerticle("cloud_encryption_keys", cloudEncryptionKeyProvider, + "cloud_encryption_keys_refresh_ms")); } if (useRemoteConfig) { - fs.add(createAndDeployRotatingStoreVerticle("runtime_config", (RuntimeConfigStore) configStore, Const.Config.ConfigScanPeriodMsProp)); + fs.add(createAndDeployRotatingStoreVerticle("runtime_config", (RuntimeConfigStore) configStore, + Const.Config.ConfigScanPeriodMsProp)); } fs.add(createAndDeployRotatingStoreVerticle("auth", clientKeyProvider, "auth_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("keyset", keysetProvider, "keyset_refresh_ms")); - fs.add(createAndDeployRotatingStoreVerticle("keysetkey", keysetKeyStore, "keysetkey_refresh_ms")); + fs.add(createAndDeployRotatingStoreVerticle("keysetkey", keysetKeyStore, "keysetkey_refresh_ms", + this.shutdownHandler::handleKeysetKeyRefreshResponse)); fs.add(createAndDeployRotatingStoreVerticle("salt", saltProvider, "salt_refresh_ms")); fs.add(createAndDeployCloudSyncStoreVerticle("optout", fsOptOut, optOutCloudSync)); CompositeFuture.all(fs).onComplete(ar -> { - if (ar.failed()) promise.fail(new Exception(ar.cause())); - else promise.complete(); + if (ar.failed()) + promise.fail(new Exception(ar.cause())); + else + promise.complete(); }); - return promise.future(); } - private Future createAndDeployRotatingStoreVerticle(String name, IMetadataVersionedStore store, String storeRefreshConfigMs) { + private Future createAndDeployRotatingStoreVerticle(String name, IMetadataVersionedStore store, + String storeRefreshConfigMs) { + return createAndDeployRotatingStoreVerticle(name, store, storeRefreshConfigMs, null); + } + + private Future createAndDeployRotatingStoreVerticle(String name, IMetadataVersionedStore store, + String storeRefreshConfigMs, Consumer refreshCallback) { final int intervalMs = config.getInteger(storeRefreshConfigMs, 10000); - RotatingStoreVerticle rotatingStoreVerticle = new RotatingStoreVerticle(name, intervalMs, store); + RotatingStoreVerticle rotatingStoreVerticle = new RotatingStoreVerticle(name, intervalMs, store, + refreshCallback); return vertx.deployVerticle(rotatingStoreVerticle); } private Future createAndDeployCloudSyncStoreVerticle(String name, ICloudStorage fsCloud, - ICloudSync cloudSync) { + ICloudSync cloudSync) { CloudSyncVerticle cloudSyncVerticle = new CloudSyncVerticle(name, fsCloud, fsLocal, cloudSync, config); return vertx.deployVerticle(cloudSyncVerticle) - .onComplete(v -> setupTimerEvent(cloudSyncVerticle.eventRefresh())); + .onComplete(v -> setupTimerEvent(cloudSyncVerticle.eventRefresh())); } private Future createAndDeployStatsCollector() { - StatsCollectorVerticle statsCollectorVerticle = new StatsCollectorVerticle(60000, config.getInteger(Const.Config.MaxInvalidPaths, 50), config.getInteger(Const.Config.MaxVersionBucketsPerSite, 50)); + StatsCollectorVerticle statsCollectorVerticle = new StatsCollectorVerticle(60000, + config.getInteger(Const.Config.MaxInvalidPaths, 50), + config.getInteger(Const.Config.MaxVersionBucketsPerSite, 50)); Future result = vertx.deployVerticle(statsCollectorVerticle); _statsCollectorQueue = statsCollectorVerticle; return result; @@ -467,31 +499,33 @@ private static Vertx createVertx() { ObjectName objectName = new ObjectName("uid2.operator:type=jmx,name=AdminApi"); MBeanServer server = ManagementFactory.getPlatformMBeanServer(); server.registerMBean(AdminApi.instance, objectName); - } catch (InstanceAlreadyExistsException | MBeanRegistrationException | NotCompliantMBeanException | MalformedObjectNameException e) { + } catch (InstanceAlreadyExistsException | MBeanRegistrationException | NotCompliantMBeanException + | MalformedObjectNameException e) { LOGGER.error("mBean initialisation failed {}", e.getMessage(), e); System.exit(-1); } final int portOffset = Utils.getPortOffset(); VertxPrometheusOptions prometheusOptions = new VertxPrometheusOptions() - .setStartEmbeddedServer(true) - .setEmbeddedServerOptions(new HttpServerOptions().setPort(Const.Port.PrometheusPortForOperator + portOffset)) - .setEnabled(true); + .setStartEmbeddedServer(true) + .setEmbeddedServerOptions( + new HttpServerOptions().setPort(Const.Port.PrometheusPortForOperator + portOffset)) + .setEnabled(true); MicrometerMetricsOptions metricOptions = new MicrometerMetricsOptions() - .setPrometheusOptions(prometheusOptions) - .setLabels(EnumSet.of(Label.HTTP_METHOD, Label.HTTP_CODE, Label.HTTP_PATH)) - .setJvmMetricsEnabled(true) - .setEnabled(true); + .setPrometheusOptions(prometheusOptions) + .setLabels(EnumSet.of(Label.HTTP_METHOD, Label.HTTP_CODE, Label.HTTP_PATH)) + .setJvmMetricsEnabled(true) + .setEnabled(true); setupMetrics(metricOptions); final int threadBlockedCheckInterval = Utils.isProductionEnvironment() - ? 60 * 1000 - : 3600 * 1000; + ? 60 * 1000 + : 3600 * 1000; VertxOptions vertxOptions = new VertxOptions() - .setMetricsOptions(metricOptions) - .setBlockedThreadCheckInterval(threadBlockedCheckInterval); + .setMetricsOptions(metricOptions) + .setBlockedThreadCheckInterval(threadBlockedCheckInterval); return Vertx.vertx(vertxOptions); } @@ -506,32 +540,35 @@ private static void setupMetrics(MicrometerMetricsOptions metricOptions) { // see also https://micrometer.io/docs/registry/prometheus prometheusRegistry.config() - // providing common renaming for prometheus metric, e.g. "hello.world" to "hello_world" - .meterFilter(new PrometheusRenameFilter()) - .meterFilter(MeterFilter.replaceTagValues(Label.HTTP_PATH.toString(), - actualPath -> HTTPPathMetricFilter.filterPath(actualPath, Endpoints.pathSet()))) - // Don't record metrics for 404s. - .meterFilter(MeterFilter.deny(id -> - id.getName().startsWith(MetricsDomain.HTTP_SERVER.getPrefix()) && - Objects.equals(id.getTag(Label.HTTP_CODE.toString()), "404"))) - .meterFilter(new MeterFilter() { - private final String httpServerResponseTime = MetricsDomain.HTTP_SERVER.getPrefix() + MetricsNaming.v4Names().getHttpResponseTime(); - - @Override - public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticConfig config) { - if (id.getName().equals(httpServerResponseTime)) { - return DistributionStatisticConfig.builder() - .percentiles(0.90, 0.95, 0.99) - .build() - .merge(config); + // providing common renaming for prometheus metric, e.g. "hello.world" to + // "hello_world" + .meterFilter(new PrometheusRenameFilter()) + .meterFilter(MeterFilter.replaceTagValues(Label.HTTP_PATH.toString(), + actualPath -> HTTPPathMetricFilter.filterPath(actualPath, Endpoints.pathSet()))) + // Don't record metrics for 404s. + .meterFilter( + MeterFilter.deny(id -> id.getName().startsWith(MetricsDomain.HTTP_SERVER.getPrefix()) && + Objects.equals(id.getTag(Label.HTTP_CODE.toString()), "404"))) + .meterFilter(new MeterFilter() { + private final String httpServerResponseTime = MetricsDomain.HTTP_SERVER.getPrefix() + + MetricsNaming.v4Names().getHttpResponseTime(); + + @Override + public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticConfig config) { + if (id.getName().equals(httpServerResponseTime)) { + return DistributionStatisticConfig.builder() + .percentiles(0.90, 0.95, 0.99) + .build() + .merge(config); + } + return config; } - return config; - } - }) - // adding common labels - .commonTags("application", "uid2-operator"); + }) + // adding common labels + .commonTags("application", "uid2-operator"); - // wire my monitoring system to global static state, see also https://micrometer.io/docs/concepts + // wire my monitoring system to global static state, see also + // https://micrometer.io/docs/concepts Metrics.addRegistry(prometheusRegistry); } @@ -556,14 +593,19 @@ private void createVertxEventLoopsMetric() { .register(Metrics.globalRegistry); } - private Map.Entry createUidClients(Vertx vertx, String attestationUrl, String clientApiToken, Handler> responseWatcher) throws Exception { - AttestationResponseHandler attestationResponseHandler = getAttestationTokenRetriever(vertx, attestationUrl, clientApiToken, responseWatcher); - UidCoreClient coreClient = new UidCoreClient(clientApiToken, CloudUtils.defaultProxy, attestationResponseHandler, this.encryptedCloudFilesEnabled, this.uidInstanceIdProvider); - UidOptOutClient optOutClient = new UidOptOutClient(clientApiToken, CloudUtils.defaultProxy, attestationResponseHandler, this.uidInstanceIdProvider); + private Map.Entry createUidClients(Vertx vertx, String attestationUrl, + String clientApiToken, Handler> responseWatcher) throws Exception { + AttestationResponseHandler attestationResponseHandler = getAttestationTokenRetriever(vertx, attestationUrl, + clientApiToken, responseWatcher); + UidCoreClient coreClient = new UidCoreClient(clientApiToken, CloudUtils.defaultProxy, + attestationResponseHandler, this.encryptedCloudFilesEnabled, this.uidInstanceIdProvider); + UidOptOutClient optOutClient = new UidOptOutClient(clientApiToken, CloudUtils.defaultProxy, + attestationResponseHandler, this.uidInstanceIdProvider); return new AbstractMap.SimpleEntry<>(coreClient, optOutClient); } - private AttestationResponseHandler getAttestationTokenRetriever(Vertx vertx, String attestationUrl, String clientApiToken, Handler> responseWatcher) throws Exception { + private AttestationResponseHandler getAttestationTokenRetriever(Vertx vertx, String attestationUrl, + String clientApiToken, Handler> responseWatcher) throws Exception { String enclavePlatform = this.config.getString(Const.Config.EnclavePlatformProp); String operatorType = this.config.getString(Const.Config.OperatorTypeProp, ""); @@ -588,14 +630,17 @@ private AttestationResponseHandler getAttestationTokenRetriever(Vertx vertx, Str break; case "azure-cc": LOGGER.info("creating uid core client with azure cc attestation protocol"); - String maaServerBaseUrl = this.config.getString(Const.Config.MaaServerBaseUrlProp, "https://sharedeus.eus.attest.azure.net"); + String maaServerBaseUrl = this.config.getString(Const.Config.MaaServerBaseUrlProp, + "https://sharedeus.eus.attest.azure.net"); attestationProvider = AttestationFactory.getAzureCCAttestation(maaServerBaseUrl); break; default: - throw new IllegalArgumentException(String.format("enclave_platform is providing the wrong value: %s", enclavePlatform)); + throw new IllegalArgumentException( + String.format("enclave_platform is providing the wrong value: %s", enclavePlatform)); } - return new AttestationResponseHandler(vertx, attestationUrl, clientApiToken, operatorType, this.appVersion, attestationProvider, responseWatcher, CloudUtils.defaultProxy, this.uidInstanceIdProvider); + return new AttestationResponseHandler(vertx, attestationUrl, clientApiToken, operatorType, this.appVersion, + attestationProvider, responseWatcher, CloudUtils.defaultProxy, this.uidInstanceIdProvider); } private IOperatorKeyRetriever createOperatorKeyRetriever() throws Exception { @@ -618,7 +663,8 @@ private IOperatorKeyRetriever createOperatorKeyRetriever() throws Exception { return OperatorKeyRetrieverFactory.getGcpOperatorKeyRetriever(secretVersionName); } default: { - throw new IllegalArgumentException(String.format("enclave_platform is providing the wrong value: %s", enclavePlatform)); + throw new IllegalArgumentException( + String.format("enclave_platform is providing the wrong value: %s", enclavePlatform)); } } } diff --git a/src/main/java/com/uid2/operator/model/KeyManager.java b/src/main/java/com/uid2/operator/model/KeyManager.java index a9796757f..09e59687d 100644 --- a/src/main/java/com/uid2/operator/model/KeyManager.java +++ b/src/main/java/com/uid2/operator/model/KeyManager.java @@ -25,8 +25,9 @@ public class KeyManager { public KeyManager(IKeysetKeyStore keysetKeyStore, RotatingKeysetProvider keysetProvider) { this(keysetKeyStore, keysetProvider, null); } - - public KeyManager(IKeysetKeyStore keysetKeyStore, RotatingKeysetProvider keysetProvider, Consumer keyAvailabilityHandler) { + + public KeyManager(IKeysetKeyStore keysetKeyStore, RotatingKeysetProvider keysetProvider, + Consumer keyAvailabilityHandler) { this.keysetKeyStore = keysetKeyStore; this.keysetProvider = keysetProvider; this.keyAvailabilityHandler = keyAvailabilityHandler; @@ -43,9 +44,11 @@ public KeyManagerSnapshot getKeyManagerSnapshot(int siteId) { public KeysetKey getActiveKeyBySiteIdWithFallback(int siteId, int fallbackSiteId, Instant asOf) { KeysetKey key = getActiveKeyBySiteId(siteId, asOf); - if (key == null) key = getActiveKeyBySiteId(fallbackSiteId, asOf); + if (key == null) + key = getActiveKeyBySiteId(fallbackSiteId, asOf); if (key == null) { - throw new NoActiveKeyException(String.format("Cannot get active key in default keyset with SITE ID %d or %d.", siteId, fallbackSiteId)); + throw new NoActiveKeyException(String + .format("Cannot get active key in default keyset with SITE ID %d or %d.", siteId, fallbackSiteId)); } return key; } @@ -58,7 +61,8 @@ private Keyset getDefaultKeysetBySiteId(int siteId) { } if (keysets.size() > 1) { - throw new IllegalArgumentException(String.format("Multiple default keysets are enabled with SITE ID %d.", siteId)); + throw new IllegalArgumentException( + String.format("Multiple default keysets are enabled with SITE ID %d.", siteId)); } return keysets.get(0); @@ -83,12 +87,12 @@ public KeysetKey getKey(int keyId) { return this.keysetKeyStore.getSnapshot().getKey(keyId); } - public List getKeysForSharingOrDsps() { Map keysetMap = this.keysetProvider.getSnapshot().getAllKeysets(); List keys = keysetKeyStore.getSnapshot().getAllKeysetKeys(); return keys - .stream().filter(k -> keysetMap.containsKey(k.getKeysetId()) && k.getKeysetId() != Const.Data.RefreshKeysetId) + .stream() + .filter(k -> keysetMap.containsKey(k.getKeysetId()) && k.getKeysetId() != Const.Data.RefreshKeysetId) .sorted(Comparator.comparing(KeysetKey::getId)).collect(Collectors.toList()); } @@ -114,10 +118,12 @@ public KeysetKey getMasterKey() { public KeysetKey getMasterKey(Instant asOf) { KeysetKey key = this.keysetKeyStore.getSnapshot().getActiveKey(Const.Data.MasterKeysetId, asOf); if (key == null) { - if (keyAvailabilityHandler != null) keyAvailabilityHandler.accept(false); - throw new NoActiveKeyException(String.format("Cannot get a master key with keyset ID %d.", Const.Data.MasterKeysetId)); + throw new NoActiveKeyException( + String.format("Cannot get a master key with keyset ID %d.", Const.Data.MasterKeysetId)); } - if (keyAvailabilityHandler != null) keyAvailabilityHandler.accept(true); + // Reset shutdown timer on successful key retrieval + if (keyAvailabilityHandler != null) + keyAvailabilityHandler.accept(true); return key; } @@ -126,15 +132,14 @@ public KeysetKey getRefreshKey() { } public KeysetKey getRefreshKey(Instant asOf) { - // TEMPORARY: Simulate keyset key unavailability to reproduce Univision issue - KeysetKey key = null; // Force key to be null to trigger exception - - // KeysetKey key = this.keysetKeyStore.getSnapshot().getActiveKey(Const.Data.RefreshKeysetId, asOf); + KeysetKey key = this.keysetKeyStore.getSnapshot().getActiveKey(Const.Data.RefreshKeysetId, asOf); if (key == null) { - if (keyAvailabilityHandler != null) keyAvailabilityHandler.accept(false); - throw new NoActiveKeyException(String.format("Cannot get a refresh key with keyset ID %d.", Const.Data.RefreshKeysetId)); + throw new NoActiveKeyException( + String.format("Cannot get a refresh key with keyset ID %d.", Const.Data.RefreshKeysetId)); } - if (keyAvailabilityHandler != null) keyAvailabilityHandler.accept(true); + // Reset shutdown timer on successful key retrieval + if (keyAvailabilityHandler != null) + keyAvailabilityHandler.accept(true); return key; } diff --git a/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java b/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java index f9961d4c7..d0edaa1ea 100644 --- a/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java +++ b/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java @@ -28,11 +28,13 @@ public class OperatorShutdownHandler { private final Clock clock; private final ShutdownService shutdownService; - public OperatorShutdownHandler(Duration attestShutdownWaitTime, Duration saltShutdownWaitTime, Clock clock, ShutdownService shutdownService) { - this(attestShutdownWaitTime, saltShutdownWaitTime, Duration.ofHours(2), clock, shutdownService); + public OperatorShutdownHandler(Duration attestShutdownWaitTime, Duration saltShutdownWaitTime, Clock clock, + ShutdownService shutdownService) { + this(attestShutdownWaitTime, saltShutdownWaitTime, Duration.ofDays(7), clock, shutdownService); } - - public OperatorShutdownHandler(Duration attestShutdownWaitTime, Duration saltShutdownWaitTime, Duration keysetKeyShutdownWaitTime, Clock clock, ShutdownService shutdownService) { + + public OperatorShutdownHandler(Duration attestShutdownWaitTime, Duration saltShutdownWaitTime, + Duration keysetKeyShutdownWaitTime, Clock clock, ShutdownService shutdownService) { this.attestShutdownWaitTime = attestShutdownWaitTime; this.saltShutdownWaitTime = saltShutdownWaitTime; this.keysetKeyShutdownWaitTime = keysetKeyShutdownWaitTime; @@ -41,14 +43,14 @@ public OperatorShutdownHandler(Duration attestShutdownWaitTime, Duration saltShu } public void handleSaltRetrievalResponse(Boolean expired) { - if(!expired) { + if (!expired) { saltFailureStartTime.set(null); } else { logSaltFailureAtInterval(); Instant t = saltFailureStartTime.get(); if (t == null) { saltFailureStartTime.set(clock.instant()); - } else if(Duration.between(t, clock.instant()).compareTo(this.saltShutdownWaitTime) > 0) { + } else if (Duration.between(t, clock.instant()).compareTo(this.saltShutdownWaitTime) > 0) { LOGGER.error("salts have been in expired state for too long. shutting down operator"); this.shutdownService.Shutdown(1); } @@ -57,7 +59,7 @@ public void handleSaltRetrievalResponse(Boolean expired) { public void logSaltFailureAtInterval() { Instant t = lastSaltFailureLogTime.get(); - if(t == null || clock.instant().isAfter(t.plus(SALT_FAILURE_LOG_INTERVAL_MINUTES, ChronoUnit.MINUTES))) { + if (t == null || clock.instant().isAfter(t.plus(SALT_FAILURE_LOG_INTERVAL_MINUTES, ChronoUnit.MINUTES))) { LOGGER.error("all salts are expired"); lastSaltFailureLogTime.set(Instant.now()); } @@ -65,12 +67,23 @@ public void logSaltFailureAtInterval() { public void handleKeysetKeyRefreshResponse(Boolean success) { if (success) { - keysetKeyFailureStartTime.set(null); + Instant previousFailureTime = keysetKeyFailureStartTime.getAndSet(null); + if (previousFailureTime != null) { + Duration failureDuration = Duration.between(previousFailureTime, clock.instant()); + // can remove later + LOGGER.info("keyset keys sync recovered after {} ({}d {}h {}m). shutdown timer reset.", + failureDuration, + failureDuration.toDays(), + failureDuration.toHoursPart(), + failureDuration.toMinutesPart()); + } } else { logKeysetKeyFailureAtInterval(); Instant t = keysetKeyFailureStartTime.get(); if (t == null) { keysetKeyFailureStartTime.set(clock.instant()); + LOGGER.warn( + "keyset keys sync started failing. shutdown timer started (will shutdown in 7 days if not recovered)"); } else if (Duration.between(t, clock.instant()).compareTo(this.keysetKeyShutdownWaitTime) > 0) { LOGGER.error("keyset keys have been failing to sync for too long. shutting down operator"); this.shutdownService.Shutdown(1); @@ -88,7 +101,8 @@ public void logKeysetKeyFailureAtInterval() { public void handleAttestResponse(Pair response) { if (response.left() == AttestationResponseCode.AttestationFailure) { - LOGGER.error("core attestation failed with AttestationFailure, shutting down operator, core response: {}", response.right()); + LOGGER.error("core attestation failed with AttestationFailure, shutting down operator, core response: {}", + response.right()); this.shutdownService.Shutdown(1); } if (response.left() == AttestationResponseCode.Success) { diff --git a/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java b/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java index 4350de394..881da4341 100644 --- a/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java +++ b/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java @@ -32,18 +32,19 @@ public class OperatorShutdownHandlerTest { private AutoCloseable mocks; - @Mock private Clock clock; - @Mock private ShutdownService shutdownService; + @Mock + private Clock clock; + @Mock + private ShutdownService shutdownService; private OperatorShutdownHandler operatorShutdownHandler; - - @BeforeEach void beforeEach() { mocks = MockitoAnnotations.openMocks(this); when(clock.instant()).thenAnswer(i -> Instant.now()); doThrow(new RuntimeException()).when(shutdownService).Shutdown(1); - this.operatorShutdownHandler = new OperatorShutdownHandler(Duration.ofHours(12), Duration.ofHours(12), clock, shutdownService); + this.operatorShutdownHandler = new OperatorShutdownHandler(Duration.ofHours(12), Duration.ofHours(12), clock, + shutdownService); } @AfterEach @@ -59,11 +60,14 @@ void shutdownOnAttestFailure(VertxTestContext testContext) { // Revoke auth try { - this.operatorShutdownHandler.handleAttestResponse(Pair.of(AttestationResponseCode.AttestationFailure, "Unauthorized")); + this.operatorShutdownHandler + .handleAttestResponse(Pair.of(AttestationResponseCode.AttestationFailure, "Unauthorized")); } catch (RuntimeException e) { verify(shutdownService).Shutdown(1); String message = logWatcher.list.get(0).getFormattedMessage(); - Assertions.assertEquals("core attestation failed with AttestationFailure, shutting down operator, core response: Unauthorized", logWatcher.list.get(0).getFormattedMessage()); + Assertions.assertEquals( + "core attestation failed with AttestationFailure, shutting down operator, core response: Unauthorized", + logWatcher.list.get(0).getFormattedMessage()); testContext.completeNow(); } } @@ -81,7 +85,8 @@ void shutdownOnAttestFailedTooLong(VertxTestContext testContext) { this.operatorShutdownHandler.handleAttestResponse(Pair.of(AttestationResponseCode.RetryableFailure, "")); } catch (RuntimeException e) { verify(shutdownService).Shutdown(1); - Assertions.assertTrue(logWatcher.list.get(0).getFormattedMessage().contains("core attestation has been in failed state for too long. shutting down operator")); + Assertions.assertTrue(logWatcher.list.get(0).getFormattedMessage() + .contains("core attestation has been in failed state for too long. shutting down operator")); testContext.completeNow(); } } @@ -119,8 +124,10 @@ void shutdownOnSaltsExpiredTooLong(VertxTestContext testContext) { }); Assertions.assertAll("Expired Salts Log Messages", () -> verify(shutdownService).Shutdown(1), - () -> Assertions.assertTrue(logWatcher.list.get(1).getFormattedMessage().contains("all salts are expired")), - () -> Assertions.assertTrue(logWatcher.list.get(2).getFormattedMessage().contains("salts have been in expired state for too long. shutting down operator")), + () -> Assertions + .assertTrue(logWatcher.list.get(1).getFormattedMessage().contains("all salts are expired")), + () -> Assertions.assertTrue(logWatcher.list.get(2).getFormattedMessage() + .contains("salts have been in expired state for too long. shutting down operator")), () -> Assertions.assertEquals(3, logWatcher.list.size())); testContext.completeNow(); @@ -175,18 +182,22 @@ void shutdownOnKeysetKeyFailedTooLong(VertxTestContext testContext) { this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(false); Assertions.assertTrue(logWatcher.list.get(0).getFormattedMessage().contains("keyset keys sync failing")); + Assertions + .assertTrue(logWatcher.list.get(1).getFormattedMessage().contains("keyset keys sync started failing")); + + when(clock.instant()).thenAnswer(i -> Instant.now().plus(7, ChronoUnit.DAYS).plusSeconds(60)); - when(clock.instant()).thenAnswer(i -> Instant.now().plus(2, ChronoUnit.HOURS).plusSeconds(60)); - Assertions.assertThrows(RuntimeException.class, () -> { this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(false); }); - + Assertions.assertAll("Keyset Key Failure Log Messages", () -> verify(shutdownService).Shutdown(1), - () -> Assertions.assertTrue(logWatcher.list.get(1).getFormattedMessage().contains("keyset keys sync failing")), - () -> Assertions.assertTrue(logWatcher.list.get(2).getFormattedMessage().contains("keyset keys have been failing to sync for too long. shutting down operator")), - () -> Assertions.assertEquals(3, logWatcher.list.size())); + () -> Assertions + .assertTrue(logWatcher.list.get(2).getFormattedMessage().contains("keyset keys sync failing")), + () -> Assertions.assertTrue(logWatcher.list.get(3).getFormattedMessage() + .contains("keyset keys have been failing to sync for too long. shutting down operator")), + () -> Assertions.assertEquals(4, logWatcher.list.size())); testContext.completeNow(); } @@ -199,15 +210,19 @@ void keysetKeyRecoverOnSuccess(VertxTestContext testContext) { this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(false); Assertions.assertTrue(logWatcher.list.get(0).getFormattedMessage().contains("keyset keys sync failing")); - - when(clock.instant()).thenAnswer(i -> Instant.now().plus(1, ChronoUnit.HOURS)); + Assertions + .assertTrue(logWatcher.list.get(1).getFormattedMessage().contains("keyset keys sync started failing")); + + when(clock.instant()).thenAnswer(i -> Instant.now().plus(3, ChronoUnit.DAYS)); this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(true); + Assertions.assertTrue(logWatcher.list.get(2).getFormattedMessage().contains("keyset keys sync recovered")); + Assertions.assertTrue(logWatcher.list.get(2).getFormattedMessage().contains("shutdown timer reset")); - when(clock.instant()).thenAnswer(i -> Instant.now().plus(3, ChronoUnit.HOURS)); + when(clock.instant()).thenAnswer(i -> Instant.now().plus(8, ChronoUnit.DAYS)); assertDoesNotThrow(() -> { this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(false); }); - + verify(shutdownService, never()).Shutdown(anyInt()); testContext.completeNow(); } @@ -220,15 +235,17 @@ void keysetKeyLogErrorAtInterval(VertxTestContext testContext) { this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(false); Assertions.assertTrue(logWatcher.list.get(0).getFormattedMessage().contains("keyset keys sync failing")); - + Assertions + .assertTrue(logWatcher.list.get(1).getFormattedMessage().contains("keyset keys sync started failing")); + when(clock.instant()).thenAnswer(i -> Instant.now().plus(9, ChronoUnit.MINUTES)); this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(false); - Assertions.assertEquals(1, logWatcher.list.size()); - + Assertions.assertEquals(2, logWatcher.list.size()); // No new logs within 10 min interval + when(clock.instant()).thenAnswer(i -> Instant.now().plus(11, ChronoUnit.MINUTES)); this.operatorShutdownHandler.handleKeysetKeyRefreshResponse(false); - Assertions.assertTrue(logWatcher.list.get(1).getFormattedMessage().contains("keyset keys sync failing")); - Assertions.assertEquals(2, logWatcher.list.size()); + Assertions.assertTrue(logWatcher.list.get(2).getFormattedMessage().contains("keyset keys sync failing")); + Assertions.assertEquals(3, logWatcher.list.size()); // One new log after 10 min interval testContext.completeNow(); } From fa99a38e57960e9a4fdb23e8b3b0441fd0b8c039 Mon Sep 17 00:00:00 2001 From: way zheng Date: Fri, 10 Oct 2025 12:00:39 -0700 Subject: [PATCH 14/28] clean up --- conf/default-config.json | 3 +- .../com/uid2/operator/model/KeyManager.java | 33 ++++--------------- 2 files changed, 8 insertions(+), 28 deletions(-) diff --git a/conf/default-config.json b/conf/default-config.json index 688f23db5..bfa6bb020 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -37,9 +37,8 @@ "optout_inmem_cache": false, "enclave_platform": null, "failure_shutdown_wait_hours": 120, - "keyset_key_shutdown_hours": 2, "sharing_token_expiry_seconds": 2592000, "operator_type": "public", "enable_remote_config": true, "uid_instance_id_prefix": "local-operator" -} +} \ No newline at end of file diff --git a/src/main/java/com/uid2/operator/model/KeyManager.java b/src/main/java/com/uid2/operator/model/KeyManager.java index 09e59687d..19bae8d07 100644 --- a/src/main/java/com/uid2/operator/model/KeyManager.java +++ b/src/main/java/com/uid2/operator/model/KeyManager.java @@ -13,24 +13,16 @@ import java.util.Comparator; import java.util.List; import java.util.Map; -import java.util.function.Consumer; import java.util.stream.Collectors; public class KeyManager { private static final Logger LOGGER = LoggerFactory.getLogger(UIDOperatorVerticle.class); private final IKeysetKeyStore keysetKeyStore; private final RotatingKeysetProvider keysetProvider; - private final Consumer keyAvailabilityHandler; public KeyManager(IKeysetKeyStore keysetKeyStore, RotatingKeysetProvider keysetProvider) { - this(keysetKeyStore, keysetProvider, null); - } - - public KeyManager(IKeysetKeyStore keysetKeyStore, RotatingKeysetProvider keysetProvider, - Consumer keyAvailabilityHandler) { this.keysetKeyStore = keysetKeyStore; this.keysetProvider = keysetProvider; - this.keyAvailabilityHandler = keyAvailabilityHandler; } public KeyManagerSnapshot getKeyManagerSnapshot(int siteId) { @@ -44,11 +36,9 @@ public KeyManagerSnapshot getKeyManagerSnapshot(int siteId) { public KeysetKey getActiveKeyBySiteIdWithFallback(int siteId, int fallbackSiteId, Instant asOf) { KeysetKey key = getActiveKeyBySiteId(siteId, asOf); - if (key == null) - key = getActiveKeyBySiteId(fallbackSiteId, asOf); + if (key == null) key = getActiveKeyBySiteId(fallbackSiteId, asOf); if (key == null) { - throw new NoActiveKeyException(String - .format("Cannot get active key in default keyset with SITE ID %d or %d.", siteId, fallbackSiteId)); + throw new NoActiveKeyException(String.format("Cannot get active key in default keyset with SITE ID %d or %d.", siteId, fallbackSiteId)); } return key; } @@ -61,8 +51,7 @@ private Keyset getDefaultKeysetBySiteId(int siteId) { } if (keysets.size() > 1) { - throw new IllegalArgumentException( - String.format("Multiple default keysets are enabled with SITE ID %d.", siteId)); + throw new IllegalArgumentException(String.format("Multiple default keysets are enabled with SITE ID %d.", siteId)); } return keysets.get(0); @@ -87,12 +76,12 @@ public KeysetKey getKey(int keyId) { return this.keysetKeyStore.getSnapshot().getKey(keyId); } + public List getKeysForSharingOrDsps() { Map keysetMap = this.keysetProvider.getSnapshot().getAllKeysets(); List keys = keysetKeyStore.getSnapshot().getAllKeysetKeys(); return keys - .stream() - .filter(k -> keysetMap.containsKey(k.getKeysetId()) && k.getKeysetId() != Const.Data.RefreshKeysetId) + .stream().filter(k -> keysetMap.containsKey(k.getKeysetId()) && k.getKeysetId() != Const.Data.RefreshKeysetId) .sorted(Comparator.comparing(KeysetKey::getId)).collect(Collectors.toList()); } @@ -118,12 +107,8 @@ public KeysetKey getMasterKey() { public KeysetKey getMasterKey(Instant asOf) { KeysetKey key = this.keysetKeyStore.getSnapshot().getActiveKey(Const.Data.MasterKeysetId, asOf); if (key == null) { - throw new NoActiveKeyException( - String.format("Cannot get a master key with keyset ID %d.", Const.Data.MasterKeysetId)); + throw new NoActiveKeyException(String.format("Cannot get a master key with keyset ID %d.", Const.Data.MasterKeysetId)); } - // Reset shutdown timer on successful key retrieval - if (keyAvailabilityHandler != null) - keyAvailabilityHandler.accept(true); return key; } @@ -134,12 +119,8 @@ public KeysetKey getRefreshKey() { public KeysetKey getRefreshKey(Instant asOf) { KeysetKey key = this.keysetKeyStore.getSnapshot().getActiveKey(Const.Data.RefreshKeysetId, asOf); if (key == null) { - throw new NoActiveKeyException( - String.format("Cannot get a refresh key with keyset ID %d.", Const.Data.RefreshKeysetId)); + throw new NoActiveKeyException(String.format("Cannot get a refresh key with keyset ID %d.", Const.Data.RefreshKeysetId)); } - // Reset shutdown timer on successful key retrieval - if (keyAvailabilityHandler != null) - keyAvailabilityHandler.accept(true); return key; } From 5df3290ec3506495aae22f32e530df792fba1606 Mon Sep 17 00:00:00 2001 From: way zheng Date: Fri, 10 Oct 2025 12:13:23 -0700 Subject: [PATCH 15/28] clean up --- src/main/java/com/uid2/operator/Main.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index b6eaf0056..22521f150 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -61,7 +61,6 @@ import java.time.Duration; import java.time.Instant; import java.util.*; -import java.util.function.Consumer; import java.util.function.Supplier; import static com.uid2.operator.Const.Config.EnableRemoteConfigProp; @@ -442,8 +441,7 @@ private Future createStoreVerticles() throws Exception { } fs.add(createAndDeployRotatingStoreVerticle("auth", clientKeyProvider, "auth_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("keyset", keysetProvider, "keyset_refresh_ms")); - fs.add(createAndDeployRotatingStoreVerticle("keysetkey", keysetKeyStore, "keysetkey_refresh_ms", - this.shutdownHandler::handleKeysetKeyRefreshResponse)); + fs.add(createAndDeployRotatingStoreVerticle("keysetkey", keysetKeyStore, "keysetkey_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("salt", saltProvider, "salt_refresh_ms")); fs.add(createAndDeployCloudSyncStoreVerticle("optout", fsOptOut, optOutCloudSync)); CompositeFuture.all(fs).onComplete(ar -> { @@ -458,15 +456,9 @@ private Future createStoreVerticles() throws Exception { private Future createAndDeployRotatingStoreVerticle(String name, IMetadataVersionedStore store, String storeRefreshConfigMs) { - return createAndDeployRotatingStoreVerticle(name, store, storeRefreshConfigMs, null); - } - - private Future createAndDeployRotatingStoreVerticle(String name, IMetadataVersionedStore store, - String storeRefreshConfigMs, Consumer refreshCallback) { final int intervalMs = config.getInteger(storeRefreshConfigMs, 10000); - RotatingStoreVerticle rotatingStoreVerticle = new RotatingStoreVerticle(name, intervalMs, store, - refreshCallback); + RotatingStoreVerticle rotatingStoreVerticle = new RotatingStoreVerticle(name, intervalMs, store); return vertx.deployVerticle(rotatingStoreVerticle); } From 04309c41a077b8bd93e29f6837dffded0b66bfda Mon Sep 17 00:00:00 2001 From: way zheng Date: Fri, 10 Oct 2025 12:24:08 -0700 Subject: [PATCH 16/28] clean up --- src/main/java/com/uid2/operator/Main.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 22521f150..8d8e9e679 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -102,7 +102,6 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.appVersion = ApplicationVersion.load("uid2-operator", "uid2-shared", "uid2-attestation-api"); HealthManager.instance.registerGenericComponent( new PodTerminationMonitor(config.getInteger(Const.Config.PodTerminationCheckInterval, 3000))); - // allow to switch between in-mem optout file cache and on-disk file cache if (config.getBoolean(Const.Config.OptOutInMemCacheProp)) { this.fsLocal = new MemCachedStorage(); From 90dabf8e5f27070cc4b3556df5c5045c3b7116c5 Mon Sep 17 00:00:00 2001 From: way zheng Date: Fri, 10 Oct 2025 12:25:36 -0700 Subject: [PATCH 17/28] clean up --- src/main/java/com/uid2/operator/Main.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 8d8e9e679..316688a7b 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -100,8 +100,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.config = config; this.appVersion = ApplicationVersion.load("uid2-operator", "uid2-shared", "uid2-attestation-api"); - HealthManager.instance.registerGenericComponent( - new PodTerminationMonitor(config.getInteger(Const.Config.PodTerminationCheckInterval, 3000))); + HealthManager.instance.registerGenericComponent(new PodTerminationMonitor(config.getInteger(Const.Config.PodTerminationCheckInterval, 3000))); // allow to switch between in-mem optout file cache and on-disk file cache if (config.getBoolean(Const.Config.OptOutInMemCacheProp)) { this.fsLocal = new MemCachedStorage(); From aefafe739e7ad07a97739b946f9788017e4413ef Mon Sep 17 00:00:00 2001 From: way zheng Date: Fri, 10 Oct 2025 12:27:03 -0700 Subject: [PATCH 18/28] clean up --- .../com/uid2/operator/vertx/OperatorShutdownHandler.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java b/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java index d0edaa1ea..5ad3b88a5 100644 --- a/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java +++ b/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java @@ -43,14 +43,14 @@ public OperatorShutdownHandler(Duration attestShutdownWaitTime, Duration saltShu } public void handleSaltRetrievalResponse(Boolean expired) { - if (!expired) { + if(!expired) { saltFailureStartTime.set(null); } else { logSaltFailureAtInterval(); Instant t = saltFailureStartTime.get(); if (t == null) { saltFailureStartTime.set(clock.instant()); - } else if (Duration.between(t, clock.instant()).compareTo(this.saltShutdownWaitTime) > 0) { + } else if(Duration.between(t, clock.instant()).compareTo(this.saltShutdownWaitTime) > 0) { LOGGER.error("salts have been in expired state for too long. shutting down operator"); this.shutdownService.Shutdown(1); } @@ -59,7 +59,7 @@ public void handleSaltRetrievalResponse(Boolean expired) { public void logSaltFailureAtInterval() { Instant t = lastSaltFailureLogTime.get(); - if (t == null || clock.instant().isAfter(t.plus(SALT_FAILURE_LOG_INTERVAL_MINUTES, ChronoUnit.MINUTES))) { + if(t == null || clock.instant().isAfter(t.plus(SALT_FAILURE_LOG_INTERVAL_MINUTES, ChronoUnit.MINUTES))) { LOGGER.error("all salts are expired"); lastSaltFailureLogTime.set(Instant.now()); } From 6bf879fb73a491b465693c7b59047739157be78f Mon Sep 17 00:00:00 2001 From: way zheng Date: Fri, 10 Oct 2025 12:27:43 -0700 Subject: [PATCH 19/28] clean up --- .../java/com/uid2/operator/vertx/OperatorShutdownHandler.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java b/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java index 5ad3b88a5..1928db48e 100644 --- a/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java +++ b/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java @@ -101,8 +101,7 @@ public void logKeysetKeyFailureAtInterval() { public void handleAttestResponse(Pair response) { if (response.left() == AttestationResponseCode.AttestationFailure) { - LOGGER.error("core attestation failed with AttestationFailure, shutting down operator, core response: {}", - response.right()); + LOGGER.error("core attestation failed with AttestationFailure, shutting down operator, core response: {}", response.right()); this.shutdownService.Shutdown(1); } if (response.left() == AttestationResponseCode.Success) { From 8a968dd7d17233cf0518205efc13cfcb7cf2e683 Mon Sep 17 00:00:00 2001 From: way zheng Date: Fri, 10 Oct 2025 12:28:35 -0700 Subject: [PATCH 20/28] clean up --- src/main/java/com/uid2/operator/Main.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 316688a7b..1b65b8b55 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -101,6 +101,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.appVersion = ApplicationVersion.load("uid2-operator", "uid2-shared", "uid2-attestation-api"); HealthManager.instance.registerGenericComponent(new PodTerminationMonitor(config.getInteger(Const.Config.PodTerminationCheckInterval, 3000))); + // allow to switch between in-mem optout file cache and on-disk file cache if (config.getBoolean(Const.Config.OptOutInMemCacheProp)) { this.fsLocal = new MemCachedStorage(); @@ -126,8 +127,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception { DownloadCloudStorage fsStores; if (coreAttestUrl != null) { - var clients = createUidClients(this.vertx, coreAttestUrl, operatorKey, - this.shutdownHandler::handleAttestResponse); + var clients = createUidClients(this.vertx, coreAttestUrl, operatorKey, this.shutdownHandler::handleAttestResponse); UidCoreClient coreClient = clients.getKey(); UidOptOutClient optOutClient = clients.getValue(); fsStores = coreClient; From b3929cc1fc5e922a53e177dbe7c5887fb7c4c747 Mon Sep 17 00:00:00 2001 From: way zheng Date: Fri, 10 Oct 2025 12:29:40 -0700 Subject: [PATCH 21/28] clean up --- src/main/java/com/uid2/operator/Main.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 1b65b8b55..c2d80aa88 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -101,7 +101,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.appVersion = ApplicationVersion.load("uid2-operator", "uid2-shared", "uid2-attestation-api"); HealthManager.instance.registerGenericComponent(new PodTerminationMonitor(config.getInteger(Const.Config.PodTerminationCheckInterval, 3000))); - + // allow to switch between in-mem optout file cache and on-disk file cache if (config.getBoolean(Const.Config.OptOutInMemCacheProp)) { this.fsLocal = new MemCachedStorage(); @@ -157,12 +157,12 @@ public Main(Vertx vertx, JsonObject config) throws Exception { new GlobalScope(new CloudPath(cloudEncryptionKeyMdPath))); String keypairMdPath = this.config.getString(Const.Config.ClientSideKeypairsMetadataPathProp); - this.clientSideKeypairProvider = new RotatingClientSideKeypairStore(fsStores, - new GlobalScope(new CloudPath(keypairMdPath)), cloudEncryptionKeyProvider); + this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, new GlobalScope(new CloudPath(clientsMdPath)), + cloudEncryptionKeyProvider); String clientsMdPath = this.config.getString(Const.Config.ClientsMetadataPathProp); this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, new GlobalScope(new CloudPath(clientsMdPath)), - cloudEncryptionKeyProvider); + cloudEncryptionKeyProvider); String keysetKeysMdPath = this.config.getString(Const.Config.KeysetKeysMetadataPathProp); this.keysetKeyStore = new RotatingKeysetKeyStore(fsStores, new GlobalScope(new CloudPath(keysetKeysMdPath)), cloudEncryptionKeyProvider); From 93638ebdb4d82d27062ce197f1eccb6e2afe35f8 Mon Sep 17 00:00:00 2001 From: way zheng Date: Fri, 10 Oct 2025 12:30:50 -0700 Subject: [PATCH 22/28] clean up --- src/main/java/com/uid2/operator/Main.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index c2d80aa88..effcd5fc3 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -157,12 +157,11 @@ public Main(Vertx vertx, JsonObject config) throws Exception { new GlobalScope(new CloudPath(cloudEncryptionKeyMdPath))); String keypairMdPath = this.config.getString(Const.Config.ClientSideKeypairsMetadataPathProp); - this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, new GlobalScope(new CloudPath(clientsMdPath)), - cloudEncryptionKeyProvider); + this.clientSideKeypairProvider = new RotatingClientSideKeypairStore(fsStores, + new GlobalScope(new CloudPath(keypairMdPath)), cloudEncryptionKeyProvider); String clientsMdPath = this.config.getString(Const.Config.ClientsMetadataPathProp); - this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, - new GlobalScope(new CloudPath(clientsMdPath)), - cloudEncryptionKeyProvider); + this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, new GlobalScope(new CloudPath(clientsMdPath)), + cloudEncryptionKeyProvider); String keysetKeysMdPath = this.config.getString(Const.Config.KeysetKeysMetadataPathProp); this.keysetKeyStore = new RotatingKeysetKeyStore(fsStores, new GlobalScope(new CloudPath(keysetKeysMdPath)), cloudEncryptionKeyProvider); From defb64c16e5a1f43f3eb5116afe07750660eeeff Mon Sep 17 00:00:00 2001 From: way zheng Date: Fri, 10 Oct 2025 12:31:44 -0700 Subject: [PATCH 23/28] clean up --- src/main/java/com/uid2/operator/Main.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index effcd5fc3..cd0ef590f 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -174,18 +174,15 @@ public Main(Vertx vertx, JsonObject config) throws Exception { String sitesMdPath = this.config.getString(Const.Config.SitesMetadataPathProp); this.siteProvider = clientSideTokenGenerate ? new RotatingSiteStore(fsStores, new GlobalScope(new CloudPath(sitesMdPath)), - cloudEncryptionKeyProvider) + cloudEncryptionKeyProvider) : null; } else { String keypairMdPath = this.config.getString(Const.Config.ClientSideKeypairsMetadataPathProp); - this.clientSideKeypairProvider = new RotatingClientSideKeypairStore(fsStores, - new GlobalScope(new CloudPath(keypairMdPath))); + this.clientSideKeypairProvider = new RotatingClientSideKeypairStore(fsStores, new GlobalScope(new CloudPath(keypairMdPath))); String clientsMdPath = this.config.getString(Const.Config.ClientsMetadataPathProp); - this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, - new GlobalScope(new CloudPath(clientsMdPath))); + this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, new GlobalScope(new CloudPath(clientsMdPath))); String keysetKeysMdPath = this.config.getString(Const.Config.KeysetKeysMetadataPathProp); - this.keysetKeyStore = new RotatingKeysetKeyStore(fsStores, - new GlobalScope(new CloudPath(keysetKeysMdPath))); + this.keysetKeyStore = new RotatingKeysetKeyStore(fsStores, new GlobalScope(new CloudPath(keysetKeysMdPath))); String keysetMdPath = this.config.getString(Const.Config.KeysetsMetadataPathProp); this.keysetProvider = new RotatingKeysetProvider(fsStores, new GlobalScope(new CloudPath(keysetMdPath))); String saltsMdPath = this.config.getString(Const.Config.SaltsMetadataPathProp); From caa9a92c810a92722a87e137367c2440d506fec4 Mon Sep 17 00:00:00 2001 From: way zheng Date: Fri, 10 Oct 2025 12:32:19 -0700 Subject: [PATCH 24/28] clean up --- src/main/java/com/uid2/operator/Main.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index cd0ef590f..2896c08d1 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -188,13 +188,11 @@ public Main(Vertx vertx, JsonObject config) throws Exception { String saltsMdPath = this.config.getString(Const.Config.SaltsMetadataPathProp); this.saltProvider = new RotatingSaltProvider(fsStores, saltsMdPath); String sitesMdPath = this.config.getString(Const.Config.SitesMetadataPathProp); - this.siteProvider = clientSideTokenGenerate - ? new RotatingSiteStore(fsStores, new GlobalScope(new CloudPath(sitesMdPath))) - : null; + this.siteProvider = clientSideTokenGenerate ? new RotatingSiteStore(fsStores, new GlobalScope(new CloudPath(sitesMdPath))) : null; } this.optOutStore = new CloudSyncOptOutStore(vertx, fsLocal, this.config, operatorKey, Clock.systemUTC()); - + if (useRemoteConfig) { String configMdPath = this.config.getString(Const.Config.RuntimeConfigMetadataPathProp); this.configStore = new RuntimeConfigStore(fsStores, configMdPath); From 1edc65204d45aa5e32620b6a182bd958606abb73 Mon Sep 17 00:00:00 2001 From: way zheng Date: Fri, 10 Oct 2025 12:32:39 -0700 Subject: [PATCH 25/28] clean up --- src/main/java/com/uid2/operator/Main.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 2896c08d1..bd93097ed 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -204,8 +204,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception { String serviceMdPath = this.config.getString(Const.Config.ServiceMetadataPathProp); this.serviceProvider = new RotatingServiceStore(fsStores, new GlobalScope(new CloudPath(serviceMdPath))); String serviceLinkMdPath = this.config.getString(Const.Config.ServiceLinkMetadataPathProp); - this.serviceLinkProvider = new RotatingServiceLinkStore(fsStores, - new GlobalScope(new CloudPath(serviceLinkMdPath))); + this.serviceLinkProvider = new RotatingServiceLinkStore(fsStores, new GlobalScope(new CloudPath(serviceLinkMdPath))); } if (useStorageMock && coreAttestUrl == null) { From 35d554cf3c165cf301ddca3df81bd749afe53820 Mon Sep 17 00:00:00 2001 From: way zheng Date: Fri, 10 Oct 2025 12:34:27 -0700 Subject: [PATCH 26/28] clean up --- src/main/java/com/uid2/operator/Main.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index bd93097ed..971e06a94 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -249,8 +249,7 @@ private KeyManager getKeyManager() { } public static void recordStartupComplete() { - if (startupBeginTime == null) - return; + if (startupBeginTime == null) return; final Duration d = Duration.between(startupBeginTime, Instant.now()); Timer.builder("uid2_operator_startup_duration").register(globalRegistry).record(d); LOGGER.info("Startup in {} ms", d.toMillis()); @@ -259,12 +258,13 @@ public static void recordStartupComplete() { public static void main(String[] args) throws Exception { startupBeginTime = Instant.now(); - java.security.Security.setProperty("networkaddress.cache.ttl", "60"); + java.security.Security.setProperty("networkaddress.cache.ttl" , "60"); final String vertxConfigPath = System.getProperty(Const.Config.VERTX_CONFIG_PATH_PROP); if (vertxConfigPath != null) { LOGGER.info("Running CUSTOM CONFIG mode, config: {}", vertxConfigPath); - } else if (!Utils.isProductionEnvironment()) { + } + else if (!Utils.isProductionEnvironment()) { LOGGER.info("Running LOCAL DEBUG mode, config: {}", Const.Config.LOCAL_CONFIG_PATH); System.setProperty(Const.Config.VERTX_CONFIG_PATH_PROP, Const.Config.LOCAL_CONFIG_PATH); } else { @@ -309,8 +309,7 @@ private ICloudStorage configureCloudOptOutStore() { private ICloudStorage configureAttestedOptOutStore(UidOptOutClient optOutClient, String coreAttestUrl) { String optOutMdPath = this.config.getString(Const.Config.OptOutMetadataPathProp); LOGGER.info("OptOut stores- Using uid2-core attestation endpoint: " + coreAttestUrl); - return this - .wrapCloudStorageForOptOut(new OptOutCloudStorage(optOutClient, optOutMdPath, CloudUtils.defaultProxy)); + return this.wrapCloudStorageForOptOut(new OptOutCloudStorage(optOutClient, optOutMdPath, CloudUtils.defaultProxy)); } private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { From 260456ba8fb10baea7d537b5d5115a14f1f9ef3b Mon Sep 17 00:00:00 2001 From: way zheng Date: Fri, 10 Oct 2025 12:35:46 -0700 Subject: [PATCH 27/28] clean up --- src/main/java/com/uid2/operator/Main.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 971e06a94..46d9f6bc4 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -285,7 +285,7 @@ else if (!Utils.isProductionEnvironment()) { app.run(); } catch (Exception e) { LOGGER.error("Error: " + e.getMessage(), e); - ((LoggerContext) org.slf4j.LoggerFactory.getILoggerFactory()).stop(); // flush logs before shutdown + ((LoggerContext)org.slf4j.LoggerFactory.getILoggerFactory()).stop(); // flush logs before shutdown vertx.close(); System.exit(1); } @@ -339,11 +339,7 @@ private void run() throws Exception { this.createVertxEventLoopsMetric(); Supplier operatorVerticleSupplier = () -> { - UIDOperatorVerticle verticle = new UIDOperatorVerticle(configStore, config, this.clientSideTokenGenerate, - siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, - optOutStore, Clock.systemUTC(), _statsCollectorQueue, - new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), - this.shutdownHandler::handleSaltRetrievalResponse, this.uidInstanceIdProvider); + UIDOperatorVerticle verticle = new UIDOperatorVerticle(configStore, config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse, this.uidInstanceIdProvider); return verticle; }; From ca148faac72d146829b4e3e94caf8865373a4d2f Mon Sep 17 00:00:00 2001 From: way zheng Date: Mon, 13 Oct 2025 15:28:33 -0700 Subject: [PATCH 28/28] Implement keyset key fail-fast feature with 7-day timeout - Update Main.java to wire keysetkey verticle callback to shutdown handler - Add Consumer import and overloaded createAndDeployRotatingStoreVerticle method - Update KeyManager to accept keyAvailabilityHandler callback - Call handler on successful/failed key retrieval in getMasterKey/getRefreshKey - Maintains backward compatibility with existing constructors This enables the operator to shut down after 7 days of consecutive keyset key sync failures, allowing Kubernetes to restart and potentially recover. --- src/main/java/com/uid2/operator/Main.java | 14 ++++-- .../com/uid2/operator/model/KeyManager.java | 45 +++++++++++++++---- 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 46d9f6bc4..01b3ae894 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -61,6 +61,7 @@ import java.time.Duration; import java.time.Instant; import java.util.*; +import java.util.function.Consumer; import java.util.function.Supplier; import static com.uid2.operator.Const.Config.EnableRemoteConfigProp; @@ -245,7 +246,8 @@ public Main(Vertx vertx, JsonObject config) throws Exception { } private KeyManager getKeyManager() { - return new KeyManager(this.keysetKeyStore, this.keysetProvider); + return new KeyManager(this.keysetKeyStore, this.keysetProvider, + hasKeys -> shutdownHandler.handleKeysetKeyRefreshResponse(hasKeys)); } public static void recordStartupComplete() { @@ -427,7 +429,8 @@ private Future createStoreVerticles() throws Exception { } fs.add(createAndDeployRotatingStoreVerticle("auth", clientKeyProvider, "auth_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("keyset", keysetProvider, "keyset_refresh_ms")); - fs.add(createAndDeployRotatingStoreVerticle("keysetkey", keysetKeyStore, "keysetkey_refresh_ms")); + fs.add(createAndDeployRotatingStoreVerticle("keysetkey", keysetKeyStore, "keysetkey_refresh_ms", + this.shutdownHandler::handleKeysetKeyRefreshResponse)); fs.add(createAndDeployRotatingStoreVerticle("salt", saltProvider, "salt_refresh_ms")); fs.add(createAndDeployCloudSyncStoreVerticle("optout", fsOptOut, optOutCloudSync)); CompositeFuture.all(fs).onComplete(ar -> { @@ -442,9 +445,14 @@ private Future createStoreVerticles() throws Exception { private Future createAndDeployRotatingStoreVerticle(String name, IMetadataVersionedStore store, String storeRefreshConfigMs) { + return createAndDeployRotatingStoreVerticle(name, store, storeRefreshConfigMs, null); + } + + private Future createAndDeployRotatingStoreVerticle(String name, IMetadataVersionedStore store, + String storeRefreshConfigMs, Consumer refreshCallback) { final int intervalMs = config.getInteger(storeRefreshConfigMs, 10000); - RotatingStoreVerticle rotatingStoreVerticle = new RotatingStoreVerticle(name, intervalMs, store); + RotatingStoreVerticle rotatingStoreVerticle = new RotatingStoreVerticle(name, intervalMs, store, refreshCallback); return vertx.deployVerticle(rotatingStoreVerticle); } diff --git a/src/main/java/com/uid2/operator/model/KeyManager.java b/src/main/java/com/uid2/operator/model/KeyManager.java index 19bae8d07..b03c1b4f4 100644 --- a/src/main/java/com/uid2/operator/model/KeyManager.java +++ b/src/main/java/com/uid2/operator/model/KeyManager.java @@ -13,16 +13,23 @@ import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import java.util.stream.Collectors; public class KeyManager { private static final Logger LOGGER = LoggerFactory.getLogger(UIDOperatorVerticle.class); private final IKeysetKeyStore keysetKeyStore; private final RotatingKeysetProvider keysetProvider; + private final Consumer keyAvailabilityHandler; public KeyManager(IKeysetKeyStore keysetKeyStore, RotatingKeysetProvider keysetProvider) { + this(keysetKeyStore, keysetProvider, null); + } + + public KeyManager(IKeysetKeyStore keysetKeyStore, RotatingKeysetProvider keysetProvider, Consumer keyAvailabilityHandler) { this.keysetKeyStore = keysetKeyStore; this.keysetProvider = keysetProvider; + this.keyAvailabilityHandler = keyAvailabilityHandler; } public KeyManagerSnapshot getKeyManagerSnapshot(int siteId) { @@ -105,11 +112,22 @@ public KeysetKey getMasterKey() { } public KeysetKey getMasterKey(Instant asOf) { - KeysetKey key = this.keysetKeyStore.getSnapshot().getActiveKey(Const.Data.MasterKeysetId, asOf); - if (key == null) { - throw new NoActiveKeyException(String.format("Cannot get a master key with keyset ID %d.", Const.Data.MasterKeysetId)); + try { + KeysetKey key = this.keysetKeyStore.getSnapshot().getActiveKey(Const.Data.MasterKeysetId, asOf); + if (key == null) { + throw new NoActiveKeyException(String.format("Cannot get a master key with keyset ID %d.", Const.Data.MasterKeysetId)); + } + // Reset timer on successful key retrieval + if (keyAvailabilityHandler != null) { + keyAvailabilityHandler.accept(true); + } + return key; + } catch (NoActiveKeyException e) { + if (keyAvailabilityHandler != null) { + keyAvailabilityHandler.accept(false); + } + throw e; } - return key; } public KeysetKey getRefreshKey() { @@ -117,11 +135,22 @@ public KeysetKey getRefreshKey() { } public KeysetKey getRefreshKey(Instant asOf) { - KeysetKey key = this.keysetKeyStore.getSnapshot().getActiveKey(Const.Data.RefreshKeysetId, asOf); - if (key == null) { - throw new NoActiveKeyException(String.format("Cannot get a refresh key with keyset ID %d.", Const.Data.RefreshKeysetId)); + try { + KeysetKey key = this.keysetKeyStore.getSnapshot().getActiveKey(Const.Data.RefreshKeysetId, asOf); + if (key == null) { + throw new NoActiveKeyException(String.format("Cannot get a refresh key with keyset ID %d.", Const.Data.RefreshKeysetId)); + } + // Reset timer on successful key retrieval + if (keyAvailabilityHandler != null) { + keyAvailabilityHandler.accept(true); + } + return key; + } catch (NoActiveKeyException e) { + if (keyAvailabilityHandler != null) { + keyAvailabilityHandler.accept(false); + } + throw e; } - return key; } public static class NoActiveKeyException extends RuntimeException {