Merge pull request #209 from IABTechLab/ian-UID2-1394-update-attest-failure-reaction

Ian UI d2 1394 update attest failure reaction

Author: Ian-Nara

pom.xml

@@ -22,7 +22,7 @@
         <enclave-aws.version>1.1.0</enclave-aws.version>
         <enclave-azure.version>1.5.0-115595d597</enclave-azure.version>
         <enclave-gcp.version>1.4.2-dd1920710d</enclave-gcp.version>
-        <uid2-shared.version>5.17.0-7c1a08e2f9</uid2-shared.version>
+        <uid2-shared.version>5.19.0-0a359c5a31</uid2-shared.version>
         <image.version>${project.version}</image.version>
     </properties>
 

src/main/java/com/uid2/operator/Main.java

@@ -8,9 +8,9 @@
 import com.uid2.operator.monitoring.OperatorMetrics;
 import com.uid2.operator.monitoring.StatsCollectorVerticle;
 import com.uid2.operator.service.SecureLinkValidatorService;
+import com.uid2.operator.vertx.OperatorShutdownHandler;
 import com.uid2.operator.store.CloudSyncOptOutStore;
 import com.uid2.operator.store.OptOutCloudStorage;
-import com.uid2.operator.vertx.OperatorDisableHandler;
 import com.uid2.operator.vertx.UIDOperatorVerticle;
 import com.uid2.shared.ApplicationVersion;
 import com.uid2.shared.Utils;
@@ -42,6 +42,7 @@
 import io.vertx.micrometer.backends.BackendRegistries;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.utils.Pair;
 
 import javax.management.*;
 import java.lang.management.ManagementFactory;
@@ -69,11 +70,11 @@ public class Main {
     private final RotatingClientSideKeypairStore clientSideKeypairProvider;
     private final RotatingSaltProvider saltProvider;
     private final CloudSyncOptOutStore optOutStore;
+    private OperatorShutdownHandler shutdownHandler = null;
     private final OperatorMetrics metrics;
     private final boolean clientSideTokenGenerate;
     private final boolean validateServiceLinks;
     private IStatsCollectorQueue _statsCollectorQueue;
-    private OperatorDisableHandler disableHandler = null;
     private RotatingServiceStore serviceProvider;
     private RotatingServiceLinkStore serviceLinkProvider;
 
@@ -101,10 +102,9 @@ public Main(Vertx vertx, JsonObject config) throws Exception {
 
         DownloadCloudStorage fsStores;
         if (coreAttestUrl != null) {
-            Duration disableWaitTime = Duration.ofHours(this.config.getInteger(Const.Config.FailureShutdownWaitHoursProp, 120));
-            this.disableHandler = new OperatorDisableHandler(disableWaitTime, Clock.systemUTC());
+            this.shutdownHandler = new OperatorShutdownHandler(Duration.ofHours(12), Clock.systemUTC());
 
-            var clients = createUidClients(this.vertx, coreAttestUrl, operatorKey, this.disableHandler::handleResponseStatus);
+            var clients = createUidClients(this.vertx, coreAttestUrl, operatorKey, this.shutdownHandler::handleResponse);
             UidCoreClient coreClient = clients.getKey();
             UidOptOutClient optOutClient = clients.getValue();
             fsStores = coreClient;
@@ -252,8 +252,6 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) {
     private void run() throws Exception {
         Supplier<Verticle> operatorVerticleSupplier = () -> {
             UIDOperatorVerticle verticle = new UIDOperatorVerticle(config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider));
-            if (this.disableHandler != null)
-                verticle.setDisableHandler(this.disableHandler);
             return verticle;
         };
 
@@ -453,15 +451,15 @@ public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticC
                 .register(globalRegistry);
     }
 
-    private Map.Entry<UidCoreClient, UidOptOutClient> createUidClients(Vertx vertx, String attestationUrl, String clientApiToken, Handler<Integer> responseWatcher) throws Exception {
+    private Map.Entry<UidCoreClient, UidOptOutClient> createUidClients(Vertx vertx, String attestationUrl, String clientApiToken, Handler<Pair<Integer, String>> responseWatcher) throws Exception {
         AttestationTokenRetriever attestationTokenRetriever = getAttestationTokenRetriever(vertx, attestationUrl, clientApiToken, responseWatcher);
         Boolean enforceHttps = this.config.getBoolean("enforce_https", true);
         UidCoreClient coreClient = new UidCoreClient(clientApiToken, CloudUtils.defaultProxy, enforceHttps, attestationTokenRetriever);
         UidOptOutClient optOutClient = new UidOptOutClient(clientApiToken, CloudUtils.defaultProxy, enforceHttps, attestationTokenRetriever);
         return new AbstractMap.SimpleEntry<>(coreClient, optOutClient);
     }
 
-    private AttestationTokenRetriever getAttestationTokenRetriever(Vertx vertx, String attestationUrl, String clientApiToken, Handler<Integer> responseWatcher) throws Exception {
+    private AttestationTokenRetriever getAttestationTokenRetriever(Vertx vertx, String attestationUrl, String clientApiToken, Handler<Pair<Integer, String>> responseWatcher) throws Exception {
         String enclavePlatform = this.config.getString("enclave_platform");
         if (Strings.isNullOrEmpty(enclavePlatform)) {
             return new AttestationTokenRetriever(vertx, attestationUrl, clientApiToken, this.appVersion, new NoAttestationProvider(), responseWatcher, CloudUtils.defaultProxy);

src/main/java/com/uid2/operator/vertx/OperatorDisableHandler.java

@@ -0,0 +1,41 @@
+package com.uid2.operator.vertx;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.utils.Pair;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class OperatorShutdownHandler {
+    private static final Logger LOGGER = LoggerFactory.getLogger(OperatorShutdownHandler.class);
+    private final Duration shutdownWaitTime;
+    private final AtomicReference<Instant> failureStartTime = new AtomicReference<>(null);
+    private final Clock clock;
+
+    public OperatorShutdownHandler(Duration shutdownWaitTime, Clock clock) {
+        this.shutdownWaitTime = shutdownWaitTime;
+        this.clock = clock;
+    }
+
+
+    public void handleResponse(Pair<Integer, String> response) {
+        if (response.left() == 401) {
+            LOGGER.error("core attestation failed with 401, shutting down operator, core response: " + response.right());
+            System.exit(1);
+        }
+        if (response.left() == 200) {
+            failureStartTime.set(null);
+        } else {
+            Instant t = failureStartTime.get();
+            if (t == null) {
+                failureStartTime.set(clock.instant());
+            } else if (Duration.between(t, clock.instant()).compareTo(this.shutdownWaitTime) > 0) {
+                LOGGER.error("core attestation has been in failed state for too long. shutting down operator");
+                System.exit(1);
+            }
+        }
+    }
+}

src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java

@@ -98,7 +98,6 @@ public class UIDOperatorVerticle extends AbstractVerticle {
     private final Map<String, Counter> _identityMapRequestWithUnmapped = new HashMap<>();
     private final IdentityScope identityScope;
     private final V2PayloadHandler v2PayloadHandler;
-    private Handler<RoutingContext> disableHandler = null;
     private final boolean phoneSupport;
     private final int tcfVendorId;
     private final IStatsCollectorQueue _statsCollectorQueue;
@@ -178,17 +177,9 @@ public void start(Promise<Void> startPromise) throws Exception {
 
     }
 
-    public void setDisableHandler(Handler<RoutingContext> h) {
-        this.disableHandler = h;
-    }
-
     private Router createRoutesSetup() throws IOException {
         final Router router = Router.router(vertx);
 
-        if (this.disableHandler != null) {
-            router.route().handler(this.disableHandler);
-        }
-
         router.allowForward(AllowForwardHeaders.X_FORWARD);
         router.route().handler(new RequestCapturingHandler());
         router.route().handler(new ClientVersionCapturingHandler("static/js", "*.js"));

src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java

@@ -0,0 +1,128 @@
+package com.uid2.operator;
+
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.read.ListAppender;
+import com.uid2.operator.vertx.OperatorShutdownHandler;
+import io.vertx.core.Vertx;
+import io.vertx.junit5.VertxExtension;
+import io.vertx.junit5.VertxTestContext;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.utils.Pair;
+
+import java.security.Permission;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(VertxExtension.class)
+public class OperatorShutdownHandlerTest {
+
+    private AutoCloseable mocks;
+    @Mock private Clock clock;
+    private OperatorShutdownHandler operatorShutdownHandler;
+
+    class NoExitSecurityManager extends SecurityManager {
+        @Override
+        public void checkPermission(Permission perm) { }
+
+        @Override
+        public void checkExit(int status) {
+            super.checkExit(status);
+            throw new RuntimeException(String.valueOf(status));
+        }
+    }
+
+    @BeforeEach
+    void beforeEach() {
+        mocks = MockitoAnnotations.openMocks(this);
+        when(clock.instant()).thenAnswer(i -> Instant.now());
+        this.operatorShutdownHandler = new OperatorShutdownHandler(Duration.ofHours(12), clock);
+    }
+
+    @AfterEach
+    void afterEach() throws Exception {
+        mocks.close();
+    }
+
+    @Test
+    void shutdownOn401(Vertx vertx, VertxTestContext testContext) {
+        SecurityManager origSecurityManager = System.getSecurityManager();
+        try {
+            System.setSecurityManager(new NoExitSecurityManager());
+
+            ListAppender<ILoggingEvent> logWatcher = new ListAppender<>();
+            logWatcher.start();
+            ((Logger) LoggerFactory.getLogger(OperatorShutdownHandler.class)).addAppender(logWatcher);
+
+            // Revoke auth
+            try {
+                this.operatorShutdownHandler.handleResponse(Pair.of(401, "Unauthorized"));
+            } catch (RuntimeException e) {
+                Assertions.assertTrue(logWatcher.list.get(0).getFormattedMessage().contains("core attestation failed with 401, shutting down operator, core response: "));
+                testContext.completeNow();
+            }
+        } finally {
+            System.setSecurityManager(origSecurityManager);
+        }
+    }
+
+    @Test
+    void shutdownOnFailedTooLong(Vertx vertx, VertxTestContext testContext) {
+        SecurityManager origSecurityManager = System.getSecurityManager();
+        try {
+            System.setSecurityManager(new NoExitSecurityManager());
+
+            ListAppender<ILoggingEvent> logWatcher = new ListAppender<>();
+            logWatcher.start();
+            ((Logger) LoggerFactory.getLogger(OperatorShutdownHandler.class)).addAppender(logWatcher);
+
+            this.operatorShutdownHandler.handleResponse(Pair.of(500, ""));
+
+            when(clock.instant()).thenAnswer(i -> Instant.now().plus(12, ChronoUnit.HOURS).plusSeconds(60));
+            try {
+                this.operatorShutdownHandler.handleResponse(Pair.of(500, ""));
+            } catch (RuntimeException e) {
+                Assertions.assertTrue(logWatcher.list.get(0).getFormattedMessage().contains("core attestation has been in failed state for too long. shutting down operator"));
+                testContext.completeNow();
+            }
+        } finally {
+            System.setSecurityManager(origSecurityManager);
+        }
+    }
+
+    @Test
+    void attestRecoverOnSuccess(Vertx vertx, VertxTestContext testContext) {
+        SecurityManager origSecurityManager = System.getSecurityManager();
+        try {
+            System.setSecurityManager(new NoExitSecurityManager());
+
+            ListAppender<ILoggingEvent> logWatcher = new ListAppender<>();
+            logWatcher.start();
+            ((Logger) LoggerFactory.getLogger(OperatorShutdownHandler.class)).addAppender(logWatcher);
+
+            this.operatorShutdownHandler.handleResponse(Pair.of(500, ""));
+            when(clock.instant()).thenAnswer(i -> Instant.now().plus(6, ChronoUnit.HOURS));
+            this.operatorShutdownHandler.handleResponse(Pair.of(200, ""));
+
+            when(clock.instant()).thenAnswer(i -> Instant.now().plus(12, ChronoUnit.HOURS));
+            assertDoesNotThrow(() -> {
+                this.operatorShutdownHandler.handleResponse(Pair.of(500, ""));
+            });
+            testContext.completeNow();
+        } finally {
+            System.setSecurityManager(origSecurityManager);
+        }
+    }
+}