Added executeBlocking() on refresh calls and adjusted instance count/pool size

conf/default-config.json

@@ -1,4 +1,5 @@
 {
+    "service_instances": 3,
     "pre_signed_url_expiry": 1800,
     "core_s3_bucket": null,
     "aws_region": null,

conf/integ-config.json

@@ -1,4 +1,5 @@
 {
+    "service_instances": 3,
     "aws_region": "<aws_region_for_s3_bucket>",
     "core_s3_bucket": "<s3_bucket_for_stores>",
     "aws_access_key_id": "<your_aws_key_here>",

conf/local-config.json

@@ -1,4 +1,5 @@
 {
+    "service_instances": 1,
     "aws_s3_endpoint": "http://localstack:5001",
     "enable_test_endpoints": true,
     "storage_mock": true,

conf/local-e2e-config.json

@@ -1,4 +1,5 @@
 {
+    "service_instances": 1,
     "enable_test_endpoints": true,
     "storage_mock": false,
     "aws_s3_endpoint": "http://localhost:5001",

conf/local-e2e-docker-config.json

@@ -1,4 +1,5 @@
 {
+  "service_instances": 1,
   "enable_test_endpoints": true,
   "storage_mock": false,
   "aws_s3_endpoint": "http://localstack:5001",

src/main/java/com/uid2/core/Const.java

@@ -12,11 +12,12 @@ public static class RoutingContextData {
         public static final String ATTESTATION_FAILURE_DATA_PROP = "attestation-failure-data";
     }
 
-    public class Config extends com.uid2.shared.Const.Config {
+    public static class Config extends com.uid2.shared.Const.Config {
+        public static final String ServiceInstancesProp = "service_instances";
         public static final String KmsAccessKeyIdProp = "kms_aws_access_key_id";
         public static final String KmsSecretAccessKeyProp = "kms_aws_secret_access_key";
         public static final String KmsEndpointProp = "kms_aws_endpoint";
     }
 
     public static final String OPERATOR_CONFIG_PATH = "conf/operator/operator-config.json";
-}
+}

src/main/java/com/uid2/core/Main.java

@@ -16,7 +16,6 @@
 import com.uid2.shared.auth.EnclaveIdentifierProvider;
 import com.uid2.shared.auth.RotatingOperatorKeyProvider;
 import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
-import com.uid2.shared.model.CloudEncryptionKey;
 import com.uid2.shared.cloud.CloudUtils;
 import com.uid2.shared.cloud.EmbeddedResourceStorage;
 import com.uid2.shared.cloud.ICloudStorage;
@@ -38,31 +37,32 @@
 import io.vertx.core.VertxOptions;
 import io.vertx.core.file.FileSystem;
 import io.vertx.core.http.HttpServerOptions;
-import io.vertx.core.http.impl.HttpUtils;
 import io.vertx.core.json.JsonObject;
 import io.vertx.micrometer.Label;
 import io.vertx.micrometer.MetricsDomain;
 import io.vertx.micrometer.MicrometerMetricsOptions;
 import io.vertx.micrometer.VertxPrometheusOptions;
 import io.vertx.micrometer.backends.BackendRegistries;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import javax.management.*;
 import java.lang.management.ManagementFactory;
 import java.util.*;
 
 public class Main {
-
-    private static final int vertxServiceInstances = 1;
+    private static final Logger LOGGER = LoggerFactory.getLogger(CoreVerticle.class);
+    private static final int VERTX_WORKER_POOL_SIZE = 1000; // Cannot set this in config file because it's needed on Vertx init
 
     public static void main(String[] args) {
         final String vertxConfigPath = System.getProperty(Const.Config.VERTX_CONFIG_PATH_PROP);
         if (vertxConfigPath != null) {
-            System.out.format("Running CUSTOM CONFIG mode, config: %s\n", vertxConfigPath);
+            LOGGER.info("Running CUSTOM CONFIG mode, config: {}", vertxConfigPath);
         } else if (!Utils.isProductionEnvironment()) {
-            System.out.format("Running LOCAL DEBUG mode, config: %s\n", Const.Config.LOCAL_CONFIG_PATH);
+            LOGGER.info("Running LOCAL DEBUG mode, config: {}", Const.Config.LOCAL_CONFIG_PATH);
             System.setProperty(Const.Config.VERTX_CONFIG_PATH_PROP, Const.Config.LOCAL_CONFIG_PATH);
         } else {
-            System.out.format("Running PRODUCTION mode, config: %s\n", Const.Config.OVERRIDE_CONFIG_PATH);
+            LOGGER.info("Running PRODUCTION mode, config: {}", Const.Config.OVERRIDE_CONFIG_PATH);
         }
 
         // create AdminApi instance
@@ -71,7 +71,7 @@ public static void main(String[] args) {
             MBeanServer server = ManagementFactory.getPlatformMBeanServer();
             server.registerMBean(AdminApi.instance, objectName);
         } catch (InstanceAlreadyExistsException | MBeanRegistrationException | NotCompliantMBeanException | MalformedObjectNameException e) {
-            System.err.format("%s", e.getMessage());
+            LOGGER.error(e.getMessage());
             System.exit(-1);
         }
 
@@ -81,17 +81,9 @@ public static void main(String[] args) {
         VertxOptions vertxOptions = getVertxOptions(metricOptions);
         Vertx vertx = Vertx.vertx(vertxOptions);
 
-        /*
-        CommandLine commandLine = parseArgs(args);
-        String configPath = commandLine.getOptionValue("config").toString();
-        String secretsPath = commandLine.getOptionValue("secrets").toString();
-        ConfigStore.Global.load(configPath);
-        SecretStore.Global.load(secretsPath);
-         */
-
         VertxUtils.createConfigRetriever(vertx).getConfig(ar -> {
             if (ar.failed()) {
-                System.out.println("failed to load config: " + ar.cause().toString());
+                LOGGER.error("failed to load config: {}", ar.cause().toString());
                 System.exit(-1);
             }
 
@@ -100,7 +92,7 @@ public static void main(String[] args) {
             SecretStore.Global.load(config);
 
             boolean useStorageMock = Optional.ofNullable(ConfigStore.Global.getBoolean("storage_mock")).orElse(false);
-            ICloudStorage cloudStorage = null;
+            ICloudStorage cloudStorage;
             if (useStorageMock) {
                 cloudStorage = new EmbeddedResourceStorage(Main.class).withUrlPrefix(ConfigStore.Global.getOrDefault("storage_mock_url_prefix", ""));
             } else {
@@ -110,24 +102,25 @@ public static void main(String[] args) {
                 cloudStorage.setPreSignedUrlExpiry(expiryInSeconds);
             }
 
-            RotatingStoreVerticle enclaveRotatingVerticle = null;
-            RotatingStoreVerticle operatorRotatingVerticle = null;
-            RotatingStoreVerticle cloudEncryptionKeyRotatingVerticle = null;
-            CoreVerticle coreVerticle = null;
             try {
+                createVertxMetrics();
+
                 CloudPath operatorMetadataPath = new CloudPath(config.getString(Const.Config.OperatorsMetadataPathProp));
                 GlobalScope operatorScope = new GlobalScope(operatorMetadataPath);
                 RotatingOperatorKeyProvider operatorKeyProvider = new RotatingOperatorKeyProvider(cloudStorage, cloudStorage, operatorScope);
-                operatorRotatingVerticle = new RotatingStoreVerticle("operators", 60000, operatorKeyProvider);
+                RotatingStoreVerticle operatorRotatingVerticle = new RotatingStoreVerticle("operators", 60000, operatorKeyProvider);
+                vertx.deployVerticle(operatorRotatingVerticle);
 
                 String enclaveMetadataPath = SecretStore.Global.get(EnclaveIdentifierProvider.ENCLAVES_METADATA_PATH);
                 EnclaveIdentifierProvider enclaveIdProvider = new EnclaveIdentifierProvider(cloudStorage, enclaveMetadataPath);
-                enclaveRotatingVerticle = new RotatingStoreVerticle("enclaves", 60000, enclaveIdProvider);
+                RotatingStoreVerticle enclaveRotatingVerticle = new RotatingStoreVerticle("enclaves", 60000, enclaveIdProvider);
+                vertx.deployVerticle(enclaveRotatingVerticle);
 
                 CloudPath cloudEncryptionKeyMetadataPath = new CloudPath(config.getString(Const.Config.CloudEncryptionKeysMetadataPathProp));
                 GlobalScope cloudEncryptionKeyScope = new GlobalScope(cloudEncryptionKeyMetadataPath);
                 RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider = new RotatingCloudEncryptionKeyProvider(cloudStorage, cloudEncryptionKeyScope);
-                cloudEncryptionKeyRotatingVerticle = new RotatingStoreVerticle("cloud_encryption_keys", 60000, cloudEncryptionKeyProvider);
+                RotatingStoreVerticle cloudEncryptionKeyRotatingVerticle = new RotatingStoreVerticle("cloud_encryption_keys", 60000, cloudEncryptionKeyProvider);
+                vertx.deployVerticle(cloudEncryptionKeyRotatingVerticle);
 
                 String corePublicUrl = ConfigStore.Global.get(Const.Config.CorePublicUrlProp);
                 AttestationService attestationService = new AttestationService()
@@ -140,7 +133,7 @@ public static void main(String[] args) {
                     Set<String> enclaveParams = null;
                     String params = config.getString(Const.Config.GcpEnclaveParamsProp);
                     if (params != null) {
-                        enclaveParams = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(params.split(","))));
+                        enclaveParams = Set.of(params.split(","));
                     }
 
                     // enable gcp-vmid attestation if requested
@@ -155,7 +148,7 @@ public static void main(String[] args) {
                 attestationService.with("gcp-oidc", new GcpOidcCoreAttestationService(corePublicUrl));
 
                 OperatorJWTTokenProvider operatorJWTTokenProvider = new OperatorJWTTokenProvider(config);
-                
+
                 IAttestationTokenService attestationTokenService = new AttestationTokenService(
                         SecretStore.Global.get(Constants.AttestationEncryptionKeyName),
                         SecretStore.Global.get(Constants.AttestationEncryptionSaltName),
@@ -164,29 +157,28 @@ public static void main(String[] args) {
 
                 JwtService jwtService = new JwtService(config);
                 FileSystem fileSystem = vertx.fileSystem();
-                coreVerticle = new CoreVerticle(cloudStorage, operatorKeyProvider, attestationService, attestationTokenService, enclaveIdProvider, operatorJWTTokenProvider, jwtService, cloudEncryptionKeyProvider, fileSystem);
+
+                vertx.deployVerticle(() -> {
+                    try {
+                        return new CoreVerticle(cloudStorage, operatorKeyProvider, attestationService, attestationTokenService, enclaveIdProvider, operatorJWTTokenProvider, jwtService, cloudEncryptionKeyProvider, fileSystem);
+                    } catch (Exception e) {
+                        LOGGER.error("failed to deploy core verticle: {}", e.getMessage());
+                        System.exit(-1);
+                        return null;
+                    }
+                }, new DeploymentOptions().setInstances(ConfigStore.Global.getInteger(com.uid2.core.Const.Config.ServiceInstancesProp)));
             } catch (Exception e) {
-                System.out.println("failed to initialize core verticle: " + e.getMessage());
+                LOGGER.error("failed to initialize core verticle: {}", e.getMessage());
                 System.exit(-1);
             }
-
-            createVertxInstancesMetric();
-            createVertxEventLoopsMetric();
-
-            vertx.deployVerticle(enclaveRotatingVerticle);
-            vertx.deployVerticle(operatorRotatingVerticle);
-            vertx.deployVerticle(cloudEncryptionKeyRotatingVerticle);
-            vertx.deployVerticle(coreVerticle, new DeploymentOptions().setInstances(vertxServiceInstances));
         });
     }
 
     private static void setupMetrics(MicrometerMetricsOptions metricOptions) {
         BackendRegistries.setupBackend(metricOptions, null);
 
         // As of now default backend registry should have been created
-        if (BackendRegistries.getDefaultNow() instanceof PrometheusMeterRegistry) {
-            PrometheusMeterRegistry prometheusRegistry = (PrometheusMeterRegistry) BackendRegistries.getDefaultNow();
-
+        if (BackendRegistries.getDefaultNow() instanceof PrometheusMeterRegistry prometheusRegistry) {
             // see also https://micrometer.io/docs/registry/prometheus
             prometheusRegistry.config()
                     // providing common renaming for prometheus metric, e.g. "hello.world" to "hello_world"
@@ -195,8 +187,8 @@ private static void setupMetrics(MicrometerMetricsOptions metricOptions) {
                             actualPath -> HTTPPathMetricFilter.filterPath(actualPath, Endpoints.pathSet())))
                     // Don't record metrics for 404s.
                     .meterFilter(MeterFilter.deny(id ->
-                        id.getName().startsWith(MetricsDomain.HTTP_SERVER.getPrefix()) &&
-                        Objects.equals(id.getTag(Label.HTTP_CODE.toString()), "404")))
+                            id.getName().startsWith(MetricsDomain.HTTP_SERVER.getPrefix()) &&
+                                    Objects.equals(id.getTag(Label.HTTP_CODE.toString()), "404")))
                     // adding common labels
                     .commonTags("application", "uid2-core");
 
@@ -206,50 +198,35 @@ private static void setupMetrics(MicrometerMetricsOptions metricOptions) {
 
         // retrieve image version (will unify when uid2-common is used)
         String version = Optional.ofNullable(System.getenv("IMAGE_VERSION")).orElse("unknown");
-        Gauge appStatus = Gauge
-                .builder("app.status", () -> 1)
+        Gauge.builder("app.status", () -> 1)
                 .description("application version and status")
                 .tags("version", version)
                 .register(Metrics.globalRegistry);
     }
 
-    private static void createVertxInstancesMetric() {
-        Gauge.builder("uid2.vertx_service_instances", () -> vertxServiceInstances)
+    private static void createVertxMetrics() {
+        Gauge.builder("uid2.vertx_service_instances", () -> ConfigStore.Global.getInteger(com.uid2.core.Const.Config.ServiceInstancesProp))
                 .description("gauge for number of vertx service instances requested")
                 .register(Metrics.globalRegistry);
-    }
 
-    private static void createVertxEventLoopsMetric() {
+        Gauge.builder("uid2.vertx_worker_pool_size", () -> VERTX_WORKER_POOL_SIZE)
+                .description("gauge for vertx worker pool size requested")
+                .register(Metrics.globalRegistry);
+
         Gauge.builder("uid2.vertx_event_loop_threads", () -> VertxOptions.DEFAULT_EVENT_LOOP_POOL_SIZE)
                 .description("gauge for number of vertx event loop threads")
                 .register(Metrics.globalRegistry);
     }
 
-
-    /*
-    private static CommandLine parseArgs(String[] args) {
-        final CLI cli = CLI.create("uid2-core")
-            .setSummary("run uid2 core service")
-            .addOption(new Option()
-                .setLongName("config")
-                .setDescription("path to configuration file")
-                .setRequired(true))
-            .addOption(new Option()
-                .setLongName("secrets")
-                .setDescription("path to secrets file")
-                .setRequired(true));
-        return cli.parse(Arrays.asList(args));
-    }
-     */
-
     private static VertxOptions getVertxOptions(MicrometerMetricsOptions metricOptions) {
         final int threadBlockedCheckInterval = Utils.isProductionEnvironment()
                 ? 60 * 1000
                 : 3600 * 1000;
 
         return new VertxOptions()
                 .setMetricsOptions(metricOptions)
-                .setBlockedThreadCheckInterval(threadBlockedCheckInterval);
+                .setBlockedThreadCheckInterval(threadBlockedCheckInterval)
+                .setWorkerPoolSize(VERTX_WORKER_POOL_SIZE);
     }
 
     private static MicrometerMetricsOptions getMetricOptions(VertxPrometheusOptions promOptions) {

src/main/java/com/uid2/core/service/ClientMetadataProvider.java

@@ -1,44 +1,18 @@
 package com.uid2.core.service;
 
-import com.uid2.core.model.SecretStore;
 import com.uid2.core.util.OperatorInfo;
-import com.uid2.shared.auth.OperatorType;
 import com.uid2.shared.cloud.ICloudStorage;
-import io.vertx.core.json.Json;
-import io.vertx.core.json.JsonObject;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-import static com.uid2.core.util.MetadataHelper.getMetadataPathName;
-import static com.uid2.core.util.MetadataHelper.readToEndAsString;
-
-public class ClientMetadataProvider implements IClientMetadataProvider {
-
-    public static final String ClientsMetadataPathName = "clients_metadata_path";
-
-    private final ICloudStorage metadataStreamProvider;
-    private final ICloudStorage downloadUrlGenerator;
-
-    @Override
-    public String getMetadata(OperatorInfo info) throws Exception {
-        String pathname = getMetadataPathName(info, SecretStore.Global.get(ClientsMetadataPathName));
-        String original = readToEndAsString(metadataStreamProvider.download(pathname));
-        JsonObject main = (JsonObject) Json.decodeValue(original);
-        JsonObject obj = main.getJsonObject("client_keys");
-        String location = obj.getString("location");
-        obj.put("location", downloadUrlGenerator.preSignUrl(location).toString());
-        return main.encode();
-    }
 
+public class ClientMetadataProvider extends MetadataProvider {
     public ClientMetadataProvider(ICloudStorage cloudStorage) {
-        this.metadataStreamProvider = this.downloadUrlGenerator = cloudStorage;
+        super(cloudStorage);
     }
 
     public ClientMetadataProvider(ICloudStorage fileStreamProvider, ICloudStorage downloadUrlGenerator) {
-        this.metadataStreamProvider = fileStreamProvider;
-        this.downloadUrlGenerator = downloadUrlGenerator;
+        super(fileStreamProvider, downloadUrlGenerator);
     }
 
+    public String getMetadata(OperatorInfo info) throws Exception {
+        return getMetadata(info, "clients_metadata_path", "client_keys");
+    }
 }

src/main/java/com/uid2/core/service/ClientSideKeypairMetadataProvider.java

@@ -1,38 +1,18 @@
 package com.uid2.core.service;
 
-import com.uid2.core.model.SecretStore;
 import com.uid2.core.util.OperatorInfo;
 import com.uid2.shared.cloud.ICloudStorage;
-import com.uid2.shared.store.CloudPath;
-import com.uid2.shared.store.scope.GlobalScope;
-import io.vertx.core.json.Json;
-import io.vertx.core.json.JsonObject;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-import static com.uid2.core.util.MetadataHelper.getMetadataPathName;
-import static com.uid2.core.util.MetadataHelper.readToEndAsString;
-
-public class ClientSideKeypairMetadataProvider implements IClientSideKeypairMetadataProvider {
-    public static final String ClientSideKeypairMetadataPathName = "client_side_keypairs_metadata_path";
-
-    private final ICloudStorage metadataStreamProvider;
-    private final ICloudStorage downloadUrlGenerator;
 
+public class ClientSideKeypairMetadataProvider extends MetadataProvider {
     public ClientSideKeypairMetadataProvider(ICloudStorage cloudStorage) {
-        this.metadataStreamProvider = this.downloadUrlGenerator = cloudStorage;
+        super(cloudStorage);
+    }
+
+    public ClientSideKeypairMetadataProvider(ICloudStorage fileStreamProvider, ICloudStorage downloadUrlGenerator) {
+        super(fileStreamProvider, downloadUrlGenerator);
     }
 
-    @Override
     public String getMetadata(OperatorInfo info) throws Exception {
-        String pathname = getMetadataPathName(info, SecretStore.Global.get(ClientSideKeypairMetadataPathName));
-        String original = readToEndAsString(metadataStreamProvider.download(pathname));
-        JsonObject main = (JsonObject) Json.decodeValue(original);
-        JsonObject obj = main.getJsonObject("client_side_keypairs");
-        String location = obj.getString("location");
-        obj.put("location", downloadUrlGenerator.preSignUrl(location).toString());
-        return main.encode();
+        return getMetadata(info, "client_side_keypairs_metadata_path", "client_side_keypairs");
     }
 }