Remote config: Use rotating store verticle and map to POJO (#1641)

- Periodically retrieve runtime config using a rotating store verticle, as core endpoint now requires attestation
- Update runtime config format to include an incrementing version number, as required by rotating store verticle
- Map runtime config JSON to POJO (`RuntimeConfig`)
- Validate runtime config value in `RuntimeConfig` constructor so consumers don't have to remember to call `isValid`
- Use builder pattern for runtime config. Makes tests cleaner. Can be replaced with Lombok in the future
- Log runtime config on retrieval to help with debugging

Tested scenarios:
- Run locally with feature flag OFF
  - Operator loads runtime config values from bootstrap config
- Run locally with feature flag ON, `storage_mock: false` and `runtime_config_metadata_path` pointing to core
  - Operator loads runtime config values from core API endpoint 
- Run locally with feature flag ON, `storage_mock: true` and `runtime_config_metadata_path` pointing to new `metadata.json` file
  - Operator loads runtime config values from local `metadata.json` file

---------

Co-authored-by: Matt Collins <matt.collins@thetradedesk.com>

Author: cYKatherine

Dockerfile

@@ -17,7 +17,6 @@ COPY ./target/${JAR_NAME}-${JAR_VERSION}-sources.jar /app
 COPY ./target/${JAR_NAME}-${JAR_VERSION}-static.tar.gz /app/static.tar.gz
 COPY ./conf/default-config.json /app/conf/
 COPY ./conf/*.xml /app/conf/
-COPY ./conf/runtime-config-defaults.json /app/conf/
 
 RUN tar xzvf /app/static.tar.gz --no-same-owner --no-same-permissions && rm -f /app/static.tar.gz
 

conf/docker-config.json

@@ -32,6 +32,7 @@
   "services_metadata_path": "/com.uid2.core/test/services/metadata.json",
   "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json",
   "cloud_encryption_keys_metadata_path": "/com.uid2.core/test/cloud_encryption_keys/metadata.json",
+  "runtime_config_metadata_path": "/com.uid2.core/test/runtime_config/metadata.json",
   "encrypted_files": true,
   "identity_token_expires_after_seconds": 3600,
   "optout_metadata_path": null,
@@ -40,14 +41,6 @@
   "failure_shutdown_wait_hours": 120,
   "salts_expired_shutdown_hours": 12,
   "operator_type": "public",
-  "runtime_config_store": {
-    "type": "file",
-    "config" : {
-      "path": "conf/runtime-config-defaults.json",
-      "format": "json"
-    },
-    "config_scan_period_ms": 5000
-  },
   "disable_optout_token": true,
   "enable_remote_config": false
 }

conf/integ-config.json

@@ -14,15 +14,9 @@
   "optout_api_token": "test-operator-key",
   "optout_api_uri": "http://localhost:8081/optout/replicate",
   "cloud_encryption_keys_metadata_path": "http://localhost:8088/cloud_encryption_keys/retrieve",
+  "runtime_config_metadata_path": "http://localhost:8088/operator/config",
   "salts_expired_shutdown_hours": 12,
   "operator_type": "public",
-  "runtime_config_store": {
-    "type": "http",
-    "config" : {
-      "url": "http://localhost:8088/operator/config"
-    },
-    "config_scan_period_ms": 300000
-  },
   "disable_optout_token": true,
   "enable_remote_config": false
 }

conf/local-config.json

@@ -10,6 +10,7 @@
   "services_metadata_path": "/com.uid2.core/test/services/metadata.json",
   "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json",
   "cloud_encryption_keys_metadata_path": "/com.uid2.core/test/cloud_encryption_keys/metadata.json",
+  "runtime_config_metadata_path": "/com.uid2.core/test/runtime_config/metadata.json",
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
   "refresh_identity_token_after_seconds": 900,
@@ -39,14 +40,6 @@
   "salts_expired_shutdown_hours": 12,
   "operator_type": "public",
   "encrypted_files": true,
-  "runtime_config_store": {
-    "type": "file",
-    "config" : {
-      "path": "conf/runtime-config-defaults.json",
-      "format": "json"
-    },
-    "config_scan_period_ms": 5000
-  },
   "disable_optout_token": true,
   "enable_remote_config": false
 }

conf/local-e2e-docker-private-config.json

@@ -12,6 +12,7 @@
   "keyset_keys_metadata_path": "http://core:8088/key/keyset-keys/refresh",
   "salts_metadata_path": "http://core:8088/salt/refresh",
   "cloud_encryption_keys_metadata_path": "http://core:8088/cloud_encryption_keys/retrieve",
+  "runtime_config_metadata_path": "http://core:8088/operator/config",
   "encrypted_files": false,
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
@@ -30,12 +31,5 @@
   "cloud_refresh_interval": 30,
   "salts_expired_shutdown_hours": 12,
   "operator_type": "private",
-  "runtime_config_store": {
-    "type": "http",
-    "config" : {
-      "url": "http://core:8088/operator/config"
-    },
-    "config_scan_period_ms": 300000
-  },
   "enable_remote_config": false
 }

conf/local-e2e-docker-public-config.json

@@ -14,6 +14,7 @@
   "services_metadata_path": "http://core:8088/services/refresh",
   "service_links_metadata_path": "http://core:8088/service_links/refresh",
   "cloud_encryption_keys_metadata_path": "http://core:8088/cloud_encryption_keys/retrieve",
+  "runtime_config_metadata_path": "http://core:8088/operator/config",
   "encrypted_files": false,
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
@@ -36,13 +37,6 @@
   "cloud_refresh_interval": 30,
   "salts_expired_shutdown_hours": 12,
   "operator_type": "public",
-  "runtime_config_store": {
-    "type": "http",
-    "config" : {
-      "url": "http://core:8088/operator/config"
-    },
-    "config_scan_period_ms": 300000
-  },
   "disable_optout_token": true,
   "enable_remote_config": false
 }

conf/local-e2e-private-config.json

@@ -14,6 +14,7 @@
   "services_metadata_path": "http://localhost:8088/services/refresh",
   "service_links_metadata_path": "http://localhost:8088/service_links/refresh",
   "cloud_encryption_keys_metadata_path": "http://localhost:8088/cloud_encryption_keys/retrieve",
+  "runtime_config_metadata_path": "http://localhost:8088/operator/config",
   "encrypted_files": false,
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
@@ -41,12 +42,5 @@
   "client_side_token_generate_log_invalid_http_origins": true,
   "salts_expired_shutdown_hours": 12,
   "operator_type": "private",
-  "runtime_config_store": {
-    "type": "http",
-    "config" : {
-      "url": "http://localhost:8088/operator/config"
-    },
-    "config_scan_period_ms": 300000
-  },
   "enable_remote_config": false
 }

conf/local-e2e-public-config.json

@@ -14,6 +14,7 @@
   "services_metadata_path": "http://localhost:8088/services/refresh",
   "service_links_metadata_path": "http://localhost:8088/service_links/refresh",
   "cloud_encryption_keys_metadata_path": "http://localhost:8088/cloud_encryption_keys/retrieve",
+  "runtime_config_metadata_path": "http://localhost:8088/operator/config",
   "encrypted_files": false,
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
@@ -42,13 +43,6 @@
   "client_side_token_generate_log_invalid_http_origins": true,
   "salts_expired_shutdown_hours": 12,
   "operator_type": "public",
-  "runtime_config_store": {
-    "type": "http",
-    "config" : {
-      "url": "http://localhost:8088/operator/config"
-    },
-    "config_scan_period_ms": 300000
-  },
   "disable_optout_token": true,
   "enable_remote_config": false
 }

conf/runtime-config-defaults.json

@@ -35,5 +35,6 @@ public class Config extends com.uid2.shared.Const.Config {
         public static final String IdentityV3Prop = "identity_v3";
         public static final String DisableOptoutTokenProp = "disable_optout_token";
         public static final String EnableRemoteConfigProp = "enable_remote_config";
+        public static final String RuntimeConfigMetadataPathProp = "runtime_config_metadata_path";
     }
 }