wzh uid2 3573 let operator retrieve their s3 keys

conf/default-config.json

@@ -30,6 +30,7 @@
   "salts_metadata_path": "salts/metadata.json",
   "services_metadata_path": "services/metadata.json",
   "service_links_metadata_path": "service_links/metadata.json",
+  "s3_keys_metadata_path": "s3encryption_keys/metadata.json",
   "optout_metadata_path": null,
   "optout_inmem_cache": false,
   "enclave_platform": null,

conf/docker-config.json

@@ -32,6 +32,7 @@
   "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json",
   "services_metadata_path": "/com.uid2.core/test/services/metadata.json",
   "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json",
+  "s3_keys_metadata_path": "/com.uid2.core/test/s3encryption_keys/metadata.json",
   "identity_token_expires_after_seconds": 3600,
   "optout_metadata_path": null,
   "optout_inmem_cache": false,

conf/integ-config.json

@@ -14,6 +14,6 @@
   "optout_api_token": "test-operator-key",
   "optout_api_uri": "http://localhost:8081/optout/replicate",
   "salts_expired_shutdown_hours": 12,
+  "s3_keys_metadata_path": "http://localhost:8088/s3encryption_keys/retrieve",
   "operator_type": "public"
-
 }

conf/local-config.json

@@ -9,6 +9,7 @@
   "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json",
   "services_metadata_path": "/com.uid2.core/test/services/metadata.json",
   "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json",
+  "s3_keys_metadata_path":"/com.uid2.core/test/s3encryption_keys/metadata.json",
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
   "refresh_identity_token_after_seconds": 900,

conf/local-e2e-docker-private-config.json

@@ -11,6 +11,7 @@
   "keysets_metadata_path": "http://core:8088/key/keyset/refresh",
   "keyset_keys_metadata_path": "http://core:8088/key/keyset-keys/refresh",
   "salts_metadata_path": "http://core:8088/salt/refresh",
+  "s3_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve",
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
   "refresh_identity_token_after_seconds": 900,

conf/local-e2e-docker-public-config.json

@@ -13,6 +13,7 @@
   "salts_metadata_path": "http://core:8088/salt/refresh",
   "services_metadata_path": "http://core:8088/services/refresh",
   "service_links_metadata_path": "http://core:8088/service_links/refresh",
+  "s3_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve",
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
   "refresh_identity_token_after_seconds": 900,

conf/local-e2e-private-config.json

@@ -13,6 +13,7 @@
   "salts_metadata_path": "http://localhost:8088/salt/refresh",
   "services_metadata_path": "http://localhost:8088/services/refresh",
   "service_links_metadata_path": "http://localhost:8088/service_links/refresh",
+  "s3_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve",
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
   "refresh_identity_token_after_seconds": 900,

conf/local-e2e-public-config.json

@@ -13,6 +13,7 @@
   "salts_metadata_path": "http://localhost:8088/salt/refresh",
   "services_metadata_path": "http://localhost:8088/services/refresh",
   "service_links_metadata_path": "http://localhost:8088/service_links/refresh",
+  "s3_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve",
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
   "refresh_identity_token_after_seconds": 900,

conf/validator-latest-e2e-docker-public-config.json

@@ -14,6 +14,7 @@
   "salts_metadata_path": "http://core:8088/salt/refresh",
   "services_metadata_path": "http://core:8088/services/refresh",
   "service_links_metadata_path": "http://core:8088/service_links/refresh",
+  "s3_keys_metadata_path": "https://core:8088/s3encryption_keys/retrieve",
   "identity_token_expires_after_seconds": 3600,
   "refresh_token_expires_after_seconds": 86400,
   "refresh_identity_token_after_seconds": 900,

src/main/java/com/uid2/operator/Main.java

@@ -8,6 +8,7 @@
 import com.uid2.operator.monitoring.IStatsCollectorQueue;
 import com.uid2.operator.monitoring.OperatorMetrics;
 import com.uid2.operator.monitoring.StatsCollectorVerticle;
+import com.uid2.operator.reader.RotatingS3KeyOperatorProvider;
 import com.uid2.operator.service.SecureLinkValidatorService;
 import com.uid2.operator.service.ShutdownService;
 import com.uid2.operator.vertx.Endpoints;
@@ -81,6 +82,7 @@ public class Main {
     private IStatsCollectorQueue _statsCollectorQueue;
     private RotatingServiceStore serviceProvider;
     private RotatingServiceLinkStore serviceLinkProvider;
+    private RotatingS3KeyOperatorProvider s3KeyProvider;
 
     public Main(Vertx vertx, JsonObject config) throws Exception {
         this.vertx = vertx;
@@ -144,6 +146,8 @@ public Main(Vertx vertx, JsonObject config) throws Exception {
         String saltsMdPath = this.config.getString(Const.Config.SaltsMetadataPathProp);
         this.saltProvider = new RotatingSaltProvider(fsStores, saltsMdPath);
         this.optOutStore = new CloudSyncOptOutStore(vertx, fsLocal, this.config, operatorKey, Clock.systemUTC());
+        String s3KeyMdPath = this.config.getString(Const.Config.S3keysMetadataPathProp);
+        this.s3KeyProvider = new RotatingS3KeyOperatorProvider(fsStores, new GlobalScope(new CloudPath(s3KeyMdPath)));
 
         if (this.validateServiceLinks) {
             String serviceMdPath = this.config.getString(Const.Config.ServiceMetadataPathProp);
@@ -163,6 +167,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception {
             this.saltProvider.loadContent();
             this.keysetProvider.loadContent();
             this.keysetKeyStore.loadContent();
+            this.s3KeyProvider.loadContent();
 
             if (this.validateServiceLinks) {
                 this.serviceProvider.loadContent();
@@ -330,6 +335,7 @@ private Future<Void> createStoreVerticles() throws Exception {
         fs.add(createAndDeployRotatingStoreVerticle("auth", clientKeyProvider, "auth_refresh_ms"));
         fs.add(createAndDeployRotatingStoreVerticle("keyset", keysetProvider, "keyset_refresh_ms"));
         fs.add(createAndDeployRotatingStoreVerticle("keysetkey", keysetKeyStore, "keysetkey_refresh_ms"));
+        fs.add(createAndDeployRotatingStoreVerticle("s3encryption_keys", s3KeyProvider, "s3keys_refresh_ms"));
         fs.add(createAndDeployRotatingStoreVerticle("salt", saltProvider, "salt_refresh_ms"));
         fs.add(createAndDeployCloudSyncStoreVerticle("optout", fsOptOut, optOutCloudSync));
         CompositeFuture.all(fs).onComplete(ar -> {

src/main/java/com/uid2/operator/reader/ApiStoreReader.java

@@ -0,0 +1,51 @@
+package com.uid2.operator.reader;
+
+import com.uid2.shared.cloud.DownloadCloudStorage;
+import com.uid2.shared.store.ScopedStoreReader;
+import com.uid2.shared.store.parser.Parser;
+import com.uid2.shared.store.parser.ParsingResult;
+import com.uid2.shared.store.scope.StoreScope;
+import io.vertx.core.json.JsonArray;
+import io.vertx.core.json.JsonObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+
+public class ApiStoreReader<T> extends ScopedStoreReader<T> {
+    private static final Logger LOGGER = LoggerFactory.getLogger(ApiStoreReader.class);
+
+    public ApiStoreReader(DownloadCloudStorage fileStreamProvider, StoreScope scope, Parser<T> parser, String dataTypeName) {
+        super(fileStreamProvider, scope, parser, dataTypeName);
+    }
+
+    @Override
+    public long loadContent(JsonObject contents, String dataType) throws Exception {
+        if (contents == null) {
+            throw new IllegalArgumentException(String.format("No contents provided for loading data type %s, cannot load content", dataType));
+        }
+
+        try {
+            JsonArray s3KeysArray = contents.getJsonArray(dataType);
+            if (s3KeysArray == null) {
+                throw new IllegalArgumentException("No array found in the contents");
+            }
+
+            String jsonString = s3KeysArray.toString();
+            InputStream inputStream = new ByteArrayInputStream(jsonString.getBytes(StandardCharsets.UTF_8));
+
+            ParsingResult<T> parsed = parser.deserialize(inputStream);
+            latestSnapshot.set(parsed.getData());
+
+            final int count = parsed.getCount();
+            latestEntryCount.set(count);
+            LOGGER.info(String.format("Loaded %d %s", count, dataTypeName));
+            return count;
+        } catch (Exception e) {
+            LOGGER.error(String.format("Unable to load %s", dataTypeName));
+            throw e;
+        }
+    }
+}

src/main/java/com/uid2/operator/reader/RotatingS3KeyOperatorProvider.java

@@ -0,0 +1,53 @@
+package com.uid2.operator.reader;
+
+import com.uid2.operator.reader.ApiStoreReader;
+import com.uid2.shared.cloud.DownloadCloudStorage;
+import com.uid2.shared.model.S3Key;
+import com.uid2.shared.store.CloudPath;
+import com.uid2.shared.store.parser.S3KeyParser;
+import com.uid2.shared.store.reader.RotatingS3KeyProvider;
+import com.uid2.shared.store.scope.StoreScope;
+import io.vertx.core.json.JsonObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Instant;
+import java.util.*;
+import java.util.stream.Collectors;
+
+public class RotatingS3KeyOperatorProvider extends RotatingS3KeyProvider {
+    private static final Logger LOGGER = LoggerFactory.getLogger(RotatingS3KeyOperatorProvider.class);
+
+    public ApiStoreReader<Map<Integer, S3Key>> apiStoreReader;
+
+    public RotatingS3KeyOperatorProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope) {
+        super(fileStreamProvider, scope);
+        this.apiStoreReader = new ApiStoreReader<>(fileStreamProvider, scope, new S3KeyParser(), "s3encryption_keys");
+    }
+
+    @Override
+    public JsonObject getMetadata() throws Exception {
+        return apiStoreReader.getMetadata();
+    }
+
+    @Override
+    public CloudPath getMetadataPath() {
+        return apiStoreReader.getMetadataPath();
+    }
+
+    @Override
+    public long loadContent(JsonObject metadata) throws Exception {
+        return apiStoreReader.loadContent(metadata, "s3Keys");
+    }
+
+    @Override
+    public Map<Integer, S3Key> getAll() {
+        Map<Integer, S3Key> keys = apiStoreReader.getSnapshot();
+        return keys != null ? keys : new HashMap<>();
+    }
+
+    @Override
+    public void loadContent() throws Exception {
+        this.loadContent(this.getMetadata());
+    }
+}

src/main/resources/com.uid2.core/test/s3encryption_keys/metadata.json

@@ -0,0 +1,7 @@
+{
+  "version": 1,
+  "generated": 1620253519,
+  "s3encryption_keys": {
+    "location": "/com.uid2.core/test/s3encryption_keys/s3encryption_keys.json"
+  }
+}

src/main/resources/com.uid2.core/test/s3encryption_keys/s3encryption_keys.json

@@ -0,0 +1,73 @@
+[ {
+  "id" : 1,
+  "siteId" : 999,
+  "activates" : 1720641670,
+  "created" : 1720641670,
+  "secret" : "mydrCudb2PZOm01Qn0SpthltmexHUAA11Hy1m+uxjVw="
+}, {
+  "id" : 2,
+  "siteId" : 999,
+  "activates" : 1720728070,
+  "created" : 1720641670,
+  "secret" : "FtdslrFSsvVXOuhOWGwEI+0QTkCvM8SGZAP3k2u3PgY="
+}, {
+  "id" : 3,
+  "siteId" : 999,
+  "activates" : 1720814470,
+  "created" : 1720641670,
+  "secret" : "/7zO6QbKrhZKIV36G+cU9UR4hZUVg5bD+KjbczICjHw="
+}, {
+  "id" : 4,
+  "siteId" : 123,
+  "activates" : 1720641671,
+  "created" : 1720641671,
+  "secret" : "XjiqRlWQQJGLr7xfV1qbueKwyzt881GVohuUkQt/ht4="
+}, {
+  "id" : 5,
+  "siteId" : 123,
+  "activates" : 1720728071,
+  "created" : 1720641671,
+  "secret" : "QmpIf5NzO+UROjl5XjB/BmF6paefM8n6ub9B2plC9aI="
+}, {
+  "id" : 6,
+  "siteId" : 123,
+  "activates" : 1720814471,
+  "created" : 1720641671,
+  "secret" : "40w9UMSYxGm+KldOWOXhBGI8QgjvUUQjivtkP4VpKV8="
+}, {
+  "id" : 7,
+  "siteId" : 124,
+  "activates" : 1720641671,
+  "created" : 1720641671,
+  "secret" : "QdwD0kQV1BwmLRD0PH1YpqgaOrgpVTfu08o98mSZ6uE="
+}, {
+  "id" : 8,
+  "siteId" : 124,
+  "activates" : 1720728071,
+  "created" : 1720641671,
+  "secret" : "yCVCM/HLf9/6k+aUNrx7w17VbyfSzI8JykLQLSR+CW0="
+}, {
+  "id" : 9,
+  "siteId" : 124,
+  "activates" : 1720814471,
+  "created" : 1720641671,
+  "secret" : "JqHl8BrTyx9XpR2lYj/5xvUpzgnibGeomETTwF4rn1U="
+}, {
+  "id" : 10,
+  "siteId" : 127,
+  "activates" : 1720641671,
+  "created" : 1720641671,
+  "secret" : "JqiG1b34AvrdO3Aj6cCcjOBJMijrDzTmrR+p9ZtP2es="
+}, {
+  "id" : 11,
+  "siteId" : 127,
+  "activates" : 1720728072,
+  "created" : 1720641672,
+  "secret" : "lp1CyHdfc7K0aO5JGpA+Ve5Z/V5LImtGEQwCg/YB0kY="
+}, {
+  "id" : 12,
+  "siteId" : 127,
+  "activates" : 1720814472,
+  "created" : 1720641672,
+  "secret" : "G99rFYJF+dnSlk/xG6fuC3WNqQxTLJbDIdVyPMbGQ6s="
+} ]