diff --git a/conf/default-config.json b/conf/default-config.json index 224df8906..70364ed8f 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -30,6 +30,7 @@ "salts_metadata_path": "salts/metadata.json", "services_metadata_path": "services/metadata.json", "service_links_metadata_path": "service_links/metadata.json", + "s3_keys_metadata_path": "s3encryption_keys/metadata.json", "optout_metadata_path": null, "optout_inmem_cache": false, "enclave_platform": null, diff --git a/conf/docker-config.json b/conf/docker-config.json index 648b922a8..3f59e8263 100644 --- a/conf/docker-config.json +++ b/conf/docker-config.json @@ -32,6 +32,7 @@ "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json", "services_metadata_path": "/com.uid2.core/test/services/metadata.json", "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json", + "s3_keys_metadata_path": "/com.uid2.core/test/s3encryption_keys/metadata.json", "identity_token_expires_after_seconds": 3600, "optout_metadata_path": null, "optout_inmem_cache": false, diff --git a/conf/integ-config.json b/conf/integ-config.json index f1cf90742..d79f97cc0 100644 --- a/conf/integ-config.json +++ b/conf/integ-config.json @@ -14,6 +14,6 @@ "optout_api_token": "test-operator-key", "optout_api_uri": "http://localhost:8081/optout/replicate", "salts_expired_shutdown_hours": 12, + "s3_keys_metadata_path": "http://localhost:8088/s3encryption_keys/retrieve", "operator_type": "public" - } \ No newline at end of file diff --git a/conf/local-config.json b/conf/local-config.json index f19a4357d..5ba76a923 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -9,6 +9,7 @@ "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json", "services_metadata_path": "/com.uid2.core/test/services/metadata.json", "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json", + "s3_keys_metadata_path":"/com.uid2.core/test/s3encryption_keys/metadata.json", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-docker-private-config.json b/conf/local-e2e-docker-private-config.json index ef05b8772..abb93d899 100644 --- a/conf/local-e2e-docker-private-config.json +++ b/conf/local-e2e-docker-private-config.json @@ -11,6 +11,7 @@ "keysets_metadata_path": "http://core:8088/key/keyset/refresh", "keyset_keys_metadata_path": "http://core:8088/key/keyset-keys/refresh", "salts_metadata_path": "http://core:8088/salt/refresh", + "s3_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-docker-public-config.json b/conf/local-e2e-docker-public-config.json index 60f0abd92..0d2389151 100644 --- a/conf/local-e2e-docker-public-config.json +++ b/conf/local-e2e-docker-public-config.json @@ -13,6 +13,7 @@ "salts_metadata_path": "http://core:8088/salt/refresh", "services_metadata_path": "http://core:8088/services/refresh", "service_links_metadata_path": "http://core:8088/service_links/refresh", + "s3_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-private-config.json b/conf/local-e2e-private-config.json index e9d3f8b53..96a1ac92d 100644 --- a/conf/local-e2e-private-config.json +++ b/conf/local-e2e-private-config.json @@ -13,6 +13,7 @@ "salts_metadata_path": "http://localhost:8088/salt/refresh", "services_metadata_path": "http://localhost:8088/services/refresh", "service_links_metadata_path": "http://localhost:8088/service_links/refresh", + "s3_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-public-config.json b/conf/local-e2e-public-config.json index cb635b103..179b8cb34 100644 --- a/conf/local-e2e-public-config.json +++ b/conf/local-e2e-public-config.json @@ -13,6 +13,7 @@ "salts_metadata_path": "http://localhost:8088/salt/refresh", "services_metadata_path": "http://localhost:8088/services/refresh", "service_links_metadata_path": "http://localhost:8088/service_links/refresh", + "s3_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/validator-latest-e2e-docker-public-config.json b/conf/validator-latest-e2e-docker-public-config.json index cabf23380..66db86a25 100644 --- a/conf/validator-latest-e2e-docker-public-config.json +++ b/conf/validator-latest-e2e-docker-public-config.json @@ -14,6 +14,7 @@ "salts_metadata_path": "http://core:8088/salt/refresh", "services_metadata_path": "http://core:8088/services/refresh", "service_links_metadata_path": "http://core:8088/service_links/refresh", + "s3_keys_metadata_path": "https://core:8088/s3encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 6aa069604..c02bf491d 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -8,6 +8,7 @@ import com.uid2.operator.monitoring.IStatsCollectorQueue; import com.uid2.operator.monitoring.OperatorMetrics; import com.uid2.operator.monitoring.StatsCollectorVerticle; +import com.uid2.operator.reader.RotatingS3KeyOperatorProvider; import com.uid2.operator.service.SecureLinkValidatorService; import com.uid2.operator.service.ShutdownService; import com.uid2.operator.vertx.Endpoints; @@ -81,6 +82,7 @@ public class Main { private IStatsCollectorQueue _statsCollectorQueue; private RotatingServiceStore serviceProvider; private RotatingServiceLinkStore serviceLinkProvider; + private RotatingS3KeyOperatorProvider s3KeyProvider; public Main(Vertx vertx, JsonObject config) throws Exception { this.vertx = vertx; @@ -144,6 +146,8 @@ public Main(Vertx vertx, JsonObject config) throws Exception { String saltsMdPath = this.config.getString(Const.Config.SaltsMetadataPathProp); this.saltProvider = new RotatingSaltProvider(fsStores, saltsMdPath); this.optOutStore = new CloudSyncOptOutStore(vertx, fsLocal, this.config, operatorKey, Clock.systemUTC()); + String s3KeyMdPath = this.config.getString(Const.Config.S3keysMetadataPathProp); + this.s3KeyProvider = new RotatingS3KeyOperatorProvider(fsStores, new GlobalScope(new CloudPath(s3KeyMdPath))); if (this.validateServiceLinks) { String serviceMdPath = this.config.getString(Const.Config.ServiceMetadataPathProp); @@ -163,6 +167,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.saltProvider.loadContent(); this.keysetProvider.loadContent(); this.keysetKeyStore.loadContent(); + this.s3KeyProvider.loadContent(); if (this.validateServiceLinks) { this.serviceProvider.loadContent(); @@ -330,6 +335,7 @@ private Future createStoreVerticles() throws Exception { fs.add(createAndDeployRotatingStoreVerticle("auth", clientKeyProvider, "auth_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("keyset", keysetProvider, "keyset_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("keysetkey", keysetKeyStore, "keysetkey_refresh_ms")); + fs.add(createAndDeployRotatingStoreVerticle("s3encryption_keys", s3KeyProvider, "s3keys_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("salt", saltProvider, "salt_refresh_ms")); fs.add(createAndDeployCloudSyncStoreVerticle("optout", fsOptOut, optOutCloudSync)); CompositeFuture.all(fs).onComplete(ar -> { diff --git a/src/main/java/com/uid2/operator/reader/ApiStoreReader.java b/src/main/java/com/uid2/operator/reader/ApiStoreReader.java new file mode 100644 index 000000000..f34da3513 --- /dev/null +++ b/src/main/java/com/uid2/operator/reader/ApiStoreReader.java @@ -0,0 +1,51 @@ +package com.uid2.operator.reader; + +import com.uid2.shared.cloud.DownloadCloudStorage; +import com.uid2.shared.store.ScopedStoreReader; +import com.uid2.shared.store.parser.Parser; +import com.uid2.shared.store.parser.ParsingResult; +import com.uid2.shared.store.scope.StoreScope; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +public class ApiStoreReader extends ScopedStoreReader { + private static final Logger LOGGER = LoggerFactory.getLogger(ApiStoreReader.class); + + public ApiStoreReader(DownloadCloudStorage fileStreamProvider, StoreScope scope, Parser parser, String dataTypeName) { + super(fileStreamProvider, scope, parser, dataTypeName); + } + + @Override + public long loadContent(JsonObject contents, String dataType) throws Exception { + if (contents == null) { + throw new IllegalArgumentException(String.format("No contents provided for loading data type %s, cannot load content", dataType)); + } + + try { + JsonArray s3KeysArray = contents.getJsonArray(dataType); + if (s3KeysArray == null) { + throw new IllegalArgumentException("No array found in the contents"); + } + + String jsonString = s3KeysArray.toString(); + InputStream inputStream = new ByteArrayInputStream(jsonString.getBytes(StandardCharsets.UTF_8)); + + ParsingResult parsed = parser.deserialize(inputStream); + latestSnapshot.set(parsed.getData()); + + final int count = parsed.getCount(); + latestEntryCount.set(count); + LOGGER.info(String.format("Loaded %d %s", count, dataTypeName)); + return count; + } catch (Exception e) { + LOGGER.error(String.format("Unable to load %s", dataTypeName)); + throw e; + } + } +} diff --git a/src/main/java/com/uid2/operator/reader/RotatingS3KeyOperatorProvider.java b/src/main/java/com/uid2/operator/reader/RotatingS3KeyOperatorProvider.java new file mode 100644 index 000000000..df73b790a --- /dev/null +++ b/src/main/java/com/uid2/operator/reader/RotatingS3KeyOperatorProvider.java @@ -0,0 +1,53 @@ +package com.uid2.operator.reader; + +import com.uid2.operator.reader.ApiStoreReader; +import com.uid2.shared.cloud.DownloadCloudStorage; +import com.uid2.shared.model.S3Key; +import com.uid2.shared.store.CloudPath; +import com.uid2.shared.store.parser.S3KeyParser; +import com.uid2.shared.store.reader.RotatingS3KeyProvider; +import com.uid2.shared.store.scope.StoreScope; +import io.vertx.core.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.util.*; +import java.util.stream.Collectors; + +public class RotatingS3KeyOperatorProvider extends RotatingS3KeyProvider { + private static final Logger LOGGER = LoggerFactory.getLogger(RotatingS3KeyOperatorProvider.class); + + public ApiStoreReader> apiStoreReader; + + public RotatingS3KeyOperatorProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope) { + super(fileStreamProvider, scope); + this.apiStoreReader = new ApiStoreReader<>(fileStreamProvider, scope, new S3KeyParser(), "s3encryption_keys"); + } + + @Override + public JsonObject getMetadata() throws Exception { + return apiStoreReader.getMetadata(); + } + + @Override + public CloudPath getMetadataPath() { + return apiStoreReader.getMetadataPath(); + } + + @Override + public long loadContent(JsonObject metadata) throws Exception { + return apiStoreReader.loadContent(metadata, "s3Keys"); + } + + @Override + public Map getAll() { + Map keys = apiStoreReader.getSnapshot(); + return keys != null ? keys : new HashMap<>(); + } + + @Override + public void loadContent() throws Exception { + this.loadContent(this.getMetadata()); + } +} diff --git a/src/main/resources/com.uid2.core/test/s3encryption_keys/metadata.json b/src/main/resources/com.uid2.core/test/s3encryption_keys/metadata.json new file mode 100644 index 000000000..4a667ec4c --- /dev/null +++ b/src/main/resources/com.uid2.core/test/s3encryption_keys/metadata.json @@ -0,0 +1,7 @@ +{ + "version": 1, + "generated": 1620253519, + "s3encryption_keys": { + "location": "/com.uid2.core/test/s3encryption_keys/s3encryption_keys.json" + } +} \ No newline at end of file diff --git a/src/main/resources/com.uid2.core/test/s3encryption_keys/s3encryption_keys.json b/src/main/resources/com.uid2.core/test/s3encryption_keys/s3encryption_keys.json new file mode 100644 index 000000000..b875d6797 --- /dev/null +++ b/src/main/resources/com.uid2.core/test/s3encryption_keys/s3encryption_keys.json @@ -0,0 +1,73 @@ +[ { + "id" : 1, + "siteId" : 999, + "activates" : 1720641670, + "created" : 1720641670, + "secret" : "mydrCudb2PZOm01Qn0SpthltmexHUAA11Hy1m+uxjVw=" +}, { + "id" : 2, + "siteId" : 999, + "activates" : 1720728070, + "created" : 1720641670, + "secret" : "FtdslrFSsvVXOuhOWGwEI+0QTkCvM8SGZAP3k2u3PgY=" +}, { + "id" : 3, + "siteId" : 999, + "activates" : 1720814470, + "created" : 1720641670, + "secret" : "/7zO6QbKrhZKIV36G+cU9UR4hZUVg5bD+KjbczICjHw=" +}, { + "id" : 4, + "siteId" : 123, + "activates" : 1720641671, + "created" : 1720641671, + "secret" : "XjiqRlWQQJGLr7xfV1qbueKwyzt881GVohuUkQt/ht4=" +}, { + "id" : 5, + "siteId" : 123, + "activates" : 1720728071, + "created" : 1720641671, + "secret" : "QmpIf5NzO+UROjl5XjB/BmF6paefM8n6ub9B2plC9aI=" +}, { + "id" : 6, + "siteId" : 123, + "activates" : 1720814471, + "created" : 1720641671, + "secret" : "40w9UMSYxGm+KldOWOXhBGI8QgjvUUQjivtkP4VpKV8=" +}, { + "id" : 7, + "siteId" : 124, + "activates" : 1720641671, + "created" : 1720641671, + "secret" : "QdwD0kQV1BwmLRD0PH1YpqgaOrgpVTfu08o98mSZ6uE=" +}, { + "id" : 8, + "siteId" : 124, + "activates" : 1720728071, + "created" : 1720641671, + "secret" : "yCVCM/HLf9/6k+aUNrx7w17VbyfSzI8JykLQLSR+CW0=" +}, { + "id" : 9, + "siteId" : 124, + "activates" : 1720814471, + "created" : 1720641671, + "secret" : "JqHl8BrTyx9XpR2lYj/5xvUpzgnibGeomETTwF4rn1U=" +}, { + "id" : 10, + "siteId" : 127, + "activates" : 1720641671, + "created" : 1720641671, + "secret" : "JqiG1b34AvrdO3Aj6cCcjOBJMijrDzTmrR+p9ZtP2es=" +}, { + "id" : 11, + "siteId" : 127, + "activates" : 1720728072, + "created" : 1720641672, + "secret" : "lp1CyHdfc7K0aO5JGpA+Ve5Z/V5LImtGEQwCg/YB0kY=" +}, { + "id" : 12, + "siteId" : 127, + "activates" : 1720814472, + "created" : 1720641672, + "secret" : "G99rFYJF+dnSlk/xG6fuC3WNqQxTLJbDIdVyPMbGQ6s=" +} ] \ No newline at end of file diff --git a/src/test/java/com/uid2/operator/ApiStoreReaderTest.java b/src/test/java/com/uid2/operator/ApiStoreReaderTest.java new file mode 100644 index 000000000..460d57b8d --- /dev/null +++ b/src/test/java/com/uid2/operator/ApiStoreReaderTest.java @@ -0,0 +1,103 @@ +package com.uid2.operator; + +import com.uid2.operator.reader.ApiStoreReader; +import com.uid2.shared.cloud.DownloadCloudStorage; +import com.uid2.shared.store.CloudPath; +import com.uid2.shared.store.parser.Parser; +import com.uid2.shared.store.parser.ParsingResult; +import com.uid2.shared.store.scope.GlobalScope; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + + class ApiStoreReaderTest { + + @Mock + private DownloadCloudStorage mockStorage; + + @Mock + private Parser> mockParser; + + private final CloudPath metadataPath = new CloudPath("test/test-metadata.json"); + private final String dataType = "test-data-type"; + private final GlobalScope scope = new GlobalScope(metadataPath); + + private ApiStoreReader> reader; + + @BeforeEach + void setUp() { + MockitoAnnotations.openMocks(this); + reader = new ApiStoreReader<>(mockStorage, scope, mockParser, dataType); + } + + @Test + void getMetadataPathReturnsPathFromScope() { + CloudPath actual = reader.getMetadataPath(); + assertThat(actual).isEqualTo(metadataPath); + } + + @Test + void loadContentThrowsExceptionWhenContentsAreNull() { + assertThatThrownBy(() -> reader.loadContent(null, dataType)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("No contents provided for loading data type"); + } + + @Test + void loadContentThrowsExceptionWhenArrayNotFound() { + JsonObject contents = new JsonObject(); + assertThatThrownBy(() -> reader.loadContent(contents, dataType)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("No array found in the contents"); + } + + @Test + void loadContentSuccessfullyLoadsData() throws Exception { + JsonObject contents = new JsonObject() + .put(dataType, new JsonArray().add("value1").add("value2")); + + List expectedData = Arrays.asList(new TestData("value1"), new TestData("value2")); + when(mockParser.deserialize(any(InputStream.class))) + .thenReturn(new ParsingResult<>(expectedData, expectedData.size())); + + long count = reader.loadContent(contents, dataType); + + assertThat(count).isEqualTo(2); + assertThat(reader.getSnapshot()).isEqualTo(expectedData); + } + + private static class TestData { + private final String value; + + TestData(String value) { + this.value = value; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TestData testData = (TestData) o; + return value.equals(testData.value); + } + + @Override + public int hashCode() { + return value.hashCode(); + } + } + } + diff --git a/src/test/java/com/uid2/operator/RotatingS3KeyOperatorProviderTest.java b/src/test/java/com/uid2/operator/RotatingS3KeyOperatorProviderTest.java new file mode 100644 index 000000000..ee53d3ab5 --- /dev/null +++ b/src/test/java/com/uid2/operator/RotatingS3KeyOperatorProviderTest.java @@ -0,0 +1,103 @@ +package com.uid2.operator; + +import com.uid2.operator.reader.ApiStoreReader; +import com.uid2.operator.reader.RotatingS3KeyOperatorProvider; +import com.uid2.shared.cloud.DownloadCloudStorage; +import com.uid2.shared.model.S3Key; +import com.uid2.shared.store.CloudPath; +import com.uid2.shared.store.scope.StoreScope; +import io.vertx.core.json.JsonObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +class RotatingS3KeyOperatorProviderTest { + + @Mock + private DownloadCloudStorage mockFileStreamProvider; + + @Mock + private StoreScope mockScope; + + @Mock + private ApiStoreReader> mockApiStoreReader; + + private RotatingS3KeyOperatorProvider rotatingS3KeyOperatorProvider; + + @BeforeEach + void setUp() { + MockitoAnnotations.openMocks(this); + rotatingS3KeyOperatorProvider = new RotatingS3KeyOperatorProvider(mockFileStreamProvider, mockScope); + rotatingS3KeyOperatorProvider.apiStoreReader = mockApiStoreReader; + } + + @Test + void testGetMetadata() throws Exception { + JsonObject expectedMetadata = new JsonObject().put("version", 1L); + when(mockApiStoreReader.getMetadata()).thenReturn(expectedMetadata); + + JsonObject metadata = rotatingS3KeyOperatorProvider.getMetadata(); + assertEquals(expectedMetadata, metadata); + verify(mockApiStoreReader).getMetadata(); + } + + @Test + void testGetMetadataPath() { + CloudPath expectedPath = new CloudPath("test/path"); + when(mockApiStoreReader.getMetadataPath()).thenReturn(expectedPath); + + CloudPath path = rotatingS3KeyOperatorProvider.getMetadataPath(); + assertEquals(expectedPath, path); + verify(mockApiStoreReader).getMetadataPath(); + } + + @Test + void testLoadContentWithMetadata() throws Exception { + JsonObject metadata = new JsonObject(); + when(mockApiStoreReader.loadContent(metadata, "s3Keys")).thenReturn(1L); + + long version = rotatingS3KeyOperatorProvider.loadContent(metadata); + assertEquals(1L, version); + verify(mockApiStoreReader).loadContent(metadata, "s3Keys"); + } + + @Test + void testGetAll() { + Map expectedKeys = new HashMap<>(); + S3Key key = new S3Key(1, 123, 1687635529, 1687808329, "secret"); + expectedKeys.put(1, key); + when(mockApiStoreReader.getSnapshot()).thenReturn(expectedKeys); + + Map keys = rotatingS3KeyOperatorProvider.getAll(); + assertEquals(expectedKeys, keys); + verify(mockApiStoreReader).getSnapshot(); + } + + @Test + void testGetAllWithNullSnapshot() { + when(mockApiStoreReader.getSnapshot()).thenReturn(null); + + Map keys = rotatingS3KeyOperatorProvider.getAll(); + assertNotNull(keys); + assertTrue(keys.isEmpty()); + verify(mockApiStoreReader).getSnapshot(); + } + + @Test + void testLoadContent() throws Exception { + JsonObject metadata = new JsonObject().put("version", 1L); + when(mockApiStoreReader.getMetadata()).thenReturn(metadata); + when(mockApiStoreReader.loadContent(metadata, "s3Keys")).thenReturn(1L); + + rotatingS3KeyOperatorProvider.loadContent(); + verify(mockApiStoreReader).getMetadata(); + verify(mockApiStoreReader).loadContent(metadata, "s3Keys"); + } +} \ No newline at end of file