diff --git a/api/kubernetes/helm-chart/values.yaml b/api/kubernetes/helm-chart/values.yaml index 1623bc272..14a92cb3d 100644 --- a/api/kubernetes/helm-chart/values.yaml +++ b/api/kubernetes/helm-chart/values.yaml @@ -172,6 +172,12 @@ argo: config: csm: platform: + twincache: + host: redis.host.changeme + password: changeme + port: 6379 + username: default + useGraphModule: true vendor: azure argo: base-uri: "http://argo-server:2746" diff --git a/api/src/main/resources/application.yml b/api/src/main/resources/application.yml index 543a54d24..084afa001 100644 --- a/api/src/main/resources/application.yml +++ b/api/src/main/resources/application.yml @@ -77,6 +77,7 @@ csm: port: "6379" username: "default_user" password: "default_password" + useGraphModule: true dataset: # TODO: Should be way less than a thousand. See open ticket maxResult: 1000 diff --git a/dataset/src/integrationTest/kotlin/com/cosmotech/dataset/service/DatasetServiceIntegrationTest.kt b/dataset/src/integrationTest/kotlin/com/cosmotech/dataset/service/DatasetServiceIntegrationTest.kt index 4da7c0c63..6e71aa30b 100644 --- a/dataset/src/integrationTest/kotlin/com/cosmotech/dataset/service/DatasetServiceIntegrationTest.kt +++ b/dataset/src/integrationTest/kotlin/com/cosmotech/dataset/service/DatasetServiceIntegrationTest.kt @@ -111,7 +111,7 @@ class DatasetServiceIntegrationTest : CsmRedisTestBase() { @Autowired lateinit var solutionApiService: SolutionApiServiceInterface @Autowired lateinit var workspaceApiService: WorkspaceApiServiceInterface @SpykBean @Autowired lateinit var csmPlatformProperties: CsmPlatformProperties - @MockK(relaxUnitFun = true) private lateinit var eventPublisher: CsmEventPublisher + @MockK(relaxUnitFun = true) lateinit var eventPublisher: CsmEventPublisher lateinit var connectorSaved: Connector lateinit var dataset: Dataset @@ -707,6 +707,7 @@ class DatasetServiceIntegrationTest : CsmRedisTestBase() { @Test fun `test uploadTwingraph status`() { organizationSaved = organizationApiService.registerOrganization(organization) + dataset.apply { sourceType = DatasetSourceType.File } datasetSaved = datasetApiService.createDataset(organizationSaved.id!!, dataset) val file = this::class.java.getResource("/integrationTest.zip")?.file val resource = ByteArrayResource(File(file!!).readBytes()) @@ -731,6 +732,7 @@ class DatasetServiceIntegrationTest : CsmRedisTestBase() { @Test fun `test uploadTwingraph fail set dataset status to error`() { organizationSaved = organizationApiService.registerOrganization(organization) + dataset.apply { sourceType = DatasetSourceType.File } datasetSaved = datasetApiService.createDataset(organizationSaved.id!!, dataset) val file = this::class.java.getResource("/brokenGraph.zip")?.file val resource = ByteArrayResource(File(file!!).readBytes()) @@ -790,6 +792,7 @@ class DatasetServiceIntegrationTest : CsmRedisTestBase() { @Test fun `reupload a twingraph in dataset with source type File`() { organizationSaved = organizationApiService.registerOrganization(organization) + dataset.apply { sourceType = DatasetSourceType.File } datasetSaved = datasetApiService.createDataset(organizationSaved.id!!, dataset) val fileName = this::class.java.getResource("/integrationTest.zip")?.file @@ -858,7 +861,7 @@ class DatasetServiceIntegrationTest : CsmRedisTestBase() { every { getCurrentAccountIdentifier(any()) } returns CONNECTED_ADMIN_USER organization = makeOrganizationWithRole("organization") organizationSaved = organizationApiService.registerOrganization(organization) - makeDatasetWithRole(sourceType = DatasetSourceType.File) + dataset = makeDatasetWithRole(sourceType = DatasetSourceType.File) datasetSaved = datasetApiService.createDataset(organizationSaved.id!!, dataset) datasetRepository.save(datasetSaved.apply { ingestionStatus = IngestionStatusEnum.ERROR }) @@ -1045,7 +1048,7 @@ class DatasetServiceIntegrationTest : CsmRedisTestBase() { fun makeDataset( organizationId: String = organizationSaved.id!!, parentId: String = "", - sourceType: DatasetSourceType = DatasetSourceType.File + sourceType: DatasetSourceType = DatasetSourceType.Twincache ): Dataset { return Dataset( id = UUID.randomUUID().toString(), @@ -1065,7 +1068,7 @@ class DatasetServiceIntegrationTest : CsmRedisTestBase() { parentId: String = "", userName: String = TEST_USER_MAIL, role: String = ROLE_ADMIN, - sourceType: DatasetSourceType = DatasetSourceType.File + sourceType: DatasetSourceType = DatasetSourceType.Twincache ): Dataset { return Dataset( id = UUID.randomUUID().toString(), diff --git a/dataset/src/integrationTest/kotlin/com/cosmotech/dataset/service/DatasetServiceRBACTest.kt b/dataset/src/integrationTest/kotlin/com/cosmotech/dataset/service/DatasetServiceRBACTest.kt index 68a17d8c9..e7ec7497c 100644 --- a/dataset/src/integrationTest/kotlin/com/cosmotech/dataset/service/DatasetServiceRBACTest.kt +++ b/dataset/src/integrationTest/kotlin/com/cosmotech/dataset/service/DatasetServiceRBACTest.kt @@ -1660,7 +1660,8 @@ class DatasetServiceRBACTest : CsmRedisTestBase() { val organization = makeOrganizationWithRole(role = role) organizationSaved = organizationApiService.registerOrganization(organization) - val dataset = makeDatasetWithRole(role = ROLE_ADMIN) + val dataset = + makeDatasetWithRole(role = ROLE_ADMIN, sourceType = DatasetSourceType.File) datasetSaved = datasetApiService.createDataset(organizationSaved.id!!, dataset) val fileName = this::class.java.getResource("/integrationTest.zip")?.file val file = File(fileName!!) @@ -1701,7 +1702,7 @@ class DatasetServiceRBACTest : CsmRedisTestBase() { val organization = makeOrganizationWithRole() organizationSaved = organizationApiService.registerOrganization(organization) - val dataset = makeDatasetWithRole(role = role) + val dataset = makeDatasetWithRole(role = role, sourceType = DatasetSourceType.File) datasetSaved = datasetApiService.createDataset(organizationSaved.id!!, dataset) val fileName = this::class.java.getResource("/integrationTest.zip")?.file val file = File(fileName!!) @@ -1830,9 +1831,9 @@ class DatasetServiceRBACTest : CsmRedisTestBase() { val organization = makeOrganizationWithRole(role = role) organizationSaved = organizationApiService.registerOrganization(organization) - val dataset = makeDatasetWithRole(role = ROLE_ADMIN) + val dataset = + makeDatasetWithRole(role = ROLE_ADMIN, sourceType = DatasetSourceType.None) datasetSaved = datasetApiService.createDataset(organizationSaved.id!!, dataset) - materializeTwingraph() every { getCurrentAccountIdentifier(any()) } returns TEST_USER_MAIL @@ -2294,7 +2295,7 @@ class DatasetServiceRBACTest : CsmRedisTestBase() { createTwingraph: Boolean = true ): Dataset { dataset.apply { - if (createTwingraph) { + if (createTwingraph && !this.twingraphId.isNullOrBlank()) { jedis.graphQuery(this.twingraphId, "CREATE (n:labelrouge)") } this.ingestionStatus = IngestionStatusEnum.SUCCESS @@ -2314,7 +2315,7 @@ class DatasetServiceRBACTest : CsmRedisTestBase() { fun makeDataset( id: String, name: String, - sourceType: DatasetSourceType = DatasetSourceType.File + sourceType: DatasetSourceType = DatasetSourceType.Twincache ): Dataset { return Dataset( id = id, @@ -2365,16 +2366,17 @@ class DatasetServiceRBACTest : CsmRedisTestBase() { parentId: String = "", id: String = TEST_USER_MAIL, role: String = ROLE_ADMIN, - sourceType: DatasetSourceType = DatasetSourceType.File + sourceType: DatasetSourceType = DatasetSourceType.Twincache ): Dataset { + val random = UUID.randomUUID().toString() return Dataset( - id = UUID.randomUUID().toString(), + id = random, name = "My datasetRbac", organizationId = organizationId, parentId = parentId, ownerId = "ownerId", connector = DatasetConnector(connectorSaved.id!!), - twingraphId = "graph", + twingraphId = "graph-${random}", source = SourceInfo("location", "name", "path"), tags = mutableListOf("dataset"), sourceType = sourceType, diff --git a/dataset/src/integrationTest/resources/application-dataset-test.yml b/dataset/src/integrationTest/resources/application-dataset-test.yml index f6daef654..04310360c 100644 --- a/dataset/src/integrationTest/resources/application-dataset-test.yml +++ b/dataset/src/integrationTest/resources/application-dataset-test.yml @@ -143,6 +143,7 @@ csm: host: "localhost" port: "6379" username: "default" + useGraphModule: true # Leave it as blank as there's no auth with test container password: dataset: diff --git a/dataset/src/main/kotlin/com/cosmotech/dataset/service/DatasetServiceImpl.kt b/dataset/src/main/kotlin/com/cosmotech/dataset/service/DatasetServiceImpl.kt index b0ed0105b..9fc6f5010 100644 --- a/dataset/src/main/kotlin/com/cosmotech/dataset/service/DatasetServiceImpl.kt +++ b/dataset/src/main/kotlin/com/cosmotech/dataset/service/DatasetServiceImpl.kt @@ -87,6 +87,8 @@ import kotlinx.coroutines.launch import org.apache.commons.compress.archivers.ArchiveStreamFactory import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVRecord +import org.apache.commons.lang3.NotImplementedException +import org.springframework.beans.factory.annotation.Value import org.springframework.context.event.EventListener import org.springframework.core.io.ByteArrayResource import org.springframework.core.io.Resource @@ -130,6 +132,13 @@ class DatasetServiceImpl( private val resourceScanner: ResourceScanner ) : CsmPhoenixService(), DatasetApiServiceInterface { + @Value("\${csm.platform.twincache.useGraphModule}") private var useGraphModule: Boolean = true + + private val notImplementedExceptionMessage = + "The API is not configured to use Graph functionnalities. " + + "This endpoint is deactivated. " + + "To change that, set the API configuration entry 'csm.platform.twincache.useGraphModule' to true" + override fun findAllDatasets(organizationId: String, page: Int?, size: Int?): List { organizationService.getVerifiedOrganization(organizationId) val defaultPageSize = csmPlatformProperties.twincache.dataset.defaultPageSize @@ -181,15 +190,19 @@ class DatasetServiceImpl( dataset.takeUnless { it.name.isNullOrBlank() } ?: throw IllegalArgumentException("Name cannot be null or blank") + val datasetSourceType = dataset.sourceType dataset.takeUnless { - dataset.sourceType in listOf(DatasetSourceType.ADT, DatasetSourceType.AzureStorage) && + datasetSourceType in listOf(DatasetSourceType.ADT, DatasetSourceType.AzureStorage) && dataset.source == null } ?: throw IllegalArgumentException( "Source cannot be null for source type 'ADT' or 'Storage'") - val twingraphId = idGenerator.generate("twingraph") - if (dataset.sourceType != null) { + var twingraphId: String? = null + + if (datasetSourceType == DatasetSourceType.Twincache && useGraphModule) { + + twingraphId = idGenerator.generate("twingraph") val twincacheConnector = getCreateTwincacheConnector() dataset.connector = DatasetConnector( @@ -197,17 +210,10 @@ class DatasetServiceImpl( parametersValues = mutableMapOf(TWINCACHE_NAME to twingraphId)) } - dataset.takeUnless { it.connector == null || dataset.connector!!.id.isNullOrBlank() } - ?: throw IllegalArgumentException("Connector or its ID cannot be null or blank") - - val existingConnector = connectorService.findConnectorById(dataset.connector!!.id!!) - logger.debug("Found connector: {}", existingConnector) - val createdDataset = dataset.copy( id = idGenerator.generate("dataset"), - twingraphId = twingraphId, - sourceType = dataset.sourceType ?: DatasetSourceType.None, + sourceType = datasetSourceType ?: DatasetSourceType.None, source = dataset.source ?: SourceInfo("none"), main = dataset.main ?: true, creationDate = Instant.now().toEpochMilli(), @@ -215,10 +221,21 @@ class DatasetServiceImpl( twincacheStatus = TwincacheStatusEnum.EMPTY, ownerId = getCurrentAuthenticatedUserName(csmPlatformProperties), organizationId = organizationId) + createdDataset.apply { + if (!twingraphId.isNullOrBlank()) { + this.twingraphId = twingraphId + } + } createdDataset.setRbac(csmRbac.initSecurity(dataset.getRbac())) - createdDataset.connector!!.apply { - name = existingConnector.name - version = existingConnector.version + + if (dataset.connector != null && !dataset.connector!!.id.isNullOrBlank()) { + val existingConnector = connectorService.findConnectorById(dataset.connector!!.id!!) + logger.debug("Found connector: {}", existingConnector) + + createdDataset.connector!!.apply { + name = existingConnector.name + version = existingConnector.version + } } return datasetRepository.save(createdDataset) @@ -229,6 +246,9 @@ class DatasetServiceImpl( datasetId: String, subDatasetGraphQuery: SubDatasetGraphQuery ): Dataset { + + checkIfGraphFunctionalityIsAvailable() + val dataset = getDatasetWithStatus(organizationId, datasetId, status = IngestionStatusEnum.SUCCESS) csmRbac.verify(dataset.getRbac(), PERMISSION_CREATE_CHILDREN) @@ -284,6 +304,12 @@ class DatasetServiceImpl( return datasetSaved } + private fun checkIfGraphFunctionalityIsAvailable() { + if (!useGraphModule) { + throw NotImplementedException(notImplementedExceptionMessage) + } + } + fun bulkQueryResult(queryBuffer: QueryBuffer, resultSet: ResultSet) { resultSet.forEach { record: Record? -> @@ -312,6 +338,8 @@ class DatasetServiceImpl( datasetId: String, body: Resource ): FileUploadValidation { + + checkIfGraphFunctionalityIsAvailable() val dataset = getDatasetWithStatus(organizationId, datasetId) csmRbac.verify(dataset.getRbac(), PERMISSION_WRITE) @@ -400,7 +428,7 @@ class DatasetServiceImpl( null -> IngestionStatusEnum.NONE.value DatasetSourceType.None -> { var twincacheStatus = TwincacheStatusEnum.EMPTY - if (unifiedJedis.exists(dataset.twingraphId!!)) { + if (useGraphModule && unifiedJedis.exists(dataset.twingraphId!!)) { twincacheStatus = TwincacheStatusEnum.FULL } datasetRepository.apply { dataset.twincacheStatus = twincacheStatus } @@ -413,7 +441,7 @@ class DatasetServiceImpl( } if (dataset.ingestionStatus == IngestionStatusEnum.ERROR) { return IngestionStatusEnum.ERROR.value - } else if (!unifiedJedis.exists(dataset.twingraphId!!)) { + } else if (useGraphModule && !unifiedJedis.exists(dataset.twingraphId!!)) { IngestionStatusEnum.PENDING.value } else { dataset @@ -483,6 +511,7 @@ class DatasetServiceImpl( } override fun refreshDataset(organizationId: String, datasetId: String): DatasetTwinGraphInfo { + checkIfGraphFunctionalityIsAvailable() val dataset = getVerifiedDataset(organizationId, datasetId, PERMISSION_WRITE) dataset.takeUnless { it.sourceType == DatasetSourceType.File } @@ -527,6 +556,7 @@ class DatasetServiceImpl( } override fun rollbackRefresh(organizationId: String, datasetId: String): String { + checkIfGraphFunctionalityIsAvailable() var dataset = getVerifiedDataset(organizationId, datasetId, PERMISSION_WRITE) val status = getDatasetTwingraphStatus(organizationId, datasetId) @@ -548,7 +578,7 @@ class DatasetServiceImpl( override fun deleteDataset(organizationId: String, datasetId: String) { val dataset = getVerifiedDataset(organizationId, datasetId, PERMISSION_DELETE) - if (unifiedJedis.exists(dataset.twingraphId!!)) { + if (useGraphModule && unifiedJedis.exists(dataset.twingraphId!!)) { unifiedJedis.del(dataset.twingraphId!!) } @@ -618,6 +648,7 @@ class DatasetServiceImpl( datasetId: String, datasetTwinGraphQuery: DatasetTwinGraphQuery ): List { + checkIfGraphFunctionalityIsAvailable() val dataset = getDatasetWithStatus(organizationId, datasetId, status = IngestionStatusEnum.SUCCESS) @@ -679,6 +710,7 @@ class DatasetServiceImpl( twinGraphQuery: DatasetTwinGraphQuery, body: Resource ): TwinGraphBatchResult { + checkIfGraphFunctionalityIsAvailable() val dataset = getDatasetWithStatus(organizationId, datasetId) csmRbac.verify(dataset.getRbac(), PERMISSION_WRITE) resourceScanner.scanMimeTypes(body, listOf("text/csv", "text/plain")) @@ -702,6 +734,7 @@ class DatasetServiceImpl( datasetId: String, datasetTwinGraphQuery: DatasetTwinGraphQuery ): DatasetTwinGraphHash { + checkIfGraphFunctionalityIsAvailable() val dataset = getDatasetWithStatus(organizationId, datasetId, status = IngestionStatusEnum.SUCCESS) val bulkQueryKey = bulkQueryKey(dataset.twingraphId!!, datasetTwinGraphQuery.query, null) @@ -724,6 +757,7 @@ class DatasetServiceImpl( } override fun downloadTwingraph(organizationId: String, hash: String): Resource { + checkIfGraphFunctionalityIsAvailable() organizationService.getVerifiedOrganization(organizationId) val bulkQueryId = bulkQueryKey(hash) @@ -774,6 +808,7 @@ class DatasetServiceImpl( type: String, graphProperties: List ): String { + checkIfGraphFunctionalityIsAvailable() val dataset = getDatasetWithStatus(organizationId, datasetId) csmRbac.verify(dataset.getRbac(), PERMISSION_WRITE) var result = "" @@ -808,6 +843,7 @@ class DatasetServiceImpl( type: String, ids: List ): String { + checkIfGraphFunctionalityIsAvailable() val dataset = getDatasetWithStatus(organizationId, datasetId) var result = "" when (type) { @@ -903,6 +939,7 @@ class DatasetServiceImpl( type: String, graphProperties: List ): String { + checkIfGraphFunctionalityIsAvailable() val dataset = getDatasetWithStatus(organizationId, datasetId) csmRbac.verify(dataset.getRbac(), PERMISSION_WRITE) var result = "" @@ -937,6 +974,7 @@ class DatasetServiceImpl( type: String, ids: List ) { + checkIfGraphFunctionalityIsAvailable() val dataset = getDatasetWithStatus(organizationId, datasetId) csmRbac.verify(dataset.getRbac(), PERMISSION_WRITE) return trx(dataset) { localDataset -> diff --git a/dataset/src/main/openapi/dataset.yaml b/dataset/src/main/openapi/dataset.yaml index 11c8b096f..bcde5f9ce 100644 --- a/dataset/src/main/openapi/dataset.yaml +++ b/dataset/src/main/openapi/dataset.yaml @@ -293,6 +293,7 @@ paths: * The id of the edge All following columns content are up to you. + Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true requestBody: required: true content: @@ -336,7 +337,9 @@ paths: tags: - dataset summary: Create a sub-dataset from the dataset in parameter - description: Create a copy of the dataset using the results of the list of queries given in parameter. + description: | + Create a copy of the dataset using the results of the list of queries given in parameter. + Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true requestBody: description: the Cypher query to filter required: true @@ -388,6 +391,7 @@ paths: Local File (import a new file) During refresh, datas are overwritten + Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true responses: '200': description: Successful response @@ -414,7 +418,9 @@ paths: tags: - dataset summary: Rollback the dataset after a failed refresh - description: Rollback the twingraph on a dataset after a failed refresh + description: | + Rollback the twingraph on a dataset after a failed refresh + Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true responses: '200': description: Successful response @@ -445,7 +451,9 @@ paths: tags: - dataset summary: Return the result of a query made on the graph instance as a json - description: Run a query on a graph instance and return the result as a json + description: | + Run a query on a graph instance and return the result as a json + Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true requestBody: description: the query to run required: true @@ -494,7 +502,9 @@ paths: tags: - dataset summary: Create new entities in a graph instance - description: create new entities in a graph instance + description: | + Create new entities in a graph instance + Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true requestBody: description: the entities to create required: true @@ -519,7 +529,9 @@ paths: tags: - dataset summary: Get entities in a graph instance - description: get entities in a graph instance + description: | + Get entities in a graph instance + Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true parameters: - name: ids in: query @@ -543,7 +555,9 @@ paths: summary: Update entities in a graph instance description: update entities in a graph instance requestBody: - description: the entities to update + description: | + The entities to update + Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true required: true content: application/json: @@ -566,7 +580,9 @@ paths: tags: - dataset summary: Delete entities in a graph instance - description: delete entities in a graph instance + description: | + Delete entities in a graph instance + Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true parameters: - name: ids in: query @@ -632,7 +648,9 @@ paths: tags: - dataset summary: "Async batch update by loading a CSV file on a graph instance " - description: "Async batch update by loading a CSV file on a graph instance " + description: | + Async batch update by loading a CSV file on a graph instance + Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true requestBody: required: true content: @@ -684,7 +702,9 @@ paths: tags: - dataset summary: Run a query on a graph instance and return the result as a zip file in async mode - description: Run a query on a graph instance and return the result as a zip file in async mode + description: | + Run a query on a graph instance and return the result as a zip file in async mode + Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true requestBody: description: the query to run required: true @@ -721,7 +741,9 @@ paths: tags: - dataset summary: Download a graph as a zip file - description: Download the compressed graph reference by the hash in a zip file + description: | + Download the compressed graph reference by the hash in a zip file + Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true responses: '200': description: Successful response diff --git a/dataset/src/test/kotlin/com/cosmotech/dataset/service/DatasetServiceImplTests.kt b/dataset/src/test/kotlin/com/cosmotech/dataset/service/DatasetServiceImplTests.kt index 531ed1b8c..54da8e008 100644 --- a/dataset/src/test/kotlin/com/cosmotech/dataset/service/DatasetServiceImplTests.kt +++ b/dataset/src/test/kotlin/com/cosmotech/dataset/service/DatasetServiceImplTests.kt @@ -168,17 +168,6 @@ class DatasetServiceImplTests { } } } - @Test - fun `createDataset should throw IllegalArgumentException when connector is empty`() { - val dataset = baseDataset() - every { - organizationService.getVerifiedOrganization(ORGANIZATION_ID, PERMISSION_CREATE_CHILDREN) - } returns Organization() - every { datasetRepository.save(any()) } returnsArgument 0 - assertThrows { - datasetService.createDataset(ORGANIZATION_ID, dataset) - } - } @Test fun `createSubDataset create Dataset copy with new id, name, description, parentId & twingraphId`() { diff --git a/doc/Apis/DatasetApi.md b/doc/Apis/DatasetApi.md index 04ef4a6ab..df93023e9 100644 --- a/doc/Apis/DatasetApi.md +++ b/doc/Apis/DatasetApi.md @@ -151,7 +151,7 @@ Create a new Dataset Create a sub-dataset from the dataset in parameter - Create a copy of the dataset using the results of the list of queries given in parameter. + Create a copy of the dataset using the results of the list of queries given in parameter. Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true ### Parameters @@ -180,7 +180,7 @@ Create a sub-dataset from the dataset in parameter Create new entities in a graph instance - create new entities in a graph instance + Create new entities in a graph instance Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true ### Parameters @@ -236,7 +236,7 @@ null (empty response body) Delete entities in a graph instance - delete entities in a graph instance + Delete entities in a graph instance Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true ### Parameters @@ -266,7 +266,7 @@ null (empty response body) Download a graph as a zip file - Download the compressed graph reference by the hash in a zip file + Download the compressed graph reference by the hash in a zip file Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true ### Parameters @@ -454,7 +454,7 @@ Get the dataset's refresh job status Get entities in a graph instance - get entities in a graph instance + Get entities in a graph instance Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true ### Parameters @@ -511,7 +511,7 @@ Get entities in a graph instance Refresh data on dataset from dataset's source - Refresh dataset from parent source. At date, sources can be: dataset (refresh from another dataset) Azure Digital twin Azure storage Local File (import a new file) During refresh, datas are overwritten + Refresh dataset from parent source. At date, sources can be: dataset (refresh from another dataset) Azure Digital twin Azure storage Local File (import a new file) During refresh, datas are overwritten Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true ### Parameters @@ -592,7 +592,7 @@ null (empty response body) Rollback the dataset after a failed refresh - Rollback the twingraph on a dataset after a failed refresh + Rollback the twingraph on a dataset after a failed refresh Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true ### Parameters @@ -675,7 +675,7 @@ Set the Dataset default security Run a query on a graph instance and return the result as a zip file in async mode - Run a query on a graph instance and return the result as a zip file in async mode + Run a query on a graph instance and return the result as a zip file in async mode Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true ### Parameters @@ -704,7 +704,7 @@ Run a query on a graph instance and return the result as a zip file in async mod Async batch update by loading a CSV file on a graph instance - Async batch update by loading a CSV file on a graph instance + Async batch update by loading a CSV file on a graph instance Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true ### Parameters @@ -734,7 +734,7 @@ Async batch update by loading a CSV file on a graph instance Return the result of a query made on the graph instance as a json - Run a query on a graph instance and return the result as a json + Run a query on a graph instance and return the result as a json Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true ### Parameters @@ -854,7 +854,7 @@ Update entities in a graph instance | **organization\_id** | **String**| the Organization identifier | [default to null] | | **dataset\_id** | **String**| the Dataset Identifier | [default to null] | | **type** | **String**| the entity model type | [default to null] [enum: node, relationship] | -| **GraphProperties** | [**List**](../Models/GraphProperties.md)| the entities to update | | +| **GraphProperties** | [**List**](../Models/GraphProperties.md)| The entities to update Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true | | ### Return type @@ -875,7 +875,7 @@ Update entities in a graph instance Upload data from zip file to dataset's twingraph - To create a new graph from flat files, you need to create a Zip file. This Zip file must countain two folders named Edges and Nodes. .zip hierarchy: *main_folder/Nodes *main_folder/Edges In each folder you can place one or multiple csv files containing your Nodes or Edges data. Your csv files must follow the following header (column name) requirements: The Nodes CSVs requires at least one column (the 1st).Column name = 'id'. It will represent the nodes ID Ids must be populated with string The Edges CSVs require three columns named, in order, * source * target * id those colomns represent * The source of the edge * The target of the edge * The id of the edge All following columns content are up to you. + To create a new graph from flat files, you need to create a Zip file. This Zip file must countain two folders named Edges and Nodes. .zip hierarchy: *main_folder/Nodes *main_folder/Edges In each folder you can place one or multiple csv files containing your Nodes or Edges data. Your csv files must follow the following header (column name) requirements: The Nodes CSVs requires at least one column (the 1st).Column name = 'id'. It will represent the nodes ID Ids must be populated with string The Edges CSVs require three columns named, in order, * source * target * id those colomns represent * The source of the edge * The target of the edge * The id of the edge All following columns content are up to you. Note: This endpoint is activated only if `csm.platform.twincache.useGraphModule` property is set to true ### Parameters diff --git a/run/src/integrationTest/resources/application-run-test.yml b/run/src/integrationTest/resources/application-run-test.yml index a0156924e..32f757226 100644 --- a/run/src/integrationTest/resources/application-run-test.yml +++ b/run/src/integrationTest/resources/application-run-test.yml @@ -127,6 +127,9 @@ csm: port: "6379" username: "default" password: "my-wonderful-password" + useGraphModule: true + run: + default-page-size: 5 scenariorun: default-page-size: 5 internalResultServices: diff --git a/runner/src/integrationTest/kotlin/com/cosmotech/runner/service/RunnerServiceIntegrationTest.kt b/runner/src/integrationTest/kotlin/com/cosmotech/runner/service/RunnerServiceIntegrationTest.kt index 68120291e..34807f43a 100644 --- a/runner/src/integrationTest/kotlin/com/cosmotech/runner/service/RunnerServiceIntegrationTest.kt +++ b/runner/src/integrationTest/kotlin/com/cosmotech/runner/service/RunnerServiceIntegrationTest.kt @@ -1012,7 +1012,7 @@ class RunnerServiceIntegrationTest : CsmRedisTestBase() { createTwingraph: Boolean = true ): Dataset { dataset.apply { - if (createTwingraph) { + if (createTwingraph && !this.twingraphId.isNullOrBlank()) { jedis.graphQuery(this.twingraphId, "MATCH (n:labelrouge) return 1") } this.ingestionStatus = IngestionStatusEnum.SUCCESS diff --git a/runner/src/integrationTest/kotlin/com/cosmotech/runner/service/RunnerServiceRBACTest.kt b/runner/src/integrationTest/kotlin/com/cosmotech/runner/service/RunnerServiceRBACTest.kt index b3d2105d0..f373b7387 100644 --- a/runner/src/integrationTest/kotlin/com/cosmotech/runner/service/RunnerServiceRBACTest.kt +++ b/runner/src/integrationTest/kotlin/com/cosmotech/runner/service/RunnerServiceRBACTest.kt @@ -4529,7 +4529,7 @@ class RunnerServiceRBACTest : CsmRedisTestBase() { private fun materializeTwingraph(dataset: Dataset, createTwingraph: Boolean = true): Dataset { dataset.apply { - if (createTwingraph) { + if (createTwingraph && !this.twingraphId.isNullOrBlank()) { jedis.graphQuery(this.twingraphId, "CREATE (n:labelrouge)") } this.ingestionStatus = IngestionStatusEnum.SUCCESS diff --git a/runner/src/integrationTest/resources/application-runner-test.yml b/runner/src/integrationTest/resources/application-runner-test.yml index 0f1fbe5a8..f24d9d9c3 100644 --- a/runner/src/integrationTest/resources/application-runner-test.yml +++ b/runner/src/integrationTest/resources/application-runner-test.yml @@ -115,6 +115,9 @@ csm: port: "6379" username: "default" password: "my-wonderful-password" + useGraphModule: true + runner: + default-page-size: 20 scenario: default-page-size: 20 rbac: diff --git a/scenario/src/integrationTest/kotlin/com/cosmotech/scenario/service/ScenarioServiceIntegrationTest.kt b/scenario/src/integrationTest/kotlin/com/cosmotech/scenario/service/ScenarioServiceIntegrationTest.kt index 7a7f5b69f..d4abd96b1 100644 --- a/scenario/src/integrationTest/kotlin/com/cosmotech/scenario/service/ScenarioServiceIntegrationTest.kt +++ b/scenario/src/integrationTest/kotlin/com/cosmotech/scenario/service/ScenarioServiceIntegrationTest.kt @@ -22,13 +22,7 @@ import com.cosmotech.connector.ConnectorApiServiceInterface import com.cosmotech.connector.domain.Connector import com.cosmotech.connector.domain.IoTypesEnum import com.cosmotech.dataset.DatasetApiServiceInterface -import com.cosmotech.dataset.domain.Dataset -import com.cosmotech.dataset.domain.DatasetAccessControl -import com.cosmotech.dataset.domain.DatasetConnector -import com.cosmotech.dataset.domain.DatasetSecurity -import com.cosmotech.dataset.domain.IngestionStatusEnum -import com.cosmotech.dataset.domain.SubDatasetGraphQuery -import com.cosmotech.dataset.domain.TwincacheStatusEnum +import com.cosmotech.dataset.domain.* import com.cosmotech.dataset.repository.DatasetRepository import com.cosmotech.dataset.service.getRbac import com.cosmotech.organization.OrganizationApiServiceInterface @@ -161,6 +155,7 @@ class ScenarioServiceIntegrationTest : CsmRedisTestBase() { makeWorkspaceEventHubInfo(false) rediSearchIndexer.createIndexFor(Organization::class.java) + rediSearchIndexer.createIndexFor(Connector::class.java) rediSearchIndexer.createIndexFor(Dataset::class.java) rediSearchIndexer.createIndexFor(Solution::class.java) rediSearchIndexer.createIndexFor(Workspace::class.java) @@ -1113,13 +1108,15 @@ class ScenarioServiceIntegrationTest : CsmRedisTestBase() { fun makeDataset( organizationId: String = organizationSaved.id!!, name: String = "name", - connector: Connector = connectorSaved + connector: Connector = connectorSaved, + sourceType: DatasetSourceType = DatasetSourceType.Twincache ): Dataset { return Dataset( name = name, organizationId = organizationId, creationDate = Instant.now().toEpochMilli(), ownerId = "ownerId", + sourceType = sourceType, connector = DatasetConnector( id = connector.id, diff --git a/scenario/src/integrationTest/kotlin/com/cosmotech/scenario/service/ScenarioServiceRBACTest.kt b/scenario/src/integrationTest/kotlin/com/cosmotech/scenario/service/ScenarioServiceRBACTest.kt index 39ff458c0..db6744f8b 100644 --- a/scenario/src/integrationTest/kotlin/com/cosmotech/scenario/service/ScenarioServiceRBACTest.kt +++ b/scenario/src/integrationTest/kotlin/com/cosmotech/scenario/service/ScenarioServiceRBACTest.kt @@ -28,11 +28,7 @@ import com.cosmotech.connector.api.ConnectorApiService import com.cosmotech.connector.domain.Connector import com.cosmotech.connector.domain.IoTypesEnum import com.cosmotech.dataset.api.DatasetApiService -import com.cosmotech.dataset.domain.Dataset -import com.cosmotech.dataset.domain.DatasetAccessControl -import com.cosmotech.dataset.domain.DatasetConnector -import com.cosmotech.dataset.domain.DatasetSecurity -import com.cosmotech.dataset.domain.IngestionStatusEnum +import com.cosmotech.dataset.domain.* import com.cosmotech.dataset.repository.DatasetRepository import com.cosmotech.organization.api.OrganizationApiService import com.cosmotech.organization.domain.Organization @@ -131,6 +127,7 @@ class ScenarioServiceRBACTest : CsmRedisTestBase() { rediSearchIndexer.createIndexFor(Dataset::class.java) rediSearchIndexer.createIndexFor(Workspace::class.java) rediSearchIndexer.createIndexFor(Scenario::class.java) + rediSearchIndexer.createIndexFor(Connector::class.java) val context = getContext(redisStackServer) val containerIp = @@ -7083,12 +7080,19 @@ class ScenarioServiceRBACTest : CsmRedisTestBase() { ioTypes = listOf(IoTypesEnum.read)) } - fun makeDataset(organizationId: String, connector: Connector, id: String, role: String): Dataset { + fun makeDataset( + organizationId: String, + connector: Connector, + id: String, + role: String, + sourceType: DatasetSourceType = DatasetSourceType.Twincache + ): Dataset { return Dataset( name = "Dataset", organizationId = organizationId, ownerId = "ownerId", ingestionStatus = IngestionStatusEnum.SUCCESS, + sourceType = sourceType, connector = DatasetConnector( id = connector.id, diff --git a/scenario/src/integrationTest/resources/application-scenario-test.yml b/scenario/src/integrationTest/resources/application-scenario-test.yml index bf3efc4aa..22872f1b1 100644 --- a/scenario/src/integrationTest/resources/application-scenario-test.yml +++ b/scenario/src/integrationTest/resources/application-scenario-test.yml @@ -145,6 +145,7 @@ csm: username: "default" # Leave it as blank as there's no auth with test container password: + useGraphModule: true scenario: default-page-size: 20 internalResultServices: diff --git a/scenariorun/src/integrationTest/resources/application-scenariorun-test.yml b/scenariorun/src/integrationTest/resources/application-scenariorun-test.yml index 80bdf1cb8..20783aa3a 100644 --- a/scenariorun/src/integrationTest/resources/application-scenariorun-test.yml +++ b/scenariorun/src/integrationTest/resources/application-scenariorun-test.yml @@ -147,6 +147,7 @@ csm: username: "default" # Leave it as blank as there's no auth with test container password: + useGraphModule: true scenariorun: default-page-size: 20 rbac: diff --git a/workspace/src/integrationTest/resources/application-workspace-test.yml b/workspace/src/integrationTest/resources/application-workspace-test.yml index eafb119b5..23ce6c41d 100644 --- a/workspace/src/integrationTest/resources/application-workspace-test.yml +++ b/workspace/src/integrationTest/resources/application-workspace-test.yml @@ -145,6 +145,7 @@ csm: username: "default" # Leave it as blank as there's no auth with test container password: + useGraphModule: true workspace: default-page-size: 5 rbac: