Add useGraphModule Boolean

Author: jreynard-code

api/src/main/resources/application.yml

@@ -81,6 +81,7 @@ csm:
       port: "6379"
       username: "default_user"
       password: "default_password"
+      useGraphModule: true
       tls:
         enabled: false
         bundle: ""

dataset/src/integrationTest/kotlin/com/cosmotech/dataset/service/DatasetServiceIntegrationTest.kt

@@ -111,7 +111,7 @@ class DatasetServiceIntegrationTest : CsmRedisTestBase() {
   @Autowired lateinit var solutionApiService: SolutionApiServiceInterface
   @Autowired lateinit var workspaceApiService: WorkspaceApiServiceInterface
   @SpykBean @Autowired lateinit var csmPlatformProperties: CsmPlatformProperties
-  @MockK(relaxUnitFun = true) private lateinit var eventPublisher: CsmEventPublisher
+  @MockK(relaxUnitFun = true) lateinit var eventPublisher: CsmEventPublisher
 
   lateinit var connectorSaved: Connector
   lateinit var dataset: Dataset

dataset/src/integrationTest/resources/application-dataset-test.yml

@@ -90,6 +90,7 @@ csm:
       host: "localhost"
       port: "6379"
       username: "default"
+      useGraphModule: true
       # Leave it as blank as there's no auth with test container
       password:
       tls:

dataset/src/main/kotlin/com/cosmotech/dataset/service/DatasetServiceImpl.kt

@@ -87,6 +87,8 @@ import kotlinx.coroutines.launch
 import org.apache.commons.compress.archivers.ArchiveStreamFactory
 import org.apache.commons.csv.CSVFormat
 import org.apache.commons.csv.CSVRecord
+import org.apache.commons.lang3.NotImplementedException
+import org.springframework.beans.factory.annotation.Value
 import org.springframework.context.event.EventListener
 import org.springframework.core.io.ByteArrayResource
 import org.springframework.core.io.Resource
@@ -127,9 +129,15 @@ class DatasetServiceImpl(
     private val unifiedJedis: UnifiedJedis,
     private val csmRbac: CsmRbac,
     private val csmAdmin: CsmAdmin,
-    private val resourceScanner: ResourceScanner
+    private val resourceScanner: ResourceScanner,
+    @Value("\${csm.platform.twincache.useGraphModule}") private val useGraphModule: Boolean
 ) : CsmPhoenixService(), DatasetApiServiceInterface {
 
+  private val notImplementedExceptionMessage =
+      "The API is not configured to use Graph functionnalities. " +
+          "This endpoint is deactivated so. " +
+          "To change that, set the API configuration entry 'csm.platform.twincache.useGraphModule' to true"
+
   override fun findAllDatasets(organizationId: String, page: Int?, size: Int?): List<Dataset> {
     organizationService.getVerifiedOrganization(organizationId)
     val defaultPageSize = csmPlatformProperties.twincache.dataset.defaultPageSize
@@ -188,8 +196,10 @@ class DatasetServiceImpl(
         ?: throw IllegalArgumentException(
             "Source cannot be null for source type 'ADT' or 'Storage'")
 
-    val twingraphId = idGenerator.generate("twingraph")
-    if (dataset.sourceType != null) {
+    var twingraphId: String? = null
+
+    if (dataset.sourceType != null && useGraphModule) {
+      twingraphId = idGenerator.generate("twingraph")
       val twincacheConnector = getCreateTwincacheConnector()
       dataset.connector =
           DatasetConnector(
@@ -229,6 +239,9 @@ class DatasetServiceImpl(
       datasetId: String,
       subDatasetGraphQuery: SubDatasetGraphQuery
   ): Dataset {
+
+    checkIfGraphFunctionalityIsAvailable()
+
     val dataset =
         getDatasetWithStatus(organizationId, datasetId, status = IngestionStatusEnum.SUCCESS)
     csmRbac.verify(dataset.getRbac(), PERMISSION_CREATE_CHILDREN)
@@ -284,6 +297,12 @@ class DatasetServiceImpl(
     return datasetSaved
   }
 
+  private fun checkIfGraphFunctionalityIsAvailable() {
+    if (!useGraphModule) {
+      throw NotImplementedException(notImplementedExceptionMessage)
+    }
+  }
+
   fun bulkQueryResult(queryBuffer: QueryBuffer, resultSet: ResultSet) {
 
     resultSet.forEach { record: Record? ->
@@ -312,6 +331,8 @@ class DatasetServiceImpl(
       datasetId: String,
       body: Resource
   ): FileUploadValidation {
+
+    checkIfGraphFunctionalityIsAvailable()
     val dataset = getDatasetWithStatus(organizationId, datasetId)
     csmRbac.verify(dataset.getRbac(), PERMISSION_WRITE)
 
@@ -400,7 +421,7 @@ class DatasetServiceImpl(
       null -> IngestionStatusEnum.NONE.value
       DatasetSourceType.None -> {
         var twincacheStatus = TwincacheStatusEnum.EMPTY
-        if (unifiedJedis.exists(dataset.twingraphId!!)) {
+        if (useGraphModule && unifiedJedis.exists(dataset.twingraphId!!)) {
           twincacheStatus = TwincacheStatusEnum.FULL
         }
         datasetRepository.apply { dataset.twincacheStatus = twincacheStatus }
@@ -413,7 +434,7 @@ class DatasetServiceImpl(
         }
         if (dataset.ingestionStatus == IngestionStatusEnum.ERROR) {
           return IngestionStatusEnum.ERROR.value
-        } else if (!unifiedJedis.exists(dataset.twingraphId!!)) {
+        } else if (useGraphModule && !unifiedJedis.exists(dataset.twingraphId!!)) {
           IngestionStatusEnum.PENDING.value
         } else {
           dataset
@@ -483,6 +504,7 @@ class DatasetServiceImpl(
   }
 
   override fun refreshDataset(organizationId: String, datasetId: String): DatasetTwinGraphInfo {
+    checkIfGraphFunctionalityIsAvailable()
     val dataset = getVerifiedDataset(organizationId, datasetId, PERMISSION_WRITE)
 
     dataset.takeUnless { it.sourceType == DatasetSourceType.File }
@@ -527,6 +549,7 @@ class DatasetServiceImpl(
   }
 
   override fun rollbackRefresh(organizationId: String, datasetId: String): String {
+    checkIfGraphFunctionalityIsAvailable()
     var dataset = getVerifiedDataset(organizationId, datasetId, PERMISSION_WRITE)
 
     val status = getDatasetTwingraphStatus(organizationId, datasetId)
@@ -556,7 +579,7 @@ class DatasetServiceImpl(
       throw CsmAccessForbiddenException("You are not allowed to delete this Resource")
     }
 
-    if (unifiedJedis.exists(dataset.twingraphId!!)) {
+    if (useGraphModule && unifiedJedis.exists(dataset.twingraphId!!)) {
       unifiedJedis.del(dataset.twingraphId!!)
     }
 
@@ -626,6 +649,7 @@ class DatasetServiceImpl(
       datasetId: String,
       datasetTwinGraphQuery: DatasetTwinGraphQuery
   ): List<Any> {
+    checkIfGraphFunctionalityIsAvailable()
     val dataset =
         getDatasetWithStatus(organizationId, datasetId, status = IngestionStatusEnum.SUCCESS)
 
@@ -686,6 +710,7 @@ class DatasetServiceImpl(
       twinGraphQuery: DatasetTwinGraphQuery,
       body: Resource
   ): TwinGraphBatchResult {
+    checkIfGraphFunctionalityIsAvailable()
     val dataset = getDatasetWithStatus(organizationId, datasetId)
     csmRbac.verify(dataset.getRbac(), PERMISSION_WRITE)
     resourceScanner.scanMimeTypes(body, listOf("text/csv", "text/plain"))
@@ -709,6 +734,7 @@ class DatasetServiceImpl(
       datasetId: String,
       datasetTwinGraphQuery: DatasetTwinGraphQuery
   ): DatasetTwinGraphHash {
+    checkIfGraphFunctionalityIsAvailable()
     val dataset =
         getDatasetWithStatus(organizationId, datasetId, status = IngestionStatusEnum.SUCCESS)
     val bulkQueryKey = bulkQueryKey(dataset.twingraphId!!, datasetTwinGraphQuery.query, null)
@@ -731,6 +757,7 @@ class DatasetServiceImpl(
   }
 
   override fun downloadTwingraph(organizationId: String, hash: String): Resource {
+    checkIfGraphFunctionalityIsAvailable()
     organizationService.getVerifiedOrganization(organizationId)
 
     val bulkQueryId = bulkQueryKey(hash)
@@ -781,6 +808,7 @@ class DatasetServiceImpl(
       type: String,
       graphProperties: List<GraphProperties>
   ): String {
+    checkIfGraphFunctionalityIsAvailable()
     val dataset = getDatasetWithStatus(organizationId, datasetId)
     csmRbac.verify(dataset.getRbac(), PERMISSION_WRITE)
     var result = ""
@@ -815,6 +843,7 @@ class DatasetServiceImpl(
       type: String,
       ids: List<String>
   ): String {
+    checkIfGraphFunctionalityIsAvailable()
     val dataset = getDatasetWithStatus(organizationId, datasetId)
     var result = ""
     when (type) {
@@ -910,6 +939,7 @@ class DatasetServiceImpl(
       type: String,
       graphProperties: List<GraphProperties>
   ): String {
+    checkIfGraphFunctionalityIsAvailable()
     val dataset = getDatasetWithStatus(organizationId, datasetId)
     csmRbac.verify(dataset.getRbac(), PERMISSION_WRITE)
     var result = ""
@@ -944,6 +974,7 @@ class DatasetServiceImpl(
       type: String,
       ids: List<String>
   ) {
+    checkIfGraphFunctionalityIsAvailable()
     val dataset = getDatasetWithStatus(organizationId, datasetId)
     csmRbac.verify(dataset.getRbac(), PERMISSION_WRITE)
     return trx(dataset) { localDataset ->