add jobId to dataset source to simplify get status

Author: lalepee

dataset/src/integrationTest/kotlin/com/cosmotech/dataset/service/DatasetServiceIntegrationTest.kt

@@ -350,8 +350,8 @@ class DatasetServiceIntegrationTest : CsmRedisTestBase() {
     datasetApiService.uploadTwingraph(organizationSaved.id!!, datasetSaved.id!!, resource)
     // add timout for while loop
     val timeout = Instant.now()
-    while (datasetApiService.getDatasetTwingraphStatus(
-        organizationSaved.id!!, datasetSaved.id!!, "randomJobId") != Dataset.Status.READY.value) {
+    while (datasetApiService.getDatasetTwingraphStatus(organizationSaved.id!!, datasetSaved.id!!) !=
+        Dataset.Status.READY.value) {
       if (Instant.now().minusSeconds(10).isAfter(timeout)) {
         throw Exception("Timeout while waiting for dataset twingraph to be ready")
       }
@@ -983,15 +983,15 @@ class DatasetServiceIntegrationTest : CsmRedisTestBase() {
                 val exception =
                     assertThrows<CsmAccessForbiddenException> {
                       datasetApiService.getDatasetTwingraphStatus(
-                          organizationSaved.id!!, datasetSaved.id!!, "randomJobId")
+                          organizationSaved.id!!, datasetSaved.id!!)
                     }
                 assertEquals(
                     "RBAC ${datasetSaved.id!!} - User does not have permission $PERMISSION_READ",
                     exception.message)
               } else {
                 assertDoesNotThrow {
                   datasetApiService.getDatasetTwingraphStatus(
-                      organizationSaved.id!!, datasetSaved.id!!, "randomJobId")
+                      organizationSaved.id!!, datasetSaved.id!!)
                 }
               }
             }

dataset/src/main/kotlin/com/cosmotech/dataset/service/DatasetServiceImpl.kt

@@ -270,7 +270,7 @@ class DatasetServiceImpl(
 
     dataset.sourceType.takeIf { it == DatasetSourceType.File }
         ?: throw CsmResourceNotFoundException("SourceType Dataset must be 'File'")
-    val uploadStatus = getDatasetTwingraphStatus(organizationId, datasetId, "jobIdRandom")
+    val uploadStatus = getDatasetTwingraphStatus(organizationId, datasetId)
     uploadStatus.takeUnless { it == Dataset.Status.PENDING.value }
         ?: throw CsmResourceNotFoundException("Dataset in use, cannot update. Retry later")
 
@@ -304,7 +304,6 @@ class DatasetServiceImpl(
   override fun getDatasetTwingraphStatus(
       organizationId: String,
       datasetId: String,
-      jobId: String
   ): String {
     // This call verify by itself that we have the read authorization in the dataset
     val dataset = findDatasetById(organizationId, datasetId)
@@ -330,7 +329,7 @@ class DatasetServiceImpl(
       DatasetSourceType.Twincache,
       DatasetSourceType.AzureStorage -> {
         val twingraphImportJobInfoRequest =
-            TwingraphImportJobInfoRequest(this, jobId!!, organizationId)
+            TwingraphImportJobInfoRequest(this, dataset.source!!.jobId!!, organizationId)
         this.eventPublisher.publishEvent(twingraphImportJobInfoRequest)
         dataset
             .takeIf { it.status == Dataset.Status.PENDING }
@@ -384,6 +383,8 @@ class DatasetServiceImpl(
             "",
             dataset.queries)
     this.eventPublisher.publishEvent(graphImportEvent)
+
+    datasetRepository.save(dataset.apply { source!!.jobId = requestJobId })
     logger.debug("refreshDataset={}", graphImportEvent.response)
     return DatasetTwinGraphInfo(
         jobId = requestJobId, datasetId = dataset.id, status = dataset.status?.value)

dataset/src/main/openapi/dataset.yaml

@@ -678,7 +678,7 @@ paths:
                 type: string
                 format: binary
 
-  /organizations/{organization_id}/datasets/{dataset_id}/job/{job_id}/status:
+  /organizations/{organization_id}/datasets/{dataset_id}/status:
     parameters:
       - name: organization_id
         in: path
@@ -692,12 +692,6 @@ paths:
         required: true
         schema:
           type: string
-      - name: job_id
-        in: path
-        description: the job identifier
-        required: true
-        schema:
-          type: string
     get:
       operationId: getDatasetTwingraphStatus
       tags:
@@ -1623,6 +1617,9 @@ components:
         path:
           type: string
           description: the source location containing the files to import
+        jobId:
+          type: string
+          description: indicate the last import jobId
       required:
         - location
     TwinGraphBatchResult:

dataset/src/test/kotlin/com/cosmotech/dataset/service/DatasetServiceImplTests.kt

@@ -319,7 +319,7 @@ class DatasetServiceImplTests {
                 twingraphId = "twingraphId")
     every { datasetRepository.findById(DATASET_ID) } returns Optional.of(dataset)
     every { csmJedisPool.resource.exists(any<String>()) } returns false
-    val result = datasetService.getDatasetTwingraphStatus(ORGANIZATION_ID, DATASET_ID, "JOB_ID")
+    val result = datasetService.getDatasetTwingraphStatus(ORGANIZATION_ID, DATASET_ID)
     assertEquals(Dataset.Status.DRAFT.value, result)
   }
 
@@ -333,8 +333,7 @@ class DatasetServiceImplTests {
                 twingraphId = "twingraphId")
     every { datasetRepository.findById(DATASET_ID) } returns Optional.of(dataset)
     every { csmJedisPool.resource.exists(any<String>()) } returns true
-    val result =
-        datasetService.getDatasetTwingraphStatus(ORGANIZATION_ID, DATASET_ID, "JobIdRandom")
+    val result = datasetService.getDatasetTwingraphStatus(ORGANIZATION_ID, DATASET_ID)
     assertEquals(Dataset.Status.READY.value, result)
   }
 
@@ -345,12 +344,13 @@ class DatasetServiceImplTests {
             .copy(
                 status = Dataset.Status.DRAFT,
                 sourceType = DatasetSourceType.ADT,
+                source = SourceInfo(location = "test", jobId = "0"),
                 twingraphId = "twingraphId")
     mockkConstructor(TwingraphImportJobInfoRequest::class)
     every { anyConstructed<TwingraphImportJobInfoRequest>().response } returns
         Dataset.Status.PENDING.value
     every { datasetRepository.findById(DATASET_ID) } returns Optional.of(dataset)
-    val result = datasetService.getDatasetTwingraphStatus(ORGANIZATION_ID, DATASET_ID, "JOB_ID")
+    val result = datasetService.getDatasetTwingraphStatus(ORGANIZATION_ID, DATASET_ID)
     assertEquals(Dataset.Status.PENDING.value, result)
   }
 
@@ -361,14 +361,15 @@ class DatasetServiceImplTests {
             .copy(
                 status = Dataset.Status.PENDING,
                 sourceType = DatasetSourceType.ADT,
+                source = SourceInfo(location = "test", jobId = "0"),
                 twingraphId = "twingraphId")
     mockkConstructor(TwingraphImportJobInfoRequest::class)
     every { anyConstructed<TwingraphImportJobInfoRequest>().response } returns
         Dataset.Status.READY.value
     every { datasetRepository.findById(DATASET_ID) } returns Optional.of(dataset)
     every { csmJedisPool.resource.exists(any<String>()) } returns true
     every { datasetRepository.save(any()) } returnsArgument 0
-    val result = datasetService.getDatasetTwingraphStatus(ORGANIZATION_ID, DATASET_ID, "JOB_ID")
+    val result = datasetService.getDatasetTwingraphStatus(ORGANIZATION_ID, DATASET_ID)
     assertEquals(Dataset.Status.READY.value, result)
   }
 
@@ -391,13 +392,13 @@ class DatasetServiceImplTests {
             .copy(
                 status = Dataset.Status.DRAFT,
                 sourceType = DatasetSourceType.ADT,
-                source = SourceInfo("http://storage.location"),
+                source = SourceInfo("http://storage.location", jobId = "0"),
                 twingraphId = "twingraphId")
     every { datasetRepository.findById(DATASET_ID) } returns Optional.of(dataset)
     every { csmJedisPool.resource.exists(any<String>()) } returns true
     every { datasetRepository.save(any()) } returnsArgument 0
     val datasetInfo = datasetService.refreshDataset(ORGANIZATION_ID, DATASET_ID)
-    verify(exactly = 1) { datasetRepository.save(any()) }
+    verify(atLeast = 1) { datasetRepository.save(any()) }
     assertEquals(dataset.id, datasetInfo.datasetId)
     assertEquals(dataset.status!!.value, datasetInfo.status)
   }

doc/Apis/DatasetApi.md

@@ -6,32 +6,32 @@ Method | HTTP request | Description
 ------------- | ------------- | -------------
 [**addDatasetAccessControl**](DatasetApi.md#addDatasetAccessControl) | **POST** /organizations/{organization_id}/datasets/{dataset_id}/security/access | Add a control access to the Dataset
 [**addOrReplaceDatasetCompatibilityElements**](DatasetApi.md#addOrReplaceDatasetCompatibilityElements) | **POST** /organizations/{organization_id}/datasets/{dataset_id}/compatibility | Add Dataset Compatibility elements.
-[**copyDataset**](DatasetApi.md#copyDataset) | **POST** /organizations/{organization_id}/datasets/copy | Copy a Dataset to another Dataset. Source must have a read capable connector and Target a write capable connector.
+[**copyDataset**](DatasetApi.md#copyDataset) | **POST** /organizations/{organization_id}/datasets/copy | Copy a Dataset to another Dataset.
 [**createDataset**](DatasetApi.md#createDataset) | **POST** /organizations/{organization_id}/datasets | Create a new Dataset
-[**createSubDataset**](DatasetApi.md#createSubDataset) | **POST** /organizations/{organization_id}/datasets/{dataset_id}/subdataset | Run a query on a dataset
+[**createSubDataset**](DatasetApi.md#createSubDataset) | **POST** /organizations/{organization_id}/datasets/{dataset_id}/subdataset | Create a sub-dataset from the dataset in parameter
 [**createTwingraphEntities**](DatasetApi.md#createTwingraphEntities) | **POST** /organizations/{organization_id}/datasets/{dataset_id}/twingraph/{type} | Create new entities in a graph instance
 [**deleteDataset**](DatasetApi.md#deleteDataset) | **DELETE** /organizations/{organization_id}/datasets/{dataset_id} | Delete a dataset
 [**deleteTwingraphEntities**](DatasetApi.md#deleteTwingraphEntities) | **DELETE** /organizations/{organization_id}/datasets/{dataset_id}/twingraph/{type} | Delete entities in a graph instance
-[**downloadTwingraph**](DatasetApi.md#downloadTwingraph) | **GET** /organizations/{organization_id}/datasets/twingraph/download/{hash} | Download a graph compressed in a zip file
+[**downloadTwingraph**](DatasetApi.md#downloadTwingraph) | **GET** /organizations/{organization_id}/datasets/twingraph/download/{hash} | Download a graph as a zip file
 [**findAllDatasets**](DatasetApi.md#findAllDatasets) | **GET** /organizations/{organization_id}/datasets | List all Datasets
 [**findDatasetById**](DatasetApi.md#findDatasetById) | **GET** /organizations/{organization_id}/datasets/{dataset_id} | Get the details of a Dataset
 [**getDatasetAccessControl**](DatasetApi.md#getDatasetAccessControl) | **GET** /organizations/{organization_id}/datasets/{dataset_id}/security/access/{identity_id} | Get a control access for the Dataset
 [**getDatasetSecurity**](DatasetApi.md#getDatasetSecurity) | **GET** /organizations/{organization_id}/datasets/{dataset_id}/security | Get the Dataset security information
 [**getDatasetSecurityUsers**](DatasetApi.md#getDatasetSecurityUsers) | **GET** /organizations/{organization_id}/datasets/{dataset_id}/security/users | Get the Dataset security users list
-[**getDatasetTwingraphStatus**](DatasetApi.md#getDatasetTwingraphStatus) | **GET** /organizations/{organization_id}/datasets/{dataset_id}/job/{job_id}/status | Get the status of twingraph import
+[**getDatasetTwingraphStatus**](DatasetApi.md#getDatasetTwingraphStatus) | **GET** /organizations/{organization_id}/datasets/{dataset_id}/status | Get the dataset's refresh job status
 [**getTwingraphEntities**](DatasetApi.md#getTwingraphEntities) | **GET** /organizations/{organization_id}/datasets/{dataset_id}/twingraph/{type} | Get entities in a graph instance
-[**refreshDataset**](DatasetApi.md#refreshDataset) | **POST** /organizations/{organization_id}/datasets/{dataset_id}/refresh | Refresh dataset
+[**refreshDataset**](DatasetApi.md#refreshDataset) | **POST** /organizations/{organization_id}/datasets/{dataset_id}/refresh | Refresh data on dataset from dataset's source
 [**removeAllDatasetCompatibilityElements**](DatasetApi.md#removeAllDatasetCompatibilityElements) | **DELETE** /organizations/{organization_id}/datasets/{dataset_id}/compatibility | Remove all Dataset Compatibility elements from the Dataset specified
 [**removeDatasetAccessControl**](DatasetApi.md#removeDatasetAccessControl) | **DELETE** /organizations/{organization_id}/datasets/{dataset_id}/security/access/{identity_id} | Remove the specified access from the given Dataset
-[**searchDatasets**](DatasetApi.md#searchDatasets) | **POST** /organizations/{organization_id}/datasets/search | Search Datasets
+[**searchDatasets**](DatasetApi.md#searchDatasets) | **POST** /organizations/{organization_id}/datasets/search | Search Datasets by tags
 [**setDatasetDefaultSecurity**](DatasetApi.md#setDatasetDefaultSecurity) | **POST** /organizations/{organization_id}/datasets/{dataset_id}/security/default | Set the Dataset default security
 [**twingraphBatchQuery**](DatasetApi.md#twingraphBatchQuery) | **POST** /organizations/{organization_id}/datasets/{dataset_id}/batch-query | Run a query on a graph instance and return the result as a zip file in async mode
 [**twingraphBatchUpdate**](DatasetApi.md#twingraphBatchUpdate) | **POST** /organizations/{organization_id}/datasets/{dataset_id}/batch | Async batch update by loading a CSV file on a graph instance 
-[**twingraphQuery**](DatasetApi.md#twingraphQuery) | **POST** /organizations/{organization_id}/datasets/{dataset_id}/twingraph | Run a query on a graph instance and return the result as a json
+[**twingraphQuery**](DatasetApi.md#twingraphQuery) | **POST** /organizations/{organization_id}/datasets/{dataset_id}/twingraph | Return the result of a query made on the graph instance as a json
 [**updateDataset**](DatasetApi.md#updateDataset) | **PATCH** /organizations/{organization_id}/datasets/{dataset_id} | Update a dataset
 [**updateDatasetAccessControl**](DatasetApi.md#updateDatasetAccessControl) | **PATCH** /organizations/{organization_id}/datasets/{dataset_id}/security/access/{identity_id} | Update the specified access to User for a Dataset
 [**updateTwingraphEntities**](DatasetApi.md#updateTwingraphEntities) | **PATCH** /organizations/{organization_id}/datasets/{dataset_id}/twingraph/{type} | Update entities in a graph instance
-[**uploadTwingraph**](DatasetApi.md#uploadTwingraph) | **POST** /organizations/{organization_id}/datasets/{dataset_id} | Upload Twingraph with ZIP File
+[**uploadTwingraph**](DatasetApi.md#uploadTwingraph) | **POST** /organizations/{organization_id}/datasets/{dataset_id} | Upload data from zip file to dataset's twingraph
 
 
 <a name="addDatasetAccessControl"></a>
@@ -92,7 +92,9 @@ Name | Type | Description  | Notes
 # **copyDataset**
 > DatasetCopyParameters copyDataset(organization\_id, DatasetCopyParameters)
 
-Copy a Dataset to another Dataset. Source must have a read capable connector and Target a write capable connector.
+Copy a Dataset to another Dataset.
+
+    Not implemented!
 
 ### Parameters
 
@@ -144,17 +146,17 @@ Name | Type | Description  | Notes
 # **createSubDataset**
 > Dataset createSubDataset(organization\_id, dataset\_id, SubDatasetGraphQuery)
 
-Run a query on a dataset
+Create a sub-dataset from the dataset in parameter
 
-    Run a query on a dataset
+    Create a copy of the dataset using the results of the list of queries given in parameter.
 
 ### Parameters
 
 Name | Type | Description  | Notes
 ------------- | ------------- | ------------- | -------------
  **organization\_id** | **String**| the Organization identifier | [default to null]
  **dataset\_id** | **String**| the Dataset identifier | [default to null]
- **SubDatasetGraphQuery** | [**SubDatasetGraphQuery**](../Models/SubDatasetGraphQuery.md)| the query to run |
+ **SubDatasetGraphQuery** | [**SubDatasetGraphQuery**](../Models/SubDatasetGraphQuery.md)| the Cypher query to filter |
 
 ### Return type
 
@@ -259,9 +261,9 @@ null (empty response body)
 # **downloadTwingraph**
 > File downloadTwingraph(organization\_id, hash)
 
-Download a graph compressed in a zip file
+Download a graph as a zip file
 
-    Download a graph compressed in a zip file
+    Download the compressed graph reference by the hash in a zip file
 
 ### Parameters
 
@@ -417,19 +419,18 @@ Name | Type | Description  | Notes
 
 <a name="getDatasetTwingraphStatus"></a>
 # **getDatasetTwingraphStatus**
-> String getDatasetTwingraphStatus(organization\_id, dataset\_id, job\_id)
+> String getDatasetTwingraphStatus(organization\_id, dataset\_id)
 
-Get the status of twingraph import
+Get the dataset&#39;s refresh job status
 
-    Get the status of a twingraph import
+    Get the status of the import workflow lauch on the dataset&#39;s refresh.
 
 ### Parameters
 
 Name | Type | Description  | Notes
 ------------- | ------------- | ------------- | -------------
  **organization\_id** | **String**| the Organization identifier | [default to null]
  **dataset\_id** | **String**| the dataset identifier | [default to null]
- **job\_id** | **String**| the job identifier | [default to null]
 
 ### Return type
 
@@ -478,9 +479,9 @@ Name | Type | Description  | Notes
 # **refreshDataset**
 > DatasetTwinGraphInfo refreshDataset(organization\_id, dataset\_id)
 
-Refresh dataset
+Refresh data on dataset from dataset&#39;s source
 
-    Refresh ADT, Storage dataset
+    Lauch a import from source (ADT or Azure Storage). This replace currently stored data with just extracted data from source.
 
 ### Parameters
 
@@ -559,7 +560,7 @@ null (empty response body)
 # **searchDatasets**
 > List searchDatasets(organization\_id, DatasetSearch, page, size)
 
-Search Datasets
+Search Datasets by tags
 
 ### Parameters
 
@@ -673,7 +674,7 @@ Name | Type | Description  | Notes
 # **twingraphQuery**
 > String twingraphQuery(organization\_id, dataset\_id, DatasetTwinGraphQuery)
 
-Run a query on a graph instance and return the result as a json
+Return the result of a query made on the graph instance as a json
 
     Run a query on a graph instance and return the result as a json
 
@@ -787,9 +788,9 @@ Name | Type | Description  | Notes
 # **uploadTwingraph**
 > uploadTwingraph(organization\_id, dataset\_id, body)
 
-Upload Twingraph with ZIP File
+Upload data from zip file to dataset&#39;s twingraph
 
-    Upload Twingraph ZIP
+    To create a new graph from flat files,  you need to create a Zip file. This Zip file must countain two folders named Edges and Nodes.  .zip hierarchy: *main_folder/Nodes *main_folder/Edges  In each folder you can place one or multiple csv files containing your Nodes or Edges data.  Your csv files must follow the following header (column name) requirements:  The Nodes CSVs requires at least one column (the 1st).Column name &#x3D; &#39;id&#39;. It will represent the nodes ID Ids must be populated with string  The Edges CSVs require three columns named, in order, * source * target * id  those colomns represent * The source of the edge * The target of the edge * The id of the edge  All following columns content are up to you. 
 
 ### Parameters
 

doc/Models/SourceInfo.md

@@ -6,6 +6,7 @@ Name | Type | Description | Notes
 **name** | **String** | the source name containing the files to import | [optional] [default to null]
 **location** | **String** | the source location containing the files to import | [default to null]
 **path** | **String** | the source location containing the files to import | [optional] [default to null]
+**jobId** | **String** | indicate the last import jobId | [optional] [default to null]
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)