Skip to content

Commit 27180dc

Browse files
sjoubertsjoubert
committed
[PROD-14915] Add free-from additionalData to Dataset and DatasetPart
1 parent 69fed68 commit 27180dc

File tree

12 files changed

+79
-6
lines changed

12 files changed

+79
-6
lines changed

‎api/src/integrationTest/kotlin/com/cosmotech/api/home/ControllerTestUtils.kt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import com.cosmotech.organization.domain.OrganizationSecurity
2525
import com.cosmotech.runner.domain.*
2626
import com.cosmotech.solution.domain.*
2727
import com.cosmotech.workspace.domain.*
28+
import kotlin.collections.mutableListOf
2829
import org.apache.commons.io.IOUtils
2930
import org.json.JSONObject
3031
import org.springframework.http.MediaType
@@ -445,13 +446,15 @@ class ControllerTestUtils {
445446
name = name,
446447
description = DATASET_DESCRIPTION,
447448
tags = mutableListOf("tag1", "tag2"),
449+
additionalData = mutableMapOf("customClient" to "customDatasetData"),
448450
runnerId = "r-12345678910",
449451
parts =
450452
mutableListOf(
451453
DatasetPartCreateRequest(
452454
name = datasetPartName,
453455
description = DATASET_PART_DESCRIPTION,
454456
tags = mutableListOf("tag_part1", "tag_part2"),
457+
additionalData = mutableMapOf("customClient" to "customDatasetPartData"),
455458
type = type,
456459
sourceName = sourceName)),
457460
security = security)
@@ -465,6 +468,7 @@ class ControllerTestUtils {
465468
name = name,
466469
description = DATASET_PART_DESCRIPTION,
467470
tags = mutableListOf("tag_part1", "tag_part3"),
471+
additionalData = mutableMapOf("customClient" to "customDatasetPartData"),
468472
type = type,
469473
sourceName = TEST_FILE_NAME)
470474
}

‎api/src/integrationTest/kotlin/com/cosmotech/api/home/workspace/WorkspaceControllerTests.kt‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,7 @@ class WorkspaceControllerTests : ControllerTestBase() {
6666
"runtemplateId1" to "datasetId1", "runtemplateId2" to "datasetId2")
6767
val additionalData =
6868
mutableMapOf(
69-
"you_can_put" to "whatever_you_want_here",
70-
"even" to mapOf("object" to "if_you_want"))
69+
"you_can_put" to "whatever_you_want_here", "even" to mapOf("object" to "if_you_want"))
7170
val workspaceSecurity =
7271
WorkspaceSecurity(
7372
default = ROLE_NONE,
@@ -148,8 +147,7 @@ class WorkspaceControllerTests : ControllerTestBase() {
148147
"runtemplateId1" to "datasetId1", "runtemplateId2" to "datasetId2")
149148
val additionalData =
150149
mutableMapOf(
151-
"you_can_put" to "whatever_you_want_here",
152-
"even" to mapOf("object" to "if_you_want"))
150+
"you_can_put" to "whatever_you_want_here", "even" to mapOf("object" to "if_you_want"))
153151

154152
val workspaceDatasetId = "d-12345678910"
155153
mvc.perform(
@@ -275,8 +273,7 @@ class WorkspaceControllerTests : ControllerTestBase() {
275273
"runtemplateId1" to "datasetId1", "runtemplateId2" to "datasetId2")
276274
val additionalData =
277275
mutableMapOf(
278-
"you_can_put" to "whatever_you_want_here",
279-
"even" to mapOf("object" to "if_you_want"))
276+
"you_can_put" to "whatever_you_want_here", "even" to mapOf("object" to "if_you_want"))
280277

281278
val workspaceDatasetId = "d-12345678910"
282279
val workspaceSecurity =

‎dataset/src/integrationTest/kotlin/com/cosmotech/dataset/service/DatasetServiceIntegrationTest.kt‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,22 +169,28 @@ class DatasetServiceIntegrationTest() : CsmTestBase() {
169169
val datasetPartName = "Customers list"
170170
val datasetPartDescription = "List of customers"
171171
val datasetPartTags = mutableListOf("part", "public", "customers")
172+
val datasetPartAdditionalData =
173+
mutableMapOf("part" to "data", "complex" to mutableMapOf("nested" to "data"))
172174
val datasetPartCreateRequest =
173175
DatasetPartCreateRequest(
174176
name = datasetPartName,
175177
sourceName = CUSTOMER_SOURCE_FILE_NAME,
176178
description = datasetPartDescription,
177179
tags = datasetPartTags,
180+
additionalData = datasetPartAdditionalData,
178181
type = DatasetPartTypeEnum.File)
179182

180183
val datasetName = "Customer Dataset"
181184
val datasetDescription = "Dataset for customers"
182185
val datasetTags = mutableListOf("dataset", "public", "customers")
186+
val datasetAdditionalData =
187+
mutableMapOf("dataset" to "data", "complex" to mutableMapOf("nested" to "data"))
183188
val datasetCreateRequest =
184189
DatasetCreateRequest(
185190
name = datasetName,
186191
description = datasetDescription,
187192
tags = datasetTags,
193+
additionalData = datasetAdditionalData,
188194
parts = mutableListOf(datasetPartCreateRequest))
189195

190196
val resourceTestFile = resourceLoader.getResource("classpath:/$CUSTOMER_SOURCE_FILE_NAME").file
@@ -218,12 +224,14 @@ class DatasetServiceIntegrationTest() : CsmTestBase() {
218224
assertEquals(datasetName, createdDataset.name)
219225
assertEquals(datasetDescription, createdDataset.description)
220226
assertEquals(datasetTags, createdDataset.tags)
227+
assertEquals(datasetAdditionalData, createdDataset.additionalData)
221228
assertEquals(1, createdDataset.parts.size)
222229
val createdDatasetPart = createdDataset.parts[0]
223230
assertNotNull(createdDatasetPart)
224231
assertEquals(datasetPartName, createdDatasetPart.name)
225232
assertEquals(datasetPartDescription, createdDatasetPart.description)
226233
assertEquals(datasetPartTags, createdDatasetPart.tags)
234+
assertEquals(datasetPartAdditionalData, createdDatasetPart.additionalData)
227235
assertEquals(CUSTOMER_SOURCE_FILE_NAME, createdDatasetPart.sourceName)
228236
assertEquals(DatasetPartTypeEnum.File, createdDatasetPart.type)
229237
}
@@ -980,6 +988,8 @@ class DatasetServiceIntegrationTest() : CsmTestBase() {
980988
val datasetPartName = "Customer list"
981989
val datasetPartDescription = "List of customers"
982990
val datasetPartTags = mutableListOf("part", "public", "customers")
991+
val datasetPartAdditionalData =
992+
mutableMapOf("part" to "data", "complex" to mutableMapOf("nested" to "data"))
983993

984994
val createDatasetPart =
985995
datasetApiService.createDatasetPart(
@@ -992,12 +1002,14 @@ class DatasetServiceIntegrationTest() : CsmTestBase() {
9921002
sourceName = CUSTOMER_SOURCE_FILE_NAME,
9931003
description = datasetPartDescription,
9941004
tags = datasetPartTags,
1005+
additionalData = datasetPartAdditionalData,
9951006
type = DatasetPartTypeEnum.File))
9961007

9971008
assertNotNull(createDatasetPart)
9981009
assertEquals(datasetPartName, createDatasetPart.name)
9991010
assertEquals(datasetPartDescription, createDatasetPart.description)
10001011
assertEquals(datasetPartTags, createDatasetPart.tags)
1012+
assertEquals(datasetPartAdditionalData, createDatasetPart.additionalData)
10011013
assertEquals(CUSTOMER_SOURCE_FILE_NAME, createDatasetPart.sourceName)
10021014

10031015
val retrievedDataset =
@@ -1882,22 +1894,28 @@ class DatasetServiceIntegrationTest() : CsmTestBase() {
18821894
val datasetPartName = "Customers list"
18831895
val datasetPartDescription = "List of customers"
18841896
val datasetPartTags = mutableListOf("part", "public", "customers")
1897+
val datasetPartAdditionalData =
1898+
mutableMapOf("part" to "data", "complex" to mutableMapOf("nested" to "data"))
18851899
val datasetPartCreateRequest =
18861900
DatasetPartCreateRequest(
18871901
name = datasetPartName,
18881902
sourceName = CUSTOMER_SOURCE_FILE_NAME,
18891903
description = datasetPartDescription,
18901904
tags = datasetPartTags,
1905+
additionalData = datasetPartAdditionalData,
18911906
type = DatasetPartTypeEnum.File)
18921907

18931908
val datasetName = "Customer Dataset"
18941909
val datasetDescription = "Dataset for customers"
18951910
val datasetTags = mutableListOf("dataset", "public", "customers")
1911+
val datasetAdditionalData =
1912+
mutableMapOf("dataset" to "data", "complex" to mutableMapOf("nested" to "data"))
18961913
val datasetCreateRequest =
18971914
DatasetCreateRequest(
18981915
name = datasetName,
18991916
description = datasetDescription,
19001917
tags = datasetTags,
1918+
additionalData = datasetAdditionalData,
19011919
parts = mutableListOf(datasetPartCreateRequest))
19021920

19031921
val resourceTestFile = resourceLoader.getResource("classpath:/$CUSTOMER_SOURCE_FILE_NAME").file
@@ -1922,17 +1940,20 @@ class DatasetServiceIntegrationTest() : CsmTestBase() {
19221940
val newDatasetPartName = "Product list"
19231941
val newDatasetPartDescription = "List of Product"
19241942
val newDatasetPartTags = mutableListOf("part", "public", "product")
1943+
val newDatasetPartAdditionalData = mutableMapOf<String, Any>("part" to "new data")
19251944
val newDatasetPartCreateRequest =
19261945
DatasetPartCreateRequest(
19271946
name = newDatasetPartName,
19281947
sourceName = INVENTORY_SOURCE_FILE_NAME,
19291948
description = newDatasetPartDescription,
19301949
tags = newDatasetPartTags,
1950+
additionalData = newDatasetPartAdditionalData,
19311951
type = DatasetPartTypeEnum.File)
19321952

19331953
val newDatasetName = "Shop Dataset"
19341954
val newDatasetDescription = "Dataset for shop"
19351955
val newDatasetTags = mutableListOf("dataset", "public", "shop")
1956+
val newDatasetAdditionalData = mutableMapOf<String, Any>("dataset" to "new data")
19361957
val newDatasetSecurity =
19371958
DatasetSecurity(
19381959
default = ROLE_NONE,
@@ -1945,6 +1966,7 @@ class DatasetServiceIntegrationTest() : CsmTestBase() {
19451966
name = newDatasetName,
19461967
description = newDatasetDescription,
19471968
tags = newDatasetTags,
1969+
additionalData = newDatasetAdditionalData,
19481970
parts = mutableListOf(newDatasetPartCreateRequest),
19491971
security = newDatasetSecurity)
19501972

@@ -2256,12 +2278,15 @@ class DatasetServiceIntegrationTest() : CsmTestBase() {
22562278
val customerPartName = "Customers list"
22572279
val customerPartDescription = "List of customers"
22582280
val customerPartTags = mutableListOf("part", "public", "customers")
2281+
val customerPartAdditionalData =
2282+
mutableMapOf("part" to "data", "complex" to mutableMapOf("nested" to "data"))
22592283
val customerPartCreateRequest =
22602284
DatasetPartCreateRequest(
22612285
name = customerPartName,
22622286
sourceName = CUSTOMER_SOURCE_FILE_NAME,
22632287
description = customerPartDescription,
22642288
tags = customerPartTags,
2289+
additionalData = customerPartAdditionalData,
22652290
type = DatasetPartTypeEnum.File)
22662291

22672292
val datasetName = "Shop Dataset"
@@ -2315,18 +2340,21 @@ class DatasetServiceIntegrationTest() : CsmTestBase() {
23152340
assertEquals(customerPartName, datasetPartToReplace.name)
23162341
assertEquals(customerPartDescription, datasetPartToReplace.description)
23172342
assertEquals(customerPartTags, datasetPartToReplace.tags)
2343+
assertEquals(customerPartAdditionalData, datasetPartToReplace.additionalData)
23182344
assertEquals(CUSTOMER_SOURCE_FILE_NAME, datasetPartToReplace.sourceName)
23192345
assertEquals(DatasetPartTypeEnum.File, datasetPartToReplace.type)
23202346

23212347
// New Part to replace the existing one in the dataset
23222348
val newDatasetSourceName = "updatedResourceFile.csv"
23232349
val newDatasetPartDescription = "New Data for customer list"
23242350
val newDatasetPartTags = mutableListOf("part", "public", "new", "customer")
2351+
val newDatasetPartAdditionalData = mutableMapOf<String, Any>("part" to "new data")
23252352
val datasetPartUpdateRequest =
23262353
DatasetPartUpdateRequest(
23272354
sourceName = newDatasetSourceName,
23282355
description = newDatasetPartDescription,
23292356
tags = newDatasetPartTags,
2357+
additionalData = newDatasetPartAdditionalData,
23302358
)
23312359

23322360
val replacedDatasetPart =
@@ -2347,6 +2375,7 @@ class DatasetServiceIntegrationTest() : CsmTestBase() {
23472375
assertEquals(newDatasetSourceName, replacedDatasetPart.sourceName)
23482376
assertEquals(newDatasetPartDescription, replacedDatasetPart.description)
23492377
assertEquals(newDatasetPartTags, replacedDatasetPart.tags)
2378+
assertEquals(newDatasetPartAdditionalData, replacedDatasetPart.additionalData)
23502379
assertEquals(newDatasetSourceName, replacedDatasetPart.sourceName)
23512380
assertEquals(DatasetPartTypeEnum.File, replacedDatasetPart.type)
23522381

‎dataset/src/main/kotlin/com/cosmotech/dataset/service/DatasetServiceImpl.kt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,7 @@ class DatasetServiceImpl(
179179
organizationId = organizationId,
180180
workspaceId = workspaceId,
181181
tags = datasetCreateRequest.tags ?: mutableListOf(),
182+
additionalData = datasetCreateRequest.additionalData ?: mutableMapOf(),
182183
parts = datasetParts ?: mutableListOf(),
183184
createInfo = createInfo,
184185
updateInfo = editInfo,
@@ -336,6 +337,7 @@ class DatasetServiceImpl(
336337
organizationId = organizationId,
337338
workspaceId = workspaceId,
338339
tags = datasetUpdateRequest.tags ?: previousDataset.tags,
340+
additionalData = datasetUpdateRequest.additionalData ?: previousDataset.additionalData,
339341
parts = newDatasetParts ?: previousDataset.parts,
340342
createInfo = previousDataset.createInfo,
341343
updateInfo =
@@ -462,6 +464,7 @@ class DatasetServiceImpl(
462464
name = datasetPartCreateRequest.name,
463465
description = datasetPartCreateRequest.description,
464466
tags = datasetPartCreateRequest.tags ?: mutableListOf(),
467+
additionalData = datasetPartCreateRequest.additionalData ?: mutableMapOf(),
465468
type = datasetPartCreateRequest.type ?: DatasetPartTypeEnum.File,
466469
organizationId = organizationId,
467470
workspaceId = workspaceId,
@@ -807,13 +810,16 @@ class DatasetServiceImpl(
807810
it.sourceName = datasetPartUpdateRequest.sourceName ?: it.sourceName
808811
it.description = datasetPartUpdateRequest.description ?: it.description
809812
it.tags = datasetPartUpdateRequest.tags ?: it.tags
813+
it.additionalData = datasetPartUpdateRequest.additionalData ?: it.additionalData
810814
it.updateInfo = editInfo
811815
}
812816

813817
val datasetPartUpdater = datasetPart.copy()
814818
datasetPartUpdater.sourceName = datasetPartUpdateRequest.sourceName ?: datasetPart.sourceName
815819
datasetPartUpdater.description = datasetPartUpdateRequest.description ?: datasetPart.description
816820
datasetPartUpdater.tags = datasetPartUpdateRequest.tags ?: datasetPart.tags
821+
datasetPartUpdater.additionalData =
822+
datasetPartUpdateRequest.additionalData ?: datasetPart.additionalData
817823
datasetPartUpdater.updateInfo = editInfo
818824

819825
datasetRepository.update(dataset)
@@ -849,6 +855,7 @@ class DatasetServiceImpl(
849855
it.sourceName = datasetPartUpdateRequest?.sourceName ?: it.sourceName
850856
it.description = datasetPartUpdateRequest?.description ?: it.description
851857
it.tags = datasetPartUpdateRequest?.tags ?: it.tags
858+
it.additionalData = datasetPartUpdateRequest?.additionalData ?: it.additionalData
852859
it.updateInfo = editInfo
853860
}
854861

‎dataset/src/main/openapi/dataset.yaml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -788,6 +788,10 @@ components:
788788
description: the list of tags
789789
items:
790790
type: string
791+
additionalData:
792+
type: object
793+
description: Free form additional data
794+
additionalProperties: true
791795
parts:
792796
type: array
793797
items:
@@ -830,6 +834,10 @@ components:
830834
items:
831835
type: string
832836
default: []
837+
additionalData:
838+
type: object
839+
description: Free form additional data
840+
additionalProperties: true
833841
parts:
834842
type: array
835843
items:
@@ -857,6 +865,10 @@ components:
857865
type: array
858866
items:
859867
type: string
868+
additionalData:
869+
type: object
870+
description: Free form additional data
871+
additionalProperties: true
860872
parts:
861873
type: array
862874
items:
@@ -890,6 +902,10 @@ components:
890902
type: array
891903
items:
892904
type: string
905+
additionalData:
906+
type: object
907+
description: Free form additional data
908+
additionalProperties: true
893909
type:
894910
$ref: '#/components/schemas/DatasetPartTypeEnum'
895911
organizationId:
@@ -946,6 +962,10 @@ components:
946962
items:
947963
type: string
948964
default: []
965+
additionalData:
966+
type: object
967+
description: Free form additional data
968+
additionalProperties: true
949969
type:
950970
$ref: '#/components/schemas/DatasetPartTypeEnum'
951971
required:
@@ -966,6 +986,10 @@ components:
966986
type: array
967987
items:
968988
type: string
989+
additionalData:
990+
type: object
991+
description: Free form additional data
992+
additionalProperties: true
969993

970994
DatasetPartTypeEnum:
971995
type: string

‎doc/Models/Dataset.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
| **organizationId** | **String** | the associated Organization Id | [default to null] |
1010
| **workspaceId** | **String** | the associated Workspace Id | [default to null] |
1111
| **tags** | **List** | the list of tags | [default to null] |
12+
| **additionalData** | [**Map**](AnyType.md) | Free form additional data | [optional] [default to null] |
1213
| **parts** | [**List**](DatasetPart.md) | | [default to null] |
1314
| **createInfo** | [**CreateInfo**](CreateInfo.md) | The details of the Dataset creation | [default to null] |
1415
| **updateInfo** | [**EditInfo**](EditInfo.md) | The details of the Dataset last update | [default to null] |

‎doc/Models/DatasetCreateRequest.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
| **name** | **String** | | [default to null] |
77
| **description** | **String** | | [optional] [default to null] |
88
| **tags** | **List** | | [optional] [default to []] |
9+
| **additionalData** | [**Map**](AnyType.md) | Free form additional data | [optional] [default to null] |
910
| **parts** | [**List**](DatasetPartCreateRequest.md) | | [optional] [default to []] |
1011
| **security** | [**DatasetSecurity**](DatasetSecurity.md) | | [optional] [default to null] |
1112
| **runnerId** | **String** | | [optional] [default to null] |

‎doc/Models/DatasetPart.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
| **sourceName** | **String** | the source data name (e.g. filename associated to the dataset part) | [default to null] |
99
| **description** | **String** | | [optional] [default to null] |
1010
| **tags** | **List** | | [default to null] |
11+
| **additionalData** | [**Map**](AnyType.md) | Free form additional data | [optional] [default to null] |
1112
| **type** | [**DatasetPartTypeEnum**](DatasetPartTypeEnum.md) | | [default to null] |
1213
| **organizationId** | **String** | the associated Organization Id | [default to null] |
1314
| **workspaceId** | **String** | the associated Workspace Id | [default to null] |

‎doc/Models/DatasetPartCreateRequest.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
| **sourceName** | **String** | the source data name (e.g. filename associated to the dataset part) | [default to null] |
88
| **description** | **String** | | [optional] [default to null] |
99
| **tags** | **List** | | [optional] [default to []] |
10+
| **additionalData** | [**Map**](AnyType.md) | Free form additional data | [optional] [default to null] |
1011
| **type** | [**DatasetPartTypeEnum**](DatasetPartTypeEnum.md) | | [optional] [default to null] |
1112

1213
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

‎doc/Models/DatasetPartUpdateRequest.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
| **sourceName** | **String** | the source data name (e.g. filename associated to the dataset part) | [optional] [default to null] |
77
| **description** | **String** | | [optional] [default to null] |
88
| **tags** | **List** | | [optional] [default to null] |
9+
| **additionalData** | [**Map**](AnyType.md) | Free form additional data | [optional] [default to null] |
910

1011
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
1112

0 commit comments

Comments
 (0)