Skip to content

Commit 8e0b8aa

Browse files
jreynard-codejreynard-code
committed
Add support for updating dataset part information
- Introduced `updateDatasetPartInformation` API to allow modification of dataset part details (e.g., source name, description, tags) via PATCH request. - Added integration and RBAC tests validating permissions and functionality for updating dataset parts. - Updated OpenAPI specification, documentation, and YAML definitions to reflect the new API endpoint and response structure.
1 parent d0c14cd commit 8e0b8aa

File tree

7 files changed

+350
-16
lines changed

7 files changed

+350
-16
lines changed

‎api/src/integrationTest/kotlin/com/cosmotech/api/home/dataset/DatasetControllerTests.kt‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -736,6 +736,56 @@ class DatasetControllerTests : ControllerTestBase() {
736736
"organizations/{organization_id}/workspaces/{workspace_id}/datasets/{dataset_id}/parts/{dataset_part_id}/PUT"))
737737
}
738738

739+
@Test
740+
@WithMockOauth2User
741+
fun update_dataset_part() {
742+
743+
val datasetId =
744+
createDatasetAndReturnId(mvc, organizationId, workspaceId, constructDatasetCreateRequest())
745+
746+
val datasetPartId =
747+
createDatasetPartAndReturnId(
748+
mvc, organizationId, workspaceId, datasetId, constructDatasetPartCreateRequest())
749+
750+
val newSourceName = "source_name_updated.csv"
751+
val newDescription = "this_a_new_description_for_dataset_part"
752+
val newTags = mutableListOf("tag_part1_updated", "tag_part2_updated")
753+
754+
mvc.perform(
755+
patch(
756+
"/organizations/$organizationId/workspaces/$workspaceId/datasets/$datasetId/parts/$datasetPartId")
757+
.content(
758+
JSONObject(
759+
DatasetPartUpdateRequest(
760+
sourceName = newSourceName,
761+
description = newDescription,
762+
tags = newTags))
763+
.toString())
764+
.contentType(MediaType.APPLICATION_JSON)
765+
.accept(MediaType.APPLICATION_JSON)
766+
.with(csrf()))
767+
.andExpect(status().is2xxSuccessful)
768+
.andDo(MockMvcResultHandlers.print())
769+
.andExpect(jsonPath("$.id").value(datasetPartId))
770+
.andExpect(jsonPath("$.name").value(DATASET_PART_NAME))
771+
.andExpect(jsonPath("$.sourceName").value(newSourceName))
772+
.andExpect(jsonPath("$.description").value(newDescription))
773+
.andExpect(jsonPath("$.createInfo.userId").value(PLATFORM_ADMIN_EMAIL))
774+
.andExpect(jsonPath("$.createInfo.timestamp").isNumber)
775+
.andExpect(jsonPath("$.createInfo.timestamp").value(greaterThan(0.toLong())))
776+
.andExpect(jsonPath("$.updateInfo.userId").value(PLATFORM_ADMIN_EMAIL))
777+
.andExpect(jsonPath("$.updateInfo.timestamp").isNumber)
778+
.andExpect(jsonPath("$.updateInfo.timestamp").value(greaterThan(0.toLong())))
779+
.andExpect(jsonPath("$.tags").value(newTags))
780+
.andExpect(jsonPath("$.type").value(DatasetPartTypeEnum.File.value))
781+
.andExpect(jsonPath("$.organizationId").value(organizationId))
782+
.andExpect(jsonPath("$.workspaceId").value(workspaceId))
783+
.andExpect(jsonPath("$.datasetId").value(datasetId))
784+
.andDo(
785+
document(
786+
"organizations/{organization_id}/workspaces/{workspace_id}/datasets/{dataset_id}/parts/{dataset_part_id}/PATCH"))
787+
}
788+
739789
@Test
740790
@WithMockOauth2User
741791
fun update_dataset() {

‎dataset/src/integrationTest/kotlin/com/cosmotech/dataset/service/DatasetServiceIntegrationTest.kt‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1862,6 +1862,121 @@ class DatasetServiceIntegrationTest() : CsmTestBase() {
18621862
exception.message)
18631863
}
18641864

1865+
@Test
1866+
fun `test updateDatasetPart`() {
1867+
1868+
// Initiate Dataset with a customer dataset part (that will be replaced by the inventory part)
1869+
val customerPartName = "Customers list"
1870+
val customerPartDescription = "List of customers"
1871+
val customerPartTags = mutableListOf("part", "public", "customers")
1872+
val customerPartCreateRequest =
1873+
DatasetPartCreateRequest(
1874+
name = customerPartName,
1875+
sourceName = CUSTOMER_SOURCE_FILE_NAME,
1876+
description = customerPartDescription,
1877+
tags = customerPartTags,
1878+
type = DatasetPartTypeEnum.File)
1879+
1880+
val datasetName = "Shop Dataset"
1881+
val datasetDescription = "Dataset for shop"
1882+
val datasetTags = mutableListOf("dataset", "public", "shop")
1883+
val datasetCreateRequest =
1884+
DatasetCreateRequest(
1885+
name = datasetName,
1886+
description = datasetDescription,
1887+
tags = datasetTags,
1888+
parts = mutableListOf(customerPartCreateRequest))
1889+
1890+
val customerTestFile = resourceLoader.getResource("classpath:/$CUSTOMER_SOURCE_FILE_NAME").file
1891+
val customerFileToSend = FileInputStream(customerTestFile)
1892+
1893+
val customerMockMultipartFile =
1894+
MockMultipartFile(
1895+
"files",
1896+
CUSTOMER_SOURCE_FILE_NAME,
1897+
MediaType.MULTIPART_FORM_DATA_VALUE,
1898+
IOUtils.toByteArray(customerFileToSend))
1899+
1900+
val createdDataset =
1901+
datasetApiService.createDataset(
1902+
organizationSaved.id,
1903+
workspaceSaved.id,
1904+
datasetCreateRequest,
1905+
arrayOf(customerMockMultipartFile))
1906+
1907+
val customerPartFilePath = constructFilePathForDatasetPart(createdDataset, 0)
1908+
assertTrue(s3Template.objectExists(csmPlatformProperties.s3.bucketName, customerPartFilePath))
1909+
1910+
val customerDownloadFile =
1911+
s3Template.download(csmPlatformProperties.s3.bucketName, customerPartFilePath)
1912+
1913+
val customerExpectedText =
1914+
FileInputStream(customerTestFile).bufferedReader().use { it.readText() }
1915+
1916+
val customerRetrievedText =
1917+
InputStreamResource(customerDownloadFile).inputStream.bufferedReader().use { it.readText() }
1918+
1919+
assertEquals(customerExpectedText, customerRetrievedText)
1920+
1921+
assertNotNull(createdDataset)
1922+
assertEquals(datasetName, createdDataset.name)
1923+
assertEquals(datasetDescription, createdDataset.description)
1924+
assertEquals(datasetTags, createdDataset.tags)
1925+
assertEquals(1, createdDataset.parts.size)
1926+
val datasetPartToReplace = createdDataset.parts[0]
1927+
assertNotNull(datasetPartToReplace)
1928+
assertEquals(customerPartName, datasetPartToReplace.name)
1929+
assertEquals(customerPartDescription, datasetPartToReplace.description)
1930+
assertEquals(customerPartTags, datasetPartToReplace.tags)
1931+
assertEquals(CUSTOMER_SOURCE_FILE_NAME, datasetPartToReplace.sourceName)
1932+
assertEquals(DatasetPartTypeEnum.File, datasetPartToReplace.type)
1933+
1934+
// New Part to replace the existing one in the dataset
1935+
val newDatasetSourceName = "updatedResourceFile.csv"
1936+
val newDatasetPartDescription = "New Data for customer list"
1937+
val newDatasetPartTags = mutableListOf("part", "public", "new", "customer")
1938+
val datasetPartUpdateRequest =
1939+
DatasetPartUpdateRequest(
1940+
sourceName = newDatasetSourceName,
1941+
description = newDatasetPartDescription,
1942+
tags = newDatasetPartTags,
1943+
)
1944+
1945+
val replacedDatasetPart =
1946+
datasetApiService.updateDatasetPart(
1947+
organizationSaved.id,
1948+
workspaceSaved.id,
1949+
createdDataset.id,
1950+
datasetPartToReplace.id,
1951+
datasetPartUpdateRequest)
1952+
1953+
val datasetWithReplacedPart =
1954+
datasetApiService.getDataset(organizationSaved.id, workspaceSaved.id, createdDataset.id)
1955+
assertTrue(datasetWithReplacedPart.parts.size == 1)
1956+
assertEquals(replacedDatasetPart, datasetWithReplacedPart.parts[0])
1957+
1958+
assertEquals(datasetPartToReplace.id, replacedDatasetPart.id)
1959+
assertEquals(datasetPartToReplace.name, replacedDatasetPart.name)
1960+
assertEquals(newDatasetSourceName, replacedDatasetPart.sourceName)
1961+
assertEquals(newDatasetPartDescription, replacedDatasetPart.description)
1962+
assertEquals(newDatasetPartTags, replacedDatasetPart.tags)
1963+
assertEquals(newDatasetSourceName, replacedDatasetPart.sourceName)
1964+
assertEquals(DatasetPartTypeEnum.File, replacedDatasetPart.type)
1965+
1966+
val datasetPartFilePath = constructFilePathForDatasetPart(datasetWithReplacedPart, 0)
1967+
assertTrue(s3Template.objectExists(csmPlatformProperties.s3.bucketName, datasetPartFilePath))
1968+
1969+
val unchangedDatasetPartDownloadFile =
1970+
s3Template.download(csmPlatformProperties.s3.bucketName, datasetPartFilePath)
1971+
1972+
val unchangedDatasetPartRetrievedText =
1973+
InputStreamResource(unchangedDatasetPartDownloadFile).inputStream.bufferedReader().use {
1974+
it.readText()
1975+
}
1976+
1977+
assertEquals(customerExpectedText, unchangedDatasetPartRetrievedText)
1978+
}
1979+
18651980
@Test
18661981
fun `test replaceDatasetPart`() {
18671982

‎dataset/src/integrationTest/kotlin/com/cosmotech/dataset/service/DatasetServiceRBACTest.kt‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1198,6 +1198,74 @@ class DatasetServiceRBACTest : CsmTestBase() {
11981198
}
11991199
}
12001200

1201+
@TestFactory
1202+
fun `test RBAC updateDatasetPart`() =
1203+
mapOf(
1204+
ROLE_VIEWER to true,
1205+
ROLE_EDITOR to false,
1206+
ROLE_USER to true,
1207+
ROLE_NONE to true,
1208+
ROLE_ADMIN to false,
1209+
)
1210+
.map { (role, shouldThrow) ->
1211+
DynamicTest.dynamicTest("Test RBAC updateDatasetPart : $role") {
1212+
organization =
1213+
makeOrganizationCreateRequest(
1214+
name = "Organization test",
1215+
userName = CONNECTED_DEFAULT_USER,
1216+
role = ROLE_USER)
1217+
organizationSaved = organizationApiService.createOrganization(organization)
1218+
1219+
solution = makeSolution()
1220+
solutionSaved = solutionApiService.createSolution(organizationSaved.id, solution)
1221+
1222+
workspace =
1223+
makeWorkspaceCreateRequest(userName = CONNECTED_DEFAULT_USER, role = ROLE_USER)
1224+
workspaceSaved = workspaceApiService.createWorkspace(organizationSaved.id, workspace)
1225+
1226+
dataset =
1227+
makeDatasetCreateRequest(
1228+
datasetSecurity =
1229+
DatasetSecurity(
1230+
default = ROLE_NONE,
1231+
accessControlList =
1232+
mutableListOf(
1233+
DatasetAccessControl(CONNECTED_ADMIN_USER, ROLE_ADMIN),
1234+
DatasetAccessControl(
1235+
id = CONNECTED_DEFAULT_USER, role = role))))
1236+
1237+
datasetSaved =
1238+
datasetApiService.createDataset(
1239+
organizationSaved.id, workspaceSaved.id, dataset, mockMultipartFiles)
1240+
1241+
every { getCurrentAccountIdentifier(any()) } returns CONNECTED_DEFAULT_USER
1242+
1243+
if (shouldThrow) {
1244+
val exception =
1245+
assertThrows<CsmAccessForbiddenException> {
1246+
datasetApiService.updateDatasetPart(
1247+
organizationSaved.id,
1248+
workspaceSaved.id,
1249+
datasetSaved.id,
1250+
datasetSaved.parts[0].id,
1251+
makeDatasetPartUpdateRequest())
1252+
}
1253+
assertEquals(
1254+
"RBAC ${datasetSaved.id} - User does not have permission $PERMISSION_WRITE",
1255+
exception.message)
1256+
} else {
1257+
assertDoesNotThrow {
1258+
datasetApiService.updateDatasetPart(
1259+
organizationSaved.id,
1260+
workspaceSaved.id,
1261+
datasetSaved.id,
1262+
datasetSaved.parts[0].id,
1263+
makeDatasetPartUpdateRequest())
1264+
}
1265+
}
1266+
}
1267+
}
1268+
12011269
@Ignore("This method is not ready yet")
12021270
@TestFactory
12031271
fun `test RBAC queryData`() =

‎dataset/src/main/kotlin/com/cosmotech/dataset/service/DatasetServiceImpl.kt‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -571,6 +571,45 @@ class DatasetServiceImpl(
571571
return emptyList()
572572
}
573573

574+
override fun updateDatasetPart(
575+
organizationId: String,
576+
workspaceId: String,
577+
datasetId: String,
578+
datasetPartId: String,
579+
datasetPartUpdateRequest: DatasetPartUpdateRequest
580+
): DatasetPart {
581+
val dataset = getVerifiedDataset(organizationId, workspaceId, datasetId, PERMISSION_WRITE)
582+
val datasetPart =
583+
datasetPartRepository
584+
.findBy(organizationId, workspaceId, datasetId, datasetPartId)
585+
.orElseThrow {
586+
CsmResourceNotFoundException(
587+
"Dataset Part $datasetPartId not found in organization $organizationId, " +
588+
"workspace $workspaceId and dataset $datasetId")
589+
}
590+
val now = Instant.now().toEpochMilli()
591+
val userId = getCurrentAccountIdentifier(csmPlatformProperties)
592+
val editInfo = EditInfo(timestamp = now, userId = userId)
593+
594+
dataset.parts
595+
.find { it.id == datasetPartId }
596+
?.let {
597+
it.sourceName = datasetPartUpdateRequest.sourceName ?: it.sourceName
598+
it.description = datasetPartUpdateRequest.description ?: it.description
599+
it.tags = datasetPartUpdateRequest.tags ?: it.tags
600+
it.updateInfo = editInfo
601+
}
602+
603+
val datasetPartUpdater = datasetPart.copy()
604+
datasetPartUpdater.sourceName = datasetPartUpdateRequest.sourceName ?: datasetPart.sourceName
605+
datasetPartUpdater.description = datasetPartUpdateRequest.description ?: datasetPart.description
606+
datasetPartUpdater.tags = datasetPartUpdateRequest.tags ?: datasetPart.tags
607+
datasetPartUpdater.updateInfo = editInfo
608+
609+
datasetRepository.update(dataset)
610+
return datasetPartRepository.update(datasetPartUpdater)
611+
}
612+
574613
override fun replaceDatasetPart(
575614
organizationId: String,
576615
workspaceId: String,

0 commit comments

Comments
 (0)