Skip to content

Commit e8b4f9b

Browse files
jreynard-codejreynard-code
committed
Enhance special character sanitization for Redis and update dataset search logic
- Improved `sanitizeForRedis` to handle a broader range of special characters. - Updated `DatasetServiceImpl` to apply sanitized tags in dataset and dataset part searches. - Added integration tests to validate handling of special characters in dataset and dataset part tags.
1 parent b47fde7 commit e8b4f9b

File tree

4 files changed

+302
-12
lines changed

4 files changed

+302
-12
lines changed

‎api/src/integrationTest/kotlin/com/cosmotech/api/home/run/RunControllerTests.kt‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ class RunControllerTests : ControllerTestBase() {
181181
}
182182

183183
mvc.perform(
184-
get("/organizations/$organizationId/workspaces/$workspaceId/runners/$runnerId}/runs")
184+
get("/organizations/$organizationId/workspaces/$workspaceId/runners/$runnerId/runs")
185185
.contentType(MediaType.APPLICATION_JSON)
186186
.accept(MediaType.APPLICATION_JSON)
187187
.with(csrf()))
@@ -219,7 +219,7 @@ class RunControllerTests : ControllerTestBase() {
219219

220220
mvc.perform(
221221
get(
222-
"/organizations/$organizationId/workspaces/$workspaceId/runners/$runnerId}/runs/$runId")
222+
"/organizations/$organizationId/workspaces/$workspaceId/runners/$runnerId/runs/$runId")
223223
.contentType(MediaType.APPLICATION_JSON)
224224
.accept(MediaType.APPLICATION_JSON))
225225
.andExpect(status().is2xxSuccessful)
@@ -264,7 +264,7 @@ class RunControllerTests : ControllerTestBase() {
264264

265265
mvc.perform(
266266
delete(
267-
"/organizations/$organizationId/workspaces/$workspaceId/runners/$runnerId}/runs/$runId")
267+
"/organizations/$organizationId/workspaces/$workspaceId/runners/$runnerId/runs/$runId")
268268
.contentType(MediaType.APPLICATION_JSON)
269269
.accept(MediaType.APPLICATION_JSON)
270270
.with(csrf()))
@@ -294,7 +294,7 @@ class RunControllerTests : ControllerTestBase() {
294294

295295
mvc.perform(
296296
get(
297-
"/organizations/$organizationId/workspaces/$workspaceId/runners/$runnerId}/runs/$runId/logs")
297+
"/organizations/$organizationId/workspaces/$workspaceId/runners/$runnerId/runs/$runId/logs")
298298
.accept(MediaType.TEXT_PLAIN))
299299
.andExpect(status().is2xxSuccessful)
300300
.andExpect(content().string(logs))
@@ -342,7 +342,7 @@ class RunControllerTests : ControllerTestBase() {
342342

343343
mvc.perform(
344344
get(
345-
"/organizations/$organizationId/workspaces/$workspaceId/runners/$runnerId}/runs/$runId/status")
345+
"/organizations/$organizationId/workspaces/$workspaceId/runners/$runnerId/runs/$runId/status")
346346
.contentType(MediaType.APPLICATION_JSON)
347347
.accept(MediaType.APPLICATION_JSON))
348348
.andExpect(status().is2xxSuccessful)

‎common/src/main/kotlin/com/cosmotech/common/utils/StringExtensions.kt‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,13 @@ fun String.sanitizeForKubernetes(maxLength: Int = KUBERNETES_RESOURCE_NAME_MAX_L
2727
.lowercase()
2828
.takeLast(maxLength)
2929

30-
fun String.sanitizeForRedis() =
31-
this.replace("@", "\\\\@").replace(".", "\\\\.").replace("-", "\\\\-")
30+
fun String.sanitizeForRedis(): String {
31+
var sanitizedString = this
32+
",./;'[]-=<>?:{}|_+!@#%^&*()`~"
33+
.forEach { sanitizedString = sanitizedString.replace(it.toString(), "\\\\$it") }
34+
"\"$".forEach { sanitizedString = sanitizedString.replace(it.toString(), "\\\\\\${it}") }
35+
return sanitizedString
36+
}
3237

3338
fun String.sanitizeDatasetPartId(): String = this.replace('-', '_')
3439

‎dataset/src/integrationTest/kotlin/com/cosmotech/dataset/service/DatasetServiceIntegrationTest.kt‎

Lines changed: 280 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1025,6 +1025,104 @@ class DatasetServiceIntegrationTest() : CsmTestBase() {
10251025
assertEquals(0, foundDatasets[0].parts.size)
10261026
}
10271027

1028+
@Test
1029+
fun `test searchDatasets with special char listed`() {
1030+
1031+
val datasetName = "Customer Dataset"
1032+
val datasetDescription = "Dataset for customers"
1033+
val datasetTags = mutableListOf("dataset", "public", "customers", "!")
1034+
val datasetCreateRequest =
1035+
DatasetCreateRequest(
1036+
name = datasetName, description = datasetDescription, tags = datasetTags)
1037+
1038+
datasetApiService.createDataset(
1039+
organizationSaved.id, workspaceSaved.id, datasetCreateRequest, arrayOf())
1040+
1041+
datasetApiService.createDataset(
1042+
organizationSaved.id,
1043+
workspaceSaved.id,
1044+
DatasetCreateRequest(
1045+
name = "Other Dataset",
1046+
description = "Other Dataset ",
1047+
tags = mutableListOf("dataset", "public", "other")),
1048+
arrayOf())
1049+
1050+
val foundDatasets =
1051+
datasetApiService.searchDatasets(
1052+
organizationSaved.id,
1053+
workspaceSaved.id,
1054+
listOf(
1055+
"other",
1056+
",",
1057+
".",
1058+
"/",
1059+
";",
1060+
"'",
1061+
"[",
1062+
"]",
1063+
"-",
1064+
"=",
1065+
"<",
1066+
">",
1067+
"?",
1068+
":",
1069+
"\"",
1070+
"{",
1071+
"}",
1072+
"|",
1073+
"_",
1074+
"+",
1075+
"!",
1076+
"@",
1077+
"#",
1078+
"%",
1079+
"^",
1080+
"&",
1081+
"*",
1082+
"(",
1083+
")",
1084+
"`",
1085+
"~",
1086+
"\$fofo"),
1087+
null,
1088+
null)
1089+
1090+
assertEquals(2, foundDatasets.size)
1091+
}
1092+
1093+
@Test
1094+
fun `test searchDatasets with special char aggregated`() {
1095+
1096+
val datasetName = "Customer Dataset"
1097+
val datasetDescription = "Dataset for customers"
1098+
val datasetTags = mutableListOf("dataset", "public", "customers", "!")
1099+
val datasetCreateRequest =
1100+
DatasetCreateRequest(
1101+
name = datasetName, description = datasetDescription, tags = datasetTags)
1102+
1103+
datasetApiService.createDataset(
1104+
organizationSaved.id, workspaceSaved.id, datasetCreateRequest, arrayOf())
1105+
1106+
datasetApiService.createDataset(
1107+
organizationSaved.id,
1108+
workspaceSaved.id,
1109+
DatasetCreateRequest(
1110+
name = "Other Dataset",
1111+
description = "Other Dataset ",
1112+
tags = mutableListOf("dataset", "public", "other")),
1113+
arrayOf())
1114+
1115+
val foundDatasets =
1116+
datasetApiService.searchDatasets(
1117+
organizationSaved.id,
1118+
workspaceSaved.id,
1119+
listOf("other", ",./;'[]-=<>?:\"{}|_+!@#%^&*()`~$"),
1120+
null,
1121+
null)
1122+
1123+
assertEquals(1, foundDatasets.size)
1124+
}
1125+
10281126
@Test
10291127
fun `test createDatasetPart type FILE`() {
10301128

@@ -1778,6 +1876,188 @@ class DatasetServiceIntegrationTest() : CsmTestBase() {
17781876
assertEquals(DatasetPartTypeEnum.File, foundDatasetParts[0].type)
17791877
}
17801878

1879+
@Test
1880+
fun `test searchDatasetParts with special char listed`() {
1881+
1882+
val datasetCreateRequest = DatasetCreateRequest(name = "Dataset Test")
1883+
1884+
val createDataset =
1885+
datasetApiService.createDataset(
1886+
organizationSaved.id, workspaceSaved.id, datasetCreateRequest, arrayOf())
1887+
1888+
assertTrue(createDataset.parts.isEmpty())
1889+
1890+
val resourceTestFile = resourceLoader.getResource("classpath:/$CUSTOMER_SOURCE_FILE_NAME").file
1891+
1892+
val fileToSend = FileInputStream(resourceTestFile)
1893+
1894+
val mockMultipartFile =
1895+
MockMultipartFile(
1896+
"file",
1897+
CUSTOMER_SOURCE_FILE_NAME,
1898+
MediaType.MULTIPART_FORM_DATA_VALUE,
1899+
IOUtils.toByteArray(fileToSend))
1900+
1901+
val datasetPartName = "Customer list"
1902+
val datasetPartDescription = "List of customers"
1903+
val datasetPartTags = mutableListOf("part", "public", "customers")
1904+
1905+
datasetApiService.createDatasetPart(
1906+
organizationSaved.id,
1907+
workspaceSaved.id,
1908+
createDataset.id,
1909+
mockMultipartFile,
1910+
DatasetPartCreateRequest(
1911+
name = datasetPartName,
1912+
sourceName = CUSTOMER_SOURCE_FILE_NAME,
1913+
description = datasetPartDescription,
1914+
tags = datasetPartTags,
1915+
type = DatasetPartTypeEnum.File))
1916+
1917+
val foundDatasetParts =
1918+
datasetApiService.searchDatasetParts(
1919+
organizationSaved.id,
1920+
workspaceSaved.id,
1921+
createDataset.id,
1922+
listOf(
1923+
"part",
1924+
",",
1925+
".",
1926+
"/",
1927+
";",
1928+
"'",
1929+
"[",
1930+
"]",
1931+
"-",
1932+
"=",
1933+
"<",
1934+
">",
1935+
"?",
1936+
":",
1937+
"\"",
1938+
"{",
1939+
"}",
1940+
"|",
1941+
"_",
1942+
"+",
1943+
"!",
1944+
"@",
1945+
"#",
1946+
"%",
1947+
"^",
1948+
"&",
1949+
"*",
1950+
"(",
1951+
")",
1952+
"`",
1953+
"~",
1954+
"\$fofo"),
1955+
null,
1956+
null)
1957+
1958+
assertEquals(1, foundDatasetParts.size)
1959+
}
1960+
1961+
@Test
1962+
fun `test searchDatasetParts with special char aggregated and no result`() {
1963+
1964+
val datasetCreateRequest = DatasetCreateRequest(name = "Dataset Test")
1965+
1966+
val createDataset =
1967+
datasetApiService.createDataset(
1968+
organizationSaved.id, workspaceSaved.id, datasetCreateRequest, arrayOf())
1969+
1970+
assertTrue(createDataset.parts.isEmpty())
1971+
1972+
val resourceTestFile = resourceLoader.getResource("classpath:/$CUSTOMER_SOURCE_FILE_NAME").file
1973+
1974+
val fileToSend = FileInputStream(resourceTestFile)
1975+
1976+
val mockMultipartFile =
1977+
MockMultipartFile(
1978+
"file",
1979+
CUSTOMER_SOURCE_FILE_NAME,
1980+
MediaType.MULTIPART_FORM_DATA_VALUE,
1981+
IOUtils.toByteArray(fileToSend))
1982+
1983+
val datasetPartName = "Customer list"
1984+
val datasetPartDescription = "List of customers"
1985+
val datasetPartTags = mutableListOf("part", "public", "customers")
1986+
1987+
datasetApiService.createDatasetPart(
1988+
organizationSaved.id,
1989+
workspaceSaved.id,
1990+
createDataset.id,
1991+
mockMultipartFile,
1992+
DatasetPartCreateRequest(
1993+
name = datasetPartName,
1994+
sourceName = CUSTOMER_SOURCE_FILE_NAME,
1995+
description = datasetPartDescription,
1996+
tags = datasetPartTags,
1997+
type = DatasetPartTypeEnum.File))
1998+
1999+
val foundDatasetParts =
2000+
datasetApiService.searchDatasetParts(
2001+
organizationSaved.id,
2002+
workspaceSaved.id,
2003+
createDataset.id,
2004+
listOf("test", ",./;'[]-=<>?:\"{}|_+!@#%^&*()`~$"),
2005+
null,
2006+
null)
2007+
2008+
assertTrue(foundDatasetParts.isEmpty())
2009+
}
2010+
2011+
@Test
2012+
fun `test searchDatasetParts with special char aggregated`() {
2013+
2014+
val datasetCreateRequest = DatasetCreateRequest(name = "Dataset Test")
2015+
2016+
val createDataset =
2017+
datasetApiService.createDataset(
2018+
organizationSaved.id, workspaceSaved.id, datasetCreateRequest, arrayOf())
2019+
2020+
assertTrue(createDataset.parts.isEmpty())
2021+
2022+
val resourceTestFile = resourceLoader.getResource("classpath:/$CUSTOMER_SOURCE_FILE_NAME").file
2023+
2024+
val fileToSend = FileInputStream(resourceTestFile)
2025+
2026+
val mockMultipartFile =
2027+
MockMultipartFile(
2028+
"file",
2029+
CUSTOMER_SOURCE_FILE_NAME,
2030+
MediaType.MULTIPART_FORM_DATA_VALUE,
2031+
IOUtils.toByteArray(fileToSend))
2032+
2033+
val datasetPartName = "Customer list"
2034+
val datasetPartDescription = "List of customers"
2035+
val datasetPartTags = mutableListOf("part", "public", "customers")
2036+
2037+
datasetApiService.createDatasetPart(
2038+
organizationSaved.id,
2039+
workspaceSaved.id,
2040+
createDataset.id,
2041+
mockMultipartFile,
2042+
DatasetPartCreateRequest(
2043+
name = datasetPartName,
2044+
sourceName = CUSTOMER_SOURCE_FILE_NAME,
2045+
description = datasetPartDescription,
2046+
tags = datasetPartTags,
2047+
type = DatasetPartTypeEnum.File))
2048+
2049+
val foundDatasetParts =
2050+
datasetApiService.searchDatasetParts(
2051+
organizationSaved.id,
2052+
workspaceSaved.id,
2053+
createDataset.id,
2054+
listOf("part", ",./;'[]-=<>?:\"{}|_+!@#%^&*()`~$"),
2055+
null,
2056+
null)
2057+
2058+
assertEquals(1, foundDatasetParts.size)
2059+
}
2060+
17812061
@Test
17822062
fun `test getDatasetPart with wrong id`() {
17832063

0 commit comments

Comments
 (0)