Skip to content

Commit e606527

Browse files
jreynard-codejreynard-code
committed
Add integration tests on dataset part DB
1 parent ef707c5 commit e606527

File tree

10 files changed

+10407
-139
lines changed

10 files changed

+10407
-139
lines changed

‎api/src/integrationTest/kotlin/com/cosmotech/api/home/ControllerTestBase.kt‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -99,9 +99,6 @@ abstract class ControllerTestBase : AbstractTestcontainersRedisTestBase() {
9999
}
100100

101101
companion object {
102-
private const val ADMIN_USER_CREDENTIALS = "adminusertest"
103-
private const val READER_USER_CREDENTIALS = "readusertest"
104-
private const val WRITER_USER_CREDENTIALS = "writeusertest"
105102
private const val DEFAULT_REDIS_PORT = 6379
106103
private const val LOCALSTACK_FULL_IMAGE_NAME = "localstack/localstack:3.5.0"
107104

@@ -152,10 +149,6 @@ abstract class ControllerTestBase : AbstractTestcontainersRedisTestBase() {
152149
private fun initPostgresConfiguration(registry: DynamicPropertyRegistry) {
153150
registry.add("csm.platform.databases.data.host") { postgres.host }
154151
registry.add("csm.platform.databases.data.port") { postgres.getMappedPort(POSTGRESQL_PORT) }
155-
// registry.add("csm.platform.databases.data.writer.username") { WRITER_USER_CREDENTIALS }
156-
// registry.add("csm.platform.databases.data.writer.password") { WRITER_USER_CREDENTIALS }
157-
// registry.add("csm.platform.databases.data.reader.username") { READER_USER_CREDENTIALS }
158-
// registry.add("csm.platform.databases.data.reader.password") { READER_USER_CREDENTIALS }
159152
}
160153
}
161154

‎dataset/src/integrationTest/kotlin/com/cosmotech/dataset/service/DatasetServiceIntegrationTest.kt‎

Lines changed: 366 additions & 118 deletions
Large diffs are not rendered by default.

‎dataset/src/integrationTest/resources/customers10000.csv‎

Lines changed: 10001 additions & 0 deletions
Large diffs are not rendered by default.

‎dataset/src/integrationTest/resources/customerswithdoublequotes.csv‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
"customerID","companyName","contactName","contactTitle","address","city","region","postalCode","country","phone"
2+
"CUST001","Blueberry Tech","Alex Greenwood","CTO","1412 Pine St","Oakridge","CA","93561","USA","555-123-4560"
3+
"CUST002","Riverbend Logistics","Taylor Morris","Operations Lead","89 North Ave","Greendale","NY","11105","USA","555-987-6521"
4+
"CUST003","Sunrise Foods","Jordan Liu","Procurement Manager","220 Cherry Road","Springdale","TX","75010","USA","555-210-7895"
5+
"CUST004","Skyline Textiles","Maria Petrova","Sales Director","501 Textile Lane","Newpark","FL","33012","USA","555-313-9240"
6+
"CUST005","Crimson Solutions","Samuel Evans","CEO","77 Baker Blvd","Woodport","GA","30310","USA","555-825-2555"
7+
"CUST006","FastFix Auto","Isabelle Dubois","Service Manager","42 Garage Loop","Montréal","QC","H3Z 2Y7","Canada","514-601-3300"
8+
"CUST007","Garden Glow Ltd","Ellie Tan","Product Specialist","180 Daisy St","Burnside","NSW","2135","Australia","02-8000-1122"
9+
"CUST008","Evergreen Crafts","Jasper Lee","Owner","96 Willow Way","Lakeside","BC","V2V 4W1","Canada","604-800-9988"
10+
"CUST009","Peak Fitness Inc","Monica Anders","Marketing Lead","311 Summit Ave","Cascade","WA","98101","USA","206-777-5544"
11+
"CUST010","Silverline Media","Nikhil Ajay","Account Executive","845 Market Plaza","Redhill","ENG","RH1 6JT","UK","020-7123-4098"

‎dataset/src/integrationTest/resources/customerswithquotes.csv‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
'customerID','companyName','contactName','contactTitle','address','city','region','postalCode','country','phone'
2+
'CUST001','Blueberry Tech','Alex Greenwood','CTO','1412 Pine St','Oakridge','CA','93561','USA','555-123-4560'
3+
'CUST002','Riverbend Logistics','Taylor Morris','Operations Lead','89 North Ave','Greendale','NY','11105','USA','555-987-6521'
4+
'CUST003','Sunrise Foods','Jordan Liu','Procurement Manager','220 Cherry Road','Springdale','TX','75010','USA','555-210-7895'
5+
'CUST004','Skyline Textiles','Maria Petrova','Sales Director','501 Textile Lane','Newpark','FL','33012','USA','555-313-9240'
6+
'CUST005','Crimson Solutions','Samuel Evans','CEO','77 Baker Blvd','Woodport','GA','30310','USA','555-825-2555'
7+
'CUST006','FastFix Auto','Isabelle Dubois','Service Manager','42 Garage Loop','Montréal','QC','H3Z 2Y7','Canada','514-601-3300'
8+
'CUST007','Garden Glow Ltd','Ellie Tan','Product Specialist','180 Daisy St','Burnside','NSW','2135','Australia','02-8000-1122'
9+
'CUST008','Evergreen Crafts','Jasper Lee','Owner','96 Willow Way','Lakeside','BC','V2V 4W1','Canada','604-800-9988'
10+
'CUST009','Peak Fitness Inc','Monica Anders','Marketing Lead','311 Summit Ave','Cascade','WA','98101','USA','206-777-5544'
11+
'CUST010','Silverline Media','Nikhil Ajay','Account Executive','845 Market Plaza','Redhill','ENG','RH1 6JT','UK','020-7123-4098'

‎dataset/src/integrationTest/resources/emptyfile.csv‎

Whitespace-only changes.

‎dataset/src/integrationTest/resources/filewithsameheaders.csv‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
heaDer,Header, header,header ,HEADER, HEADER

‎dataset/src/integrationTest/resources/filewithunallowedcharinheaders.csv‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"%1325 \ sr","-#()char'\`\""

‎dataset/src/main/kotlin/com/cosmotech/dataset/part/services/RelationalDatasetPartManagementService.kt‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import java.io.ByteArrayOutputStream
1111
import java.io.InputStream
1212
import java.sql.SQLException
1313
import kotlin.use
14+
import org.apache.commons.lang3.StringUtils
1415
import org.postgresql.copy.CopyManager
1516
import org.postgresql.core.BaseConnection
1617
import org.slf4j.LoggerFactory
@@ -26,7 +27,7 @@ import org.springframework.web.multipart.MultipartFile
2627
* This service provides methods to manage parts of a dataset stored in a relational database,
2728
* including saving and deleting dataset part entities.
2829
*/
29-
@Service("Relational")
30+
@Service("DB")
3031
class RelationalDatasetPartManagementService(
3132
val writerJdbcTemplate: JdbcTemplate,
3233
val readerJdbcTemplate: JdbcTemplate,
@@ -59,7 +60,7 @@ class RelationalDatasetPartManagementService(
5960
writerJdbcTemplate.dataSource!!.connection.use { connection ->
6061
try {
6162
connection.autoCommit = false
62-
val tableName = "${DATASET_INPUTS_SCHEMA}.${sanitizePartId(datasetPart.id)}"
63+
val tableName = "${DATASET_INPUTS_SCHEMA}.${datasetPart.id.sanitizeDatasetPartId()}"
6364
if (overwrite) {
6465
val prepareStatement = connection.prepareStatement("DROP TABLE IF EXISTS $tableName")
6566
prepareStatement.execute()
@@ -87,26 +88,29 @@ class RelationalDatasetPartManagementService(
8788
}
8889

8990
private fun validateHeaders(reader: BufferedReader): List<String> {
90-
val headers = reader.readLine().split(",", "\n")
91+
val headers = reader.readLine()?.split(",", "\n") ?: emptyList()
9192

9293
require(headers.isNotEmpty()) { "No headers found in dataset part file" }
9394
require(headers.all { it.isNotBlank() }) { "Empty headers found in dataset part file" }
94-
require(headers.distinct().size == headers.size) {
95+
require(headers.map { it.lowercase().trim() }.distinct().size == headers.size) {
9596
"Duplicate headers found in dataset part file"
9697
}
97-
require(headers.all { Regex("[a-zA-Z0-9_]+").matches(it) }) {
98-
"Invalid header name found in dataset part file: header name must match [a-zA-Z0-9_]+ (found: ${headers})"
98+
require(headers.all { Regex("[a-zA-Z0-9_\"\' ]+").matches(it) }) {
99+
"Invalid header name found in dataset part file: header name must match [a-zA-Z0-9_\"\' ]+ (found: ${headers})"
99100
}
100101

101102
return headers
102103
}
103104

104105
private fun constructSQLColumnsValues(headers: List<String>): String =
105-
headers.joinToString(
106-
separator = "\" TEXT, \"", prefix = "(\"", postfix = "\" TEXT)", transform = String::trim)
106+
headers
107+
.map { it.trim() }
108+
.joinToString(separator = " TEXT, ", prefix = "(", postfix = " TEXT)") {
109+
StringUtils.wrapIfMissing(it, "\"")
110+
}
107111

108112
override fun getData(datasetPart: DatasetPart): Resource {
109-
val tableName = "${DATASET_INPUTS_SCHEMA}.${sanitizePartId(datasetPart.id)}"
113+
val tableName = "${DATASET_INPUTS_SCHEMA}.${datasetPart.id.sanitizeDatasetPartId()}"
110114
val out = ByteArrayOutputStream()
111115
writerJdbcTemplate.dataSource!!.connection.use { connection ->
112116
CopyManager(connection as BaseConnection)
@@ -116,7 +120,7 @@ class RelationalDatasetPartManagementService(
116120
}
117121

118122
override fun delete(datasetPart: DatasetPart) {
119-
val tableName = "${DATASET_INPUTS_SCHEMA}.${sanitizePartId(datasetPart.id)}"
123+
val tableName = "${DATASET_INPUTS_SCHEMA}.${datasetPart.id.sanitizeDatasetPartId()}"
120124
writerJdbcTemplate.dataSource!!.connection.use { connection ->
121125
try {
122126
connection.autoCommit = false
@@ -132,7 +136,5 @@ class RelationalDatasetPartManagementService(
132136
}
133137
}
134138

135-
private fun sanitizePartId(name: String): String {
136-
return name.replace('-', '_')
137-
}
139+
fun String.sanitizeDatasetPartId(): String = this.replace('-', '_')
138140
}

‎dataset/src/main/openapi/dataset.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -922,7 +922,7 @@ components:
922922
description: The dataset part Type
923923
enum:
924924
- File
925-
- Relational
925+
- DB
926926
default: File
927927

928928
DatasetPartFiles:

0 commit comments

Comments
 (0)