Skip to content

Commit bc9d704

Browse files
lalepeelalepee
committed
add test on create and get db type datasetpart
1 parent 7126fe4 commit bc9d704

File tree

7 files changed

+177
-63
lines changed

7 files changed

+177
-63
lines changed

‎api/src/integrationTest/kotlin/com/cosmotech/api/home/ControllerTestBase.kt‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -152,10 +152,10 @@ abstract class ControllerTestBase : AbstractTestcontainersRedisTestBase() {
152152
private fun initPostgresConfiguration(registry: DynamicPropertyRegistry) {
153153
registry.add("csm.platform.databases.data.host") { postgres.host }
154154
registry.add("csm.platform.databases.data.port") { postgres.getMappedPort(POSTGRESQL_PORT) }
155-
registry.add("csm.platform.databases.data.writer.username") { WRITER_USER_CREDENTIALS }
156-
registry.add("csm.platform.databases.data.writer.password") { WRITER_USER_CREDENTIALS }
157-
registry.add("csm.platform.databases.data.reader.username") { READER_USER_CREDENTIALS }
158-
registry.add("csm.platform.databases.data.reader.password") { READER_USER_CREDENTIALS }
155+
// registry.add("csm.platform.databases.data.writer.username") { WRITER_USER_CREDENTIALS }
156+
// registry.add("csm.platform.databases.data.writer.password") { WRITER_USER_CREDENTIALS }
157+
// registry.add("csm.platform.databases.data.reader.username") { READER_USER_CREDENTIALS }
158+
// registry.add("csm.platform.databases.data.reader.password") { READER_USER_CREDENTIALS }
159159
}
160160
}
161161

‎common/src/main/kotlin/com/cosmotech/common/tests/CsmTestBase.kt‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@ import org.testcontainers.utility.MountableFile
2020
open class CsmTestBase : AbstractTestcontainersRedisTestBase() {
2121

2222
companion object {
23-
private const val READER_USER_CREDENTIALS = "readusertest"
24-
private const val WRITER_USER_CREDENTIALS = "writeusertest"
2523
private const val DEFAULT_REDIS_PORT = 6379
2624
private const val LOCALSTACK_FULL_IMAGE_NAME = "localstack/localstack:3.5.0"
2725

@@ -72,10 +70,6 @@ open class CsmTestBase : AbstractTestcontainersRedisTestBase() {
7270
private fun initPostgresConfiguration(registry: DynamicPropertyRegistry) {
7371
registry.add("csm.platform.databases.data.host") { postgres.host }
7472
registry.add("csm.platform.databases.data.port") { postgres.getMappedPort(POSTGRESQL_PORT) }
75-
registry.add("csm.platform.databases.data.writer.username") { WRITER_USER_CREDENTIALS }
76-
registry.add("csm.platform.databases.data.writer.password") { WRITER_USER_CREDENTIALS }
77-
registry.add("csm.platform.databases.data.reader.username") { READER_USER_CREDENTIALS }
78-
registry.add("csm.platform.databases.data.reader.password") { READER_USER_CREDENTIALS }
7973
}
8074
}
8175

‎dataset/src/integrationTest/kotlin/com/cosmotech/dataset/service/DatasetServiceIntegrationTest.kt‎

Lines changed: 130 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
package com.cosmotech.dataset.service
44

55
import com.cosmotech.common.config.CsmPlatformProperties
6+
import com.cosmotech.common.config.existTable
67
import com.cosmotech.common.exceptions.CsmAccessForbiddenException
78
import com.cosmotech.common.exceptions.CsmResourceNotFoundException
89
import com.cosmotech.common.rbac.ROLE_ADMIN
@@ -1022,7 +1023,7 @@ class DatasetServiceIntegrationTest() : CsmTestBase() {
10221023
}
10231024

10241025
@Test
1025-
fun `test createDatasetPart`() {
1026+
fun `test createDatasetPart type FILE`() {
10261027

10271028
val datasetCreateRequest = DatasetCreateRequest(name = "Dataset Test")
10281029

@@ -1085,6 +1086,74 @@ class DatasetServiceIntegrationTest() : CsmTestBase() {
10851086
assertEquals(expectedText, retrievedText)
10861087
}
10871088

1089+
@Test
1090+
fun `test createDatasetPart type DB`() {
1091+
1092+
val datasetCreateRequest = DatasetCreateRequest(name = "Dataset Test")
1093+
1094+
val createDataset =
1095+
datasetApiService.createDataset(
1096+
organizationSaved.id, workspaceSaved.id, datasetCreateRequest, arrayOf())
1097+
1098+
assertTrue(createDataset.parts.isEmpty())
1099+
1100+
val resourceTestFile = resourceLoader.getResource("classpath:/$CUSTOMER_SOURCE_FILE_NAME").file
1101+
1102+
val fileToSend = FileInputStream(resourceTestFile)
1103+
1104+
val mockMultipartFile =
1105+
MockMultipartFile(
1106+
"file",
1107+
CUSTOMER_SOURCE_FILE_NAME,
1108+
MediaType.MULTIPART_FORM_DATA_VALUE,
1109+
IOUtils.toByteArray(fileToSend))
1110+
1111+
val datasetPartName = "Customer list"
1112+
val datasetPartDescription = "List of customers"
1113+
val datasetPartTags = mutableListOf("part", "public", "customers")
1114+
1115+
val createDatasetPart =
1116+
datasetApiService.createDatasetPart(
1117+
organizationSaved.id,
1118+
workspaceSaved.id,
1119+
createDataset.id,
1120+
mockMultipartFile,
1121+
DatasetPartCreateRequest(
1122+
name = datasetPartName,
1123+
sourceName = CUSTOMER_SOURCE_FILE_NAME,
1124+
description = datasetPartDescription,
1125+
tags = datasetPartTags,
1126+
type = DatasetPartTypeEnum.Relational))
1127+
1128+
assertNotNull(createDatasetPart)
1129+
assertEquals(datasetPartName, createDatasetPart.name)
1130+
assertEquals(datasetPartDescription, createDatasetPart.description)
1131+
assertEquals(datasetPartTags, createDatasetPart.tags)
1132+
assertEquals(CUSTOMER_SOURCE_FILE_NAME, createDatasetPart.sourceName)
1133+
1134+
val retrievedDataset =
1135+
datasetApiService.getDataset(organizationSaved.id, workspaceSaved.id, createDataset.id)
1136+
1137+
assertTrue(retrievedDataset.parts.isNotEmpty())
1138+
assertTrue(retrievedDataset.parts.size == 1)
1139+
assertEquals(createDatasetPart, retrievedDataset.parts[0])
1140+
1141+
val fileKeyPath = constructFilePathForDatasetPart(retrievedDataset, 0)
1142+
1143+
writerJdbcTemplate.queryForList("select * from pg_tables where schemaname='inputs'").forEach {
1144+
logger.error("Size : ${it.size}")
1145+
it.forEach { (key, value) -> logger.error("$key : $value") }
1146+
}
1147+
assertTrue(writerJdbcTemplate.existTable(createDatasetPart.id.replace('-', '_')))
1148+
1149+
val datasetPartFile =
1150+
datasetApiService.downloadDatasetPart(
1151+
organizationSaved.id, workspaceSaved.id, createDataset.id, createDatasetPart.id)
1152+
val expectedText = FileInputStream(resourceTestFile).bufferedReader().use { it.readText() }
1153+
val retrievedText = datasetPartFile.inputStream.bufferedReader().use { it.readText() }
1154+
assertEquals(expectedText, retrievedText)
1155+
}
1156+
10881157
@Test
10891158
fun `test createDatasetPart with unallowed mimetype`() {
10901159

@@ -1233,8 +1302,65 @@ class DatasetServiceIntegrationTest() : CsmTestBase() {
12331302
exception.message)
12341303
}
12351304

1305+
@Test
1306+
fun `test deleteDatasetPart FILE`() {
1307+
1308+
val datasetCreateRequest = DatasetCreateRequest(name = "Dataset Test")
1309+
1310+
val createDataset =
1311+
datasetApiService.createDataset(
1312+
organizationSaved.id, workspaceSaved.id, datasetCreateRequest, arrayOf())
1313+
1314+
assertTrue(createDataset.parts.isEmpty())
1315+
1316+
val resourceTestFile = resourceLoader.getResource("classpath:/$CUSTOMER_SOURCE_FILE_NAME").file
1317+
1318+
val fileToSend = FileInputStream(resourceTestFile)
1319+
1320+
val mockMultipartFile =
1321+
MockMultipartFile(
1322+
"file",
1323+
CUSTOMER_SOURCE_FILE_NAME,
1324+
MediaType.MULTIPART_FORM_DATA_VALUE,
1325+
IOUtils.toByteArray(fileToSend))
1326+
1327+
val datasetPartName = "Customer list"
1328+
val datasetPartDescription = "List of customers"
1329+
val datasetPartTags = mutableListOf("part", "public", "customers")
1330+
1331+
val createDatasetPart =
1332+
datasetApiService.createDatasetPart(
1333+
organizationSaved.id,
1334+
workspaceSaved.id,
1335+
createDataset.id,
1336+
mockMultipartFile,
1337+
DatasetPartCreateRequest(
1338+
name = datasetPartName,
1339+
sourceName = CUSTOMER_SOURCE_FILE_NAME,
1340+
description = datasetPartDescription,
1341+
tags = datasetPartTags,
1342+
type = DatasetPartTypeEnum.File))
1343+
1344+
assertNotNull(createDatasetPart)
1345+
datasetApiService.deleteDatasetPart(
1346+
organizationSaved.id, workspaceSaved.id, createDataset.id, createDatasetPart.id)
1347+
1348+
val retrievedDataset =
1349+
datasetApiService.getDataset(organizationSaved.id, workspaceSaved.id, createDataset.id)
1350+
1351+
assertTrue(retrievedDataset.parts.isEmpty())
1352+
1353+
val deletedFileKeyPath =
1354+
retrievedDataset.organizationId +
1355+
"/${retrievedDataset.workspaceId}" +
1356+
"/${retrievedDataset.id}" +
1357+
"/${createDatasetPart.id}" +
1358+
"/${createDatasetPart.sourceName}"
1359+
1360+
assertFalse(s3Template.objectExists(csmPlatformProperties.s3.bucketName, deletedFileKeyPath))
1361+
}
12361362
@Test
1237-
fun `test deleteDatasetPart`() {
1363+
fun `test deleteDatasetPart DB`() {
12381364

12391365
val datasetCreateRequest = DatasetCreateRequest(name = "Dataset Test")
12401366

@@ -1270,7 +1396,7 @@ class DatasetServiceIntegrationTest() : CsmTestBase() {
12701396
sourceName = CUSTOMER_SOURCE_FILE_NAME,
12711397
description = datasetPartDescription,
12721398
tags = datasetPartTags,
1273-
type = DatasetPartTypeEnum.File))
1399+
type = DatasetPartTypeEnum.Relational))
12741400

12751401
assertNotNull(createDatasetPart)
12761402
datasetApiService.deleteDatasetPart(
@@ -1281,14 +1407,7 @@ class DatasetServiceIntegrationTest() : CsmTestBase() {
12811407

12821408
assertTrue(retrievedDataset.parts.isEmpty())
12831409

1284-
val deletedFileKeyPath =
1285-
retrievedDataset.organizationId +
1286-
"/${retrievedDataset.workspaceId}" +
1287-
"/${retrievedDataset.id}" +
1288-
"/${createDatasetPart.id}" +
1289-
"/${createDatasetPart.sourceName}"
1290-
1291-
assertFalse(s3Template.objectExists(csmPlatformProperties.s3.bucketName, deletedFileKeyPath))
1410+
assertFalse(writerJdbcTemplate.existTable(createDatasetPart.id.replace('-', '_')))
12921411
}
12931412

12941413
@Test

‎dataset/src/integrationTest/resources/application-dataset-test.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,10 +110,10 @@ csm:
110110
host: "localhost"
111111
port: 5432
112112
reader:
113-
password: "password"
113+
password: "cosmotech_api_reader_pass"
114114
username: cosmotech_api_reader
115115
writer:
116-
password: "password"
116+
password: "cosmotech_api_writer_pass"
117117
username: cosmotech_api_writer
118118
database: "test"
119119
rbac:

‎dataset/src/integrationTest/resources/customers.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,4 @@ CUST006,FastFix Auto,Isabelle Dubois,Service Manager,42 Garage Loop,Montréal,QC
88
CUST007,Garden Glow Ltd,Ellie Tan,Product Specialist,180 Daisy St,Burnside,NSW,2135,Australia,02-8000-1122
99
CUST008,Evergreen Crafts,Jasper Lee,Owner,96 Willow Way,Lakeside,BC,V2V 4W1,Canada,604-800-9988
1010
CUST009,Peak Fitness Inc,Monica Anders,Marketing Lead,311 Summit Ave,Cascade,WA,98101,USA,206-777-5544
11-
CUST010,Silverline Media,Nikhil Ajay,Account Executive,845 Market Plaza,Redhill,ENG,RH1 6JT,UK,020-7123-4098
11+
CUST010,Silverline Media,Nikhil Ajay,Account Executive,845 Market Plaza,Redhill,ENG,RH1 6JT,UK,020-7123-4098

‎dataset/src/integrationTest/resources/init-db.sql‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
CREATE USER readusertest WITH PASSWORD 'readusertest';
2-
CREATE USER writeusertest WITH PASSWORD 'writeusertest';
3-
CREATE USER adminusertest WITH PASSWORD 'adminusertest';
1+
CREATE ROLE cosmotech_api_reader WITH LOGIN PASSWORD 'cosmotech_api_reader_pass';
2+
CREATE ROLE cosmotech_api_writer WITH LOGIN PASSWORD 'cosmotech_api_writer_pass';
3+
CREATE ROLE cosmotech_api_admin WITH LOGIN PASSWORD 'cosmotech_api_admin_pass';
44

5-
CREATE SCHEMA inputs AUTHORIZATION writeusertest;
6-
CREATE SCHEMA outputs AUTHORIZATION writeusertest;
7-
GRANT USAGE ON SCHEMA inputs TO readusertest;
8-
GRANT USAGE ON SCHEMA outputs TO readusertest;
5+
CREATE SCHEMA inputs AUTHORIZATION cosmotech_api_writer;
6+
CREATE SCHEMA outputs AUTHORIZATION cosmotech_api_writer;
7+
GRANT USAGE ON SCHEMA inputs TO cosmotech_api_reader;
8+
GRANT USAGE ON SCHEMA outputs TO cosmotech_api_reader;

‎dataset/src/main/kotlin/com/cosmotech/dataset/part/services/RelationalDatasetPartManagementService.kt‎

Lines changed: 33 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@
33
package com.cosmotech.dataset.part.services
44

55
import com.cosmotech.common.config.CsmPlatformProperties
6+
import com.cosmotech.common.config.DATASET_INPUTS_SCHEMA
67
import com.cosmotech.common.config.existTable
78
import com.cosmotech.dataset.domain.DatasetPart
9+
import java.io.BufferedReader
810
import java.io.ByteArrayOutputStream
911
import java.io.InputStream
1012
import java.sql.SQLException
@@ -17,7 +19,6 @@ import org.springframework.core.io.Resource
1719
import org.springframework.jdbc.core.JdbcTemplate
1820
import org.springframework.stereotype.Service
1921
import org.springframework.web.multipart.MultipartFile
20-
import java.io.BufferedReader
2122

2223
/**
2324
* Service implementation for managing dataset parts in a relational database.
@@ -59,18 +60,15 @@ class RelationalDatasetPartManagementService(
5960
writerJdbcTemplate.dataSource!!.connection.use { connection ->
6061
try {
6162
connection.autoCommit = false
62-
val tableName = "inputs.${datasetPart.id.replace('-', '_')}"
63+
val tableName = "${DATASET_INPUTS_SCHEMA}.${sanitizePartId(datasetPart.id)}"
6364
if (overwrite) {
64-
val prepareStatement = connection.prepareStatement("DROP TABLE IF EXISTS ?")
65-
prepareStatement.setString(1, tableName)
66-
prepareStatement.execute()
65+
val prepareStatement = connection.prepareStatement("DROP TABLE IF EXISTS $tableName")
66+
prepareStatement.execute()
6767
}
6868

6969
if (!tableExists) {
70-
val prepareStatement = connection.prepareStatement("CREATE TABLE IF NOT EXISTS ? ?")
71-
prepareStatement.setString(1, tableName)
72-
prepareStatement.setString(2, values)
73-
prepareStatement.execute()
70+
val prepareStatement = connection.prepareStatement("CREATE TABLE IF NOT EXISTS $tableName ${constructValues(headers)};")
71+
prepareStatement.execute()
7472
}
7573
val insertedRows =
7674
CopyManager(connection as BaseConnection)
@@ -87,43 +85,42 @@ class RelationalDatasetPartManagementService(
8785
}
8886
}
8987

90-
private fun validateHeaders(reader: BufferedReader): List<String>{
91-
val headers = reader
92-
.readLine()
93-
.split(",", "\n")
94-
95-
require(headers.isNotEmpty())
96-
{ "No headers found in dataset part file" }
97-
require(headers.all { it.isNotBlank() })
98-
{ "Empty headers found in dataset part file" }
99-
require(headers.distinct().size == headers.size)
100-
{ "Duplicate headers found in dataset part file" }
101-
require(headers.all { Regex.fromLiteral("[a-zA-Z0-9_]+").matches(it)})
102-
{"Invalid header name found in dataset part file: header name must match [a-zA-Z0-9_]+"}
103-
104-
return headers
88+
private fun validateHeaders(reader: BufferedReader): List<String> {
89+
val headers = reader.readLine().split(",", "\n")
90+
91+
require(headers.isNotEmpty()) { "No headers found in dataset part file" }
92+
require(headers.all { it.isNotBlank() }) { "Empty headers found in dataset part file" }
93+
require(headers.distinct().size == headers.size) {
94+
"Duplicate headers found in dataset part file"
95+
}
96+
require(headers.all { Regex("[a-zA-Z0-9_]+").matches(it) }) {
97+
"Invalid header name found in dataset part file: header name must match [a-zA-Z0-9_]+ (found: ${headers})"
10598
}
10699

107-
private fun constructValues(headers: List<String>): String = headers
108-
.joinToString(
109-
separator = " TEXT, ", prefix = "(", postfix = " TEXT)", transform = String::trim
110-
)
100+
return headers
101+
}
111102

112-
override fun getData(datasetPart: DatasetPart): Resource {
103+
private fun constructValues(headers: List<String>): String =
104+
headers.joinToString(
105+
separator = "\" TEXT, \"", prefix = "(\"", postfix = "\" TEXT)", transform = String::trim)
106+
107+
override fun getData(datasetPart: DatasetPart): Resource {
108+
val tableName = "${DATASET_INPUTS_SCHEMA}.${sanitizePartId(datasetPart.id)}"
113109
val out = ByteArrayOutputStream()
114-
readerJdbcTemplate.dataSource!!.connection.use { connection ->
110+
writerJdbcTemplate.dataSource!!.connection.use { connection ->
115111
CopyManager(connection as BaseConnection)
116-
.copyOut("COPY ${datasetPart.id} TO STDOUT WITH FORMAT CSV, HEADER TRUE", out)
112+
.copyOut("COPY $tableName TO STDOUT WITH CSV HEADER", out)
117113
}
118114
return ByteArrayResource(out.toByteArray())
119115
}
120116

121117
override fun delete(datasetPart: DatasetPart) {
118+
val tableName = "${DATASET_INPUTS_SCHEMA}.${sanitizePartId(datasetPart.id)}"
122119
writerJdbcTemplate.dataSource!!.connection.use { connection ->
123120
try {
124121
connection.autoCommit = false
125122
connection.createStatement().use { statement ->
126-
statement.execute("DROP TABLE ${datasetPart.id};")
123+
statement.execute("DROP TABLE $tableName;")
127124
}
128125
connection.commit()
129126
} catch (ex: SQLException) {
@@ -133,4 +130,8 @@ class RelationalDatasetPartManagementService(
133130
}
134131
}
135132
}
133+
134+
private fun sanitizePartId(name: String): String {
135+
return name.replace('-', '_')
136+
}
136137
}

0 commit comments

Comments
 (0)