Skip to content

Lal/postgres connection prod 14804 #1066

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Lal/postgres connection prod 14804 #1066

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a15c4df
add postgres Jdbc template minimal configuration
lalepee Sep 26, 2025
83e1cce
add jdbc configuration on postgres
lalepee Sep 29, 2025
f61e057
add writer and reader jdbc bean
lalepee Oct 1, 2025
89ac5a1
Refactor InternalResultServices
jreynard-code Oct 2, 2025
5495be3
Remove RDS and Postgres admin role in API configuration
jreynard-code Oct 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 9 additions & 22 deletions api/src/integrationTest/kotlin/com/cosmotech/api/home/ControllerTestBase.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,40 +150,27 @@ abstract class ControllerTestBase : AbstractTestcontainersRedisTestBase() {
}

private fun initPostgresConfiguration(registry: DynamicPropertyRegistry) {
registry.add("csm.platform.internalResultServices.storage.host") { postgres.host }
registry.add("csm.platform.internalResultServices.storage.port") {
postgres.getMappedPort(POSTGRESQL_PORT)
}
registry.add("csm.platform.internalResultServices.storage.admin.username") {
ADMIN_USER_CREDENTIALS
}
registry.add("csm.platform.internalResultServices.storage.admin.password") {
ADMIN_USER_CREDENTIALS
}
registry.add("csm.platform.internalResultServices.storage.writer.username") {
WRITER_USER_CREDENTIALS
}
registry.add("csm.platform.internalResultServices.storage.writer.password") {
WRITER_USER_CREDENTIALS
}
registry.add("csm.platform.internalResultServices.storage.reader.username") {
READER_USER_CREDENTIALS
}
registry.add("csm.platform.internalResultServices.storage.reader.password") {
READER_USER_CREDENTIALS
}
registry.add("csm.platform.databases.data.host") { postgres.host }
registry.add("csm.platform.databases.data.port") { postgres.getMappedPort(POSTGRESQL_PORT) }
registry.add("csm.platform.databases.data.writer.username") { WRITER_USER_CREDENTIALS }
registry.add("csm.platform.databases.data.writer.password") { WRITER_USER_CREDENTIALS }
registry.add("csm.platform.databases.data.reader.username") { READER_USER_CREDENTIALS }
registry.add("csm.platform.databases.data.reader.password") { READER_USER_CREDENTIALS }
}
}

@BeforeAll
fun beforeAll() {
redisStackServer.start()
localStackServer.start()
postgres.start()
}

@AfterAll
fun afterAll() {
postgres.stop()
localStackServer.stop()
redisStackServer.stop()
}

override fun redisServers(): MutableCollection<RedisServer> {
Expand Down
103 changes: 0 additions & 103 deletions api/src/integrationTest/kotlin/com/cosmotech/api/home/run/RunControllerTests.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,6 @@ import com.cosmotech.api.home.run.RunConstants.RequestContent.CONTAINER_NAME
import com.cosmotech.api.home.run.RunConstants.RequestContent.CONTAINER_NODE_LABEL
import com.cosmotech.api.home.run.RunConstants.RequestContent.CONTAINER_RUN_ARGS
import com.cosmotech.api.home.run.RunConstants.RequestContent.CSM_SIMULATION_RUN
import com.cosmotech.api.home.run.RunConstants.RequestContent.CUSTOM_DATA_QUERY
import com.cosmotech.api.home.run.RunConstants.RequestContent.CUSTOM_DATA_TABLE_NAME
import com.cosmotech.api.home.run.RunConstants.RequestContent.DATASET_LIST
import com.cosmotech.api.home.run.RunConstants.RequestContent.DESCRIPTION
import com.cosmotech.api.home.run.RunConstants.RequestContent.HOST_NODE_NAME
Expand Down Expand Up @@ -277,107 +275,6 @@ class RunControllerTests : ControllerTestBase() {
"organizations/{organization_id}/workspaces/{workspace_id}/runners/{runner_id}/runs/{run_id}/DELETE"))
}

@Test
@WithMockOauth2User
fun send_data_run() {

every { eventPublisher.publishEvent(any()) } answers
{
firstArg<UpdateRunnerStatus>().response = "Running"
}

val dataToSend =
"""{
"id": "my_table",
"data": [
{
"additionalProp1": {},
"additionalProp2": "test",
"additionalProp3": 100
},
{
"additionalProp1": {},
"additionalProp2": "test",
"additionalProp4": 1000
}
]
}"""
mvc.perform(
post(
"/organizations/$organizationId/workspaces/$workspaceId/runners/$runnerId}/runs/$runId/data/send")
.contentType(MediaType.APPLICATION_JSON)
.content(dataToSend)
.accept(MediaType.APPLICATION_JSON)
.with(csrf()))
.andExpect(status().is2xxSuccessful)
.andExpect(jsonPath("$.database_name").value(runId))
.andExpect(jsonPath("$.table_name").value(CUSTOM_DATA_TABLE_NAME))
.andExpect(jsonPath("$.data[0].additionalProp1").value(""))
.andExpect(jsonPath("$.data[0].additionalProp2").value("test"))
.andExpect(jsonPath("$.data[0].additionalProp3").value(100))
.andExpect(jsonPath("$.data[1].additionalProp1").value(""))
.andExpect(jsonPath("$.data[1].additionalProp2").value("test"))
.andExpect(jsonPath("$.data[1].additionalProp4").value(1000))
.andDo(MockMvcResultHandlers.print())
.andDo(
document(
"organizations/{organization_id}/workspaces/{workspace_id}/runners/{runner_id}/runs/{run_id}/data/send/POST"))
}

@Test
@WithMockOauth2User
fun query_data_run() {

every { eventPublisher.publishEvent(any()) } answers
{
firstArg<UpdateRunnerStatus>().response = "Running"
}

val dataToSend =
"""{
"id": "my_table",
"data": [
{
"additionalProp1": {},
"additionalProp2": "test",
"additionalProp3": 100
},
{
"additionalProp1": {},
"additionalProp2": "test",
"additionalProp4": 1000
}
]
}"""
mvc.perform(
post(
"/organizations/$organizationId/workspaces/$workspaceId/runners/$runnerId}/runs/$runId/data/send")
.contentType(MediaType.APPLICATION_JSON)
.content(dataToSend)
.accept(MediaType.APPLICATION_JSON)
.with(csrf()))
.andExpect(status().is2xxSuccessful)

mvc.perform(
post(
"/organizations/$organizationId/workspaces/$workspaceId/runners/$runnerId}/runs/$runId/data/query")
.contentType(MediaType.APPLICATION_JSON)
.content(CUSTOM_DATA_QUERY)
.accept(MediaType.APPLICATION_JSON)
.with(csrf()))
.andExpect(status().is2xxSuccessful)
.andExpect(jsonPath("$.result[0].additionalprop1").value(""))
.andExpect(jsonPath("$.result[0].additionalprop2").value("test"))
.andExpect(jsonPath("$.result[0].additionalprop3").value(100))
.andExpect(jsonPath("$.result[1].additionalprop1").value(""))
.andExpect(jsonPath("$.result[1].additionalprop2").value("test"))
.andExpect(jsonPath("$.result[1].additionalprop4").value(1000))
.andDo(MockMvcResultHandlers.print())
.andDo(
document(
"organizations/{organization_id}/workspaces/{workspace_id}/runners/{runner_id}/runs/{run_id}/data/query/POST"))
}

@Test
@WithMockOauth2User
fun get_run_logs() {
Expand Down
36 changes: 20 additions & 16 deletions api/src/integrationTest/resources/application-test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,9 @@ spring:

csm:
platform:
internalResultServices:
enabled: true
eventBus:
enabled: false
containerRegistry:
checkSolutionImage: false
identityProvider:
code: keycloak
authorizationUrl: "http://fake_url:8080/authorize"
tokenUrl: "http://fake_url:8080/token"
containerScopes:
Expand Down Expand Up @@ -102,17 +97,26 @@ csm:
- ReadWriteOnce
requests:
storage: 1Gi
twincache:
host: "localhost"
port: "6379"
username: "default"
# Leave it as blank as there's no auth with test container
password:
tls:
enabled: false
bundle: ""
connector:
default-page-size: 5
databases:
resources:
host: "localhost"
port: "6379"
username: "default"
# Leave it as blank as there's no auth with test container
password:
tls:
enabled: false
bundle: ""
data:
database: "postgres"
host: "localhost"
port: 5432
reader:
password: "password"
username: cosmotech_api_reader
writer:
password: "password"
username: cosmotech_api_writer
rbac:
enabled: true
s3:
Expand Down
1 change: 0 additions & 1 deletion api/src/integrationTest/resources/init-db.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,2 @@
CREATE USER readusertest WITH PASSWORD 'readusertest';
CREATE USER adminusertest WITH SUPERUSER PASSWORD 'adminusertest';
CREATE USER writeusertest WITH PASSWORD 'writeusertest';
63 changes: 20 additions & 43 deletions api/src/main/resources/application.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,14 @@ logging:
OkHttpClient: WARN

spring:
rabbitmq:
host: ${csm.platform.internalResultServices.eventBus.host}
port: ${csm.platform.internalResultServices.eventBus.port}
password: ${csm.platform.internalResultServices.eventBus.listener.password}
username: ${csm.platform.internalResultServices.eventBus.listener.username}
ssl:
enabled: ${csm.platform.internalResultServices.eventBus.tls.enabled}
data:
redis:
host: ${csm.platform.twincache.host}
port: ${csm.platform.twincache.port}
host: ${csm.platform.databases.resources.host}
port: ${csm.platform.databases.resources.port}
ssl:
enabled: ${csm.platform.twincache.tls.enabled}
password: ${csm.platform.twincache.password}
username: ${csm.platform.twincache.username}
enabled: ${csm.platform.databases.resources.tls.enabled}
password: ${csm.platform.databases.resources.password}
username: ${csm.platform.databases.resources.username}
timeout: 60000
client-type: jedis
servlet:
Expand Down Expand Up @@ -86,23 +79,23 @@ csm:
downSamplingDefaultEnabled: false
downSamplingRetentionDays: 400
downSamplingBucketDurationMs: 3600000
twincache:
host: "localhost"
port: "6379"
username: "default_user"
password: "default_password"
tls:
enabled: false
bundle: ""
run:
maxResult: 200
runner:
maxResult: 200
internalResultServices:
enabled: false
storage:
databases:
resources:
host: "localhost"
port: "6379"
username: "default_user"
password: "default_password"
tls:
enabled: false
bundle: ""
run:
defaultPageSize: 200
runner:
defaultPageSize: 200
data:
host: "localhost"
port: 5432
schema: "cosmotech"
reader:
username: "storage_reader_username"
password: "storage_reader_password"
Expand All @@ -112,22 +105,6 @@ csm:
admin:
username: "storage_admin_username"
password: "storage_admin_password"
eventBus:
enabled: true
tls:
enabled: false
bundle: ""
host: "localhost"
port: 5672
default-exchange: "csm-exchange"
default-queue: "csm"
default-routing-key: "csm"
listener:
username: "eventbus_admin_username"
password: "eventbus_admin_password"
sender:
username: "eventbus_sender_username"
password: "eventbus_sender_password"
s3:
endpointUrl: "http://localhost:9000"
bucketName: "cosmotech-api"
Expand Down
Loading