From a15c4dff97acdb4348f0a317e81e99d4bf60bf38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Laurent=20Al=C3=A9p=C3=A9e?= Date: Fri, 26 Sep 2025 17:20:12 +0200 Subject: [PATCH 1/5] add postgres Jdbc template minimal configuration --- .../common/postgres/PostgresConfiguration.kt | 62 +++++++++++++++++++ config/application-dev.sample.yml | 7 ++- 2 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 common/src/main/kotlin/com/cosmotech/common/postgres/PostgresConfiguration.kt diff --git a/common/src/main/kotlin/com/cosmotech/common/postgres/PostgresConfiguration.kt b/common/src/main/kotlin/com/cosmotech/common/postgres/PostgresConfiguration.kt new file mode 100644 index 000000000..021bbd144 --- /dev/null +++ b/common/src/main/kotlin/com/cosmotech/common/postgres/PostgresConfiguration.kt @@ -0,0 +1,62 @@ +package com.cosmotech.common.postgres + +import org.springframework.beans.factory.annotation.Qualifier +import org.springframework.beans.factory.annotation.Value +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.jdbc.core.JdbcTemplate +import org.springframework.jdbc.datasource.DriverManagerDataSource +import javax.sql.DataSource + +@Configuration +class PostgresConfiguration { + @Value("\${csm.platform.internalResultServices.storage.admin.username}") + private lateinit var adminStorageUsername: String + @Value("\${csm.platform.internalResultServices.storage.admin.password}") + private lateinit var adminStoragePassword: String + @Value("\${csm.platform.internalResultServices.storage.host}") private lateinit var host: String + @Value("\${csm.platform.internalResultServices.storage.port}") private lateinit var port: String + @Value("\${csm.platform.internalResultServices.storage.db.name}") private lateinit var dbName: String + @Value("\${csm.platform.internalResultServices.storage.db.schema}") private lateinit var schema: String + + private val jdbcdriverClass = "org.postgresql.Driver" + + @Bean + fun adminDatasource(): DriverManagerDataSource { + val dataSource = + DriverManagerDataSource( + "jdbc:postgresql://$host:$port/$dbName", adminStorageUsername, adminStoragePassword) + dataSource.setDriverClassName(jdbcdriverClass) + return dataSource + } + + @Bean + fun adminJdbcTemplate( + @Qualifier("adminDatasource") dataSource: DataSource + ): JdbcTemplate { + return JdbcTemplate(dataSource) + } +} + +fun JdbcTemplate.existDB(name: String): Boolean { + return this.queryForList("SELECT * FROM pg_catalog.pg_database WHERE datname='$name'").size == 1 +} + +fun JdbcTemplate.existTable(name: String): Boolean { + return this.queryForList( + "SELECT * FROM information_schema.tables " + "WHERE table_name ilike '${name}'") + .size >= 1 +} + +fun String.toDataTableName(isProbeData: Boolean): String = + (if (isProbeData) "P_$this" else "CD_$this").lowercase() + +fun JdbcTemplate.createDB(name: String, comment: String? = null): String { + this.execute("CREATE DATABASE \"$name\"") + if (comment != null) this.execute("COMMENT ON DATABASE \"$name\" IS '$comment'") + return name +} + +fun JdbcTemplate.dropDB(name: String) { + if (this.existDB(name)) this.execute("DROP DATABASE \"$name\"") +} diff --git a/config/application-dev.sample.yml b/config/application-dev.sample.yml index 4c29e8de1..2b0e249ac 100644 --- a/config/application-dev.sample.yml +++ b/config/application-dev.sample.yml @@ -106,16 +106,19 @@ csm: tls: enabled: false storage: + host: "[fill-this-value]" # postgresql-NAMESPACE.NAMESPACE.svc.cluster.local + port: 5432 admin: password: "[fill-this-value]" username: "[fill-this-value]" - host: "[fill-this-value]" # postgresql-NAMESPACE.NAMESPACE.svc.cluster.local - port: 5432 reader: password: "[fill-this-value]" username: cosmotech_api_reader writer: password: "[fill-this-value]" username: cosmotech_api_writer + db: + name: cosmotech + schema: datasets From 83e1cce00e8e55039f88f1b3e8ff2a3ce1189fbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Laurent=20Al=C3=A9p=C3=A9e?= Date: Mon, 29 Sep 2025 17:46:30 +0200 Subject: [PATCH 2/5] add jdbc configuration on postgres --- .../common/postgres/PostgresConfiguration.kt | 72 ++++++------------- config/application-dev.sample.yml | 2 +- 2 files changed, 22 insertions(+), 52 deletions(-) diff --git a/common/src/main/kotlin/com/cosmotech/common/postgres/PostgresConfiguration.kt b/common/src/main/kotlin/com/cosmotech/common/postgres/PostgresConfiguration.kt index 021bbd144..d3f2e5503 100644 --- a/common/src/main/kotlin/com/cosmotech/common/postgres/PostgresConfiguration.kt +++ b/common/src/main/kotlin/com/cosmotech/common/postgres/PostgresConfiguration.kt @@ -1,62 +1,32 @@ +// Copyright (c) Cosmo Tech. +// Licensed under the MIT license. package com.cosmotech.common.postgres -import org.springframework.beans.factory.annotation.Qualifier import org.springframework.beans.factory.annotation.Value import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.jdbc.core.JdbcTemplate import org.springframework.jdbc.datasource.DriverManagerDataSource -import javax.sql.DataSource @Configuration class PostgresConfiguration { - @Value("\${csm.platform.internalResultServices.storage.admin.username}") - private lateinit var adminStorageUsername: String - @Value("\${csm.platform.internalResultServices.storage.admin.password}") - private lateinit var adminStoragePassword: String - @Value("\${csm.platform.internalResultServices.storage.host}") private lateinit var host: String - @Value("\${csm.platform.internalResultServices.storage.port}") private lateinit var port: String - @Value("\${csm.platform.internalResultServices.storage.db.name}") private lateinit var dbName: String - @Value("\${csm.platform.internalResultServices.storage.db.schema}") private lateinit var schema: String - - private val jdbcdriverClass = "org.postgresql.Driver" - - @Bean - fun adminDatasource(): DriverManagerDataSource { - val dataSource = - DriverManagerDataSource( - "jdbc:postgresql://$host:$port/$dbName", adminStorageUsername, adminStoragePassword) - dataSource.setDriverClassName(jdbcdriverClass) - return dataSource - } - - @Bean - fun adminJdbcTemplate( - @Qualifier("adminDatasource") dataSource: DataSource - ): JdbcTemplate { - return JdbcTemplate(dataSource) - } -} - -fun JdbcTemplate.existDB(name: String): Boolean { - return this.queryForList("SELECT * FROM pg_catalog.pg_database WHERE datname='$name'").size == 1 -} - -fun JdbcTemplate.existTable(name: String): Boolean { - return this.queryForList( - "SELECT * FROM information_schema.tables " + "WHERE table_name ilike '${name}'") - .size >= 1 -} - -fun String.toDataTableName(isProbeData: Boolean): String = - (if (isProbeData) "P_$this" else "CD_$this").lowercase() - -fun JdbcTemplate.createDB(name: String, comment: String? = null): String { - this.execute("CREATE DATABASE \"$name\"") - if (comment != null) this.execute("COMMENT ON DATABASE \"$name\" IS '$comment'") - return name -} - -fun JdbcTemplate.dropDB(name: String) { - if (this.existDB(name)) this.execute("DROP DATABASE \"$name\"") + @Value("\${csm.platform.internalResultServices.storage.admin.username}") + private lateinit var adminStorageUsername: String + @Value("\${csm.platform.internalResultServices.storage.admin.password}") + private lateinit var adminStoragePassword: String + @Value("\${csm.platform.internalResultServices.storage.host}") private lateinit var host: String + @Value("\${csm.platform.internalResultServices.storage.port}") private lateinit var port: String + @Value("\${csm.platform.internalResultServices.storage.datasets.name:cosmotech}") + private lateinit var dbName: String + + private val jdbcdriverClass = "org.postgresql.Driver" + + @Bean + fun adminJdbcTemplate(): JdbcTemplate { + val dataSource = + DriverManagerDataSource( + "jdbc:postgresql://$host:$port/$dbName", adminStorageUsername, adminStoragePassword) + dataSource.setDriverClassName(jdbcdriverClass) + return JdbcTemplate(dataSource) + } } diff --git a/config/application-dev.sample.yml b/config/application-dev.sample.yml index 2b0e249ac..6ad2a5ac7 100644 --- a/config/application-dev.sample.yml +++ b/config/application-dev.sample.yml @@ -117,7 +117,7 @@ csm: writer: password: "[fill-this-value]" username: cosmotech_api_writer - db: + datasets: name: cosmotech schema: datasets From f61e0578c6cb39ef060352583b6a1dfc93fcc4fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Laurent=20Al=C3=A9p=C3=A9e?= Date: Wed, 1 Oct 2025 18:12:16 +0200 Subject: [PATCH 3/5] add writer and reader jdbc bean --- .../common/postgres/PostgresConfiguration.kt | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/common/src/main/kotlin/com/cosmotech/common/postgres/PostgresConfiguration.kt b/common/src/main/kotlin/com/cosmotech/common/postgres/PostgresConfiguration.kt index d3f2e5503..2f609f5d4 100644 --- a/common/src/main/kotlin/com/cosmotech/common/postgres/PostgresConfiguration.kt +++ b/common/src/main/kotlin/com/cosmotech/common/postgres/PostgresConfiguration.kt @@ -14,6 +14,14 @@ class PostgresConfiguration { private lateinit var adminStorageUsername: String @Value("\${csm.platform.internalResultServices.storage.admin.password}") private lateinit var adminStoragePassword: String + @Value("\${csm.platform.internalResultServices.storage.reader.username}") + private lateinit var readerStorageUsername: String + @Value("\${csm.platform.internalResultServices.storage.reader.password}") + private lateinit var readerStoragePassword: String + @Value("\${csm.platform.internalResultServices.storage.writer.username}") + private lateinit var writerStorageUsername: String + @Value("\${csm.platform.internalResultServices.storage.writer.password}") + private lateinit var writerStoragePassword: String @Value("\${csm.platform.internalResultServices.storage.host}") private lateinit var host: String @Value("\${csm.platform.internalResultServices.storage.port}") private lateinit var port: String @Value("\${csm.platform.internalResultServices.storage.datasets.name:cosmotech}") @@ -29,4 +37,22 @@ class PostgresConfiguration { dataSource.setDriverClassName(jdbcdriverClass) return JdbcTemplate(dataSource) } + + @Bean + fun readerJdbcTemplate(): JdbcTemplate { + val dataSource = + DriverManagerDataSource( + "jdbc:postgresql://$host:$port/$dbName", readerStorageUsername, readerStoragePassword) + dataSource.setDriverClassName(jdbcdriverClass) + return JdbcTemplate(dataSource) + } + + @Bean + fun writerJdbcTemplate(): JdbcTemplate { + val dataSource = + DriverManagerDataSource( + "jdbc:postgresql://$host:$port/$dbName", writerStorageUsername, writerStoragePassword) + dataSource.setDriverClassName(jdbcdriverClass) + return JdbcTemplate(dataSource) + } } From 89ac5a1fe0e02ce25f09f8e0ca3711dbadbc1c60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Reynard?= Date: Thu, 2 Oct 2025 11:13:23 +0200 Subject: [PATCH 4/5] Refactor InternalResultServices - Deleted RabbitMQ-related configurations, services, models, listeners, and associated event handling logic. - Removed AMQP client implementation, event bus dependencies, and integration tests. - Refactored remaining components to eliminate references to `InternalResultServices`, `eventBus`, and AMQP-related environments. - Replace property `internalResultServices` by `databases` that now include `twincache` configuration and `storage` configuration - Adjust configurations --- .../cosmotech/api/home/ControllerTestBase.kt | 33 ++-- .../resources/application-test.yml | 41 ++-- api/src/main/resources/application.yml | 63 ++----- .../common/config/CsmPlatformProperties.kt | 175 +++++++----------- .../common/config/PostgresConfiguration.kt | 75 ++++++++ .../common/postgres/PostgresConfiguration.kt | 58 ------ .../com/cosmotech/common/redis/RedisConfig.kt | 4 +- .../com/cosmotech/common/tests/CsmTestBase.kt | 31 +--- config/application-dev.sample.yml | 38 +--- .../resources/application-dataset-test.yml | 56 +++--- .../dataset/service/DatasetServiceImpl.kt | 8 +- .../OrganizationServiceIntegrationTest.kt | 6 +- .../application-organization-test.yml | 55 +++--- .../service/OrganizationServiceImpl.kt | 2 +- run/build.gradle.kts | 6 - .../run/service/RunServiceIntegrationTest.kt | 23 ++- .../resources/application-run-test.yml | 70 +++---- .../com/cosmotech/run/RunContainerFactory.kt | 28 +-- .../cosmotech/run/config/RabbitMqConfig.kt | 65 ------- .../run/config/RabbitMqConfigModel.kt | 23 --- .../cosmotech/run/config/RunStorageConfig.kt | 65 ------- .../cosmotech/run/service/RunServiceImpl.kt | 125 ++++++------- .../amqp/AddQueueOnRunStartListener.kt | 25 --- .../run/service/amqp/AmqpClientServiceImpl.kt | 117 ------------ .../amqp/AmqpClientServiceInterface.kt | 10 - .../RemoveQueueOnWorkspaceDeletedListener.kt | 24 --- .../cosmotech/run/ContainerFactoryTests.kt | 63 ++----- .../service/RunnerServiceIntegrationTest.kt | 4 +- .../resources/application-runner-test.yml | 55 +++--- .../runner/service/RunnerApiServiceImpl.kt | 2 +- .../cosmotech/runner/service/RunnerService.kt | 2 +- .../service/SolutionServiceIntegrationTest.kt | 2 +- .../resources/application-solution-test.yml | 55 +++--- .../solution/service/SolutionServiceImpl.kt | 4 +- .../WorkspaceServiceIntegrationTest.kt | 2 +- .../resources/application-workspace-test.yml | 55 +++--- .../workspace/service/WorkspaceServiceImpl.kt | 8 +- 37 files changed, 512 insertions(+), 966 deletions(-) create mode 100644 common/src/main/kotlin/com/cosmotech/common/config/PostgresConfiguration.kt delete mode 100644 common/src/main/kotlin/com/cosmotech/common/postgres/PostgresConfiguration.kt delete mode 100644 run/src/main/kotlin/com/cosmotech/run/config/RabbitMqConfig.kt delete mode 100644 run/src/main/kotlin/com/cosmotech/run/config/RabbitMqConfigModel.kt delete mode 100644 run/src/main/kotlin/com/cosmotech/run/config/RunStorageConfig.kt delete mode 100644 run/src/main/kotlin/com/cosmotech/run/service/amqp/AddQueueOnRunStartListener.kt delete mode 100644 run/src/main/kotlin/com/cosmotech/run/service/amqp/AmqpClientServiceImpl.kt delete mode 100644 run/src/main/kotlin/com/cosmotech/run/service/amqp/AmqpClientServiceInterface.kt delete mode 100644 run/src/main/kotlin/com/cosmotech/run/service/amqp/RemoveQueueOnWorkspaceDeletedListener.kt diff --git a/api/src/integrationTest/kotlin/com/cosmotech/api/home/ControllerTestBase.kt b/api/src/integrationTest/kotlin/com/cosmotech/api/home/ControllerTestBase.kt index 0bf7be6ce..533a03c2a 100644 --- a/api/src/integrationTest/kotlin/com/cosmotech/api/home/ControllerTestBase.kt +++ b/api/src/integrationTest/kotlin/com/cosmotech/api/home/ControllerTestBase.kt @@ -150,40 +150,29 @@ abstract class ControllerTestBase : AbstractTestcontainersRedisTestBase() { } private fun initPostgresConfiguration(registry: DynamicPropertyRegistry) { - registry.add("csm.platform.internalResultServices.storage.host") { postgres.host } - registry.add("csm.platform.internalResultServices.storage.port") { - postgres.getMappedPort(POSTGRESQL_PORT) - } - registry.add("csm.platform.internalResultServices.storage.admin.username") { - ADMIN_USER_CREDENTIALS - } - registry.add("csm.platform.internalResultServices.storage.admin.password") { - ADMIN_USER_CREDENTIALS - } - registry.add("csm.platform.internalResultServices.storage.writer.username") { - WRITER_USER_CREDENTIALS - } - registry.add("csm.platform.internalResultServices.storage.writer.password") { - WRITER_USER_CREDENTIALS - } - registry.add("csm.platform.internalResultServices.storage.reader.username") { - READER_USER_CREDENTIALS - } - registry.add("csm.platform.internalResultServices.storage.reader.password") { - READER_USER_CREDENTIALS - } + registry.add("csm.platform.databases.data.host") { postgres.host } + registry.add("csm.platform.databases.data.port") { postgres.getMappedPort(POSTGRESQL_PORT) } + registry.add("csm.platform.databases.data.admin.username") { ADMIN_USER_CREDENTIALS } + registry.add("csm.platform.databases.data.admin.password") { ADMIN_USER_CREDENTIALS } + registry.add("csm.platform.databases.data.writer.username") { WRITER_USER_CREDENTIALS } + registry.add("csm.platform.databases.data.writer.password") { WRITER_USER_CREDENTIALS } + registry.add("csm.platform.databases.data.reader.username") { READER_USER_CREDENTIALS } + registry.add("csm.platform.databases.data.reader.password") { READER_USER_CREDENTIALS } } } @BeforeAll fun beforeAll() { redisStackServer.start() + localStackServer.start() postgres.start() } @AfterAll fun afterAll() { postgres.stop() + localStackServer.stop() + redisStackServer.stop() } override fun redisServers(): MutableCollection { diff --git a/api/src/integrationTest/resources/application-test.yml b/api/src/integrationTest/resources/application-test.yml index f666be051..0c9fe92b0 100644 --- a/api/src/integrationTest/resources/application-test.yml +++ b/api/src/integrationTest/resources/application-test.yml @@ -29,14 +29,9 @@ spring: csm: platform: - internalResultServices: - enabled: true - eventBus: - enabled: false containerRegistry: checkSolutionImage: false identityProvider: - code: keycloak authorizationUrl: "http://fake_url:8080/authorize" tokenUrl: "http://fake_url:8080/token" containerScopes: @@ -102,17 +97,31 @@ csm: - ReadWriteOnce requests: storage: 1Gi - twincache: - host: "localhost" - port: "6379" - username: "default" - # Leave it as blank as there's no auth with test container - password: - tls: - enabled: false - bundle: "" - connector: - default-page-size: 5 + databases: + resources: + host: "localhost" + port: "6379" + username: "default" + # Leave it as blank as there's no auth with test container + password: + tls: + enabled: false + bundle: "" + connector: + default-page-size: 5 + data: + schema: "postgres" + admin: + password: "password" + username: cosmotech_api_admin + host: "localhost" + port: 5432 + reader: + password: "password" + username: cosmotech_api_reader + writer: + password: "password" + username: cosmotech_api_writer rbac: enabled: true s3: diff --git a/api/src/main/resources/application.yml b/api/src/main/resources/application.yml index 68bcb1801..a483dd04b 100644 --- a/api/src/main/resources/application.yml +++ b/api/src/main/resources/application.yml @@ -5,21 +5,14 @@ logging: OkHttpClient: WARN spring: - rabbitmq: - host: ${csm.platform.internalResultServices.eventBus.host} - port: ${csm.platform.internalResultServices.eventBus.port} - password: ${csm.platform.internalResultServices.eventBus.listener.password} - username: ${csm.platform.internalResultServices.eventBus.listener.username} - ssl: - enabled: ${csm.platform.internalResultServices.eventBus.tls.enabled} data: redis: - host: ${csm.platform.twincache.host} - port: ${csm.platform.twincache.port} + host: ${csm.platform.databases.resources.host} + port: ${csm.platform.databases.resources.port} ssl: - enabled: ${csm.platform.twincache.tls.enabled} - password: ${csm.platform.twincache.password} - username: ${csm.platform.twincache.username} + enabled: ${csm.platform.databases.resources.tls.enabled} + password: ${csm.platform.databases.resources.password} + username: ${csm.platform.databases.resources.username} timeout: 60000 client-type: jedis servlet: @@ -86,23 +79,23 @@ csm: downSamplingDefaultEnabled: false downSamplingRetentionDays: 400 downSamplingBucketDurationMs: 3600000 - twincache: - host: "localhost" - port: "6379" - username: "default_user" - password: "default_password" - tls: - enabled: false - bundle: "" - run: - maxResult: 200 - runner: - maxResult: 200 - internalResultServices: - enabled: false - storage: + databases: + resources: + host: "localhost" + port: "6379" + username: "default_user" + password: "default_password" + tls: + enabled: false + bundle: "" + run: + defaultPageSize: 200 + runner: + defaultPageSize: 200 + data: host: "localhost" port: 5432 + schema: "cosmotech" reader: username: "storage_reader_username" password: "storage_reader_password" @@ -112,22 +105,6 @@ csm: admin: username: "storage_admin_username" password: "storage_admin_password" - eventBus: - enabled: true - tls: - enabled: false - bundle: "" - host: "localhost" - port: 5672 - default-exchange: "csm-exchange" - default-queue: "csm" - default-routing-key: "csm" - listener: - username: "eventbus_admin_username" - password: "eventbus_admin_password" - sender: - username: "eventbus_sender_username" - password: "eventbus_sender_password" s3: endpointUrl: "http://localhost:9000" bucketName: "cosmotech-api" diff --git a/common/src/main/kotlin/com/cosmotech/common/config/CsmPlatformProperties.kt b/common/src/main/kotlin/com/cosmotech/common/config/CsmPlatformProperties.kt index 4882bb2f0..f3e88a3dc 100644 --- a/common/src/main/kotlin/com/cosmotech/common/config/CsmPlatformProperties.kt +++ b/common/src/main/kotlin/com/cosmotech/common/config/CsmPlatformProperties.kt @@ -3,7 +3,6 @@ package com.cosmotech.common.config import com.cosmotech.common.security.ROLE_ORGANIZATION_USER -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.ConfigurationProperties /** Configuration Properties for the Cosmo Tech Platform */ @@ -33,8 +32,8 @@ data class CsmPlatformProperties( /** Identity provider used if openapi default configuration needs to be overwritten */ val identityProvider: CsmIdentityProvider, - /** Twin Data Layer configuration */ - val twincache: CsmTwinCacheProperties, + /** Data Layer configuration */ + val databases: CsmDatabasesProperties, /** RBAC / ACL configuration */ val rbac: CsmRbac = CsmRbac(), @@ -46,78 +45,8 @@ data class CsmPlatformProperties( val namespace: String = "phoenix", /** Persistent metrics configuration */ - val metrics: Metrics = Metrics(), - - /** Internal result data service configuration */ - val internalResultServices: CsmServiceResult?, + val metrics: Metrics = Metrics() ) { - @ConditionalOnProperty( - prefix = "csm.platform.internalResultServices.enabled", - havingValue = "true", - matchIfMissing = false) - data class CsmServiceResult( - /** Define if current API use internal result data service or cloud one */ - val enabled: Boolean = false, - - /** Storage configuration */ - val storage: CsmStorage, - - /** Queue configuration */ - val eventBus: CsmEventBus - ) { - data class CsmStorage( - /** Define if current API use internal storage for probes or not */ - val enabled: Boolean = true, - - /** Storage host */ - val host: String, - - /** Storage port */ - val port: Int = 5432, - - /** Storage reader user configuration */ - val reader: CsmStorageUser, - - /** Storage writer user configuration */ - val writer: CsmStorageUser, - - /** Storage admin user configuration */ - val admin: CsmStorageUser - ) { - data class CsmStorageUser(val username: String, val password: String) - } - - data class CsmEventBus( - /** Define if current API use event bus within internal result data service or not */ - val enabled: Boolean = true, - - /** TLS Platform bundle config */ - val tls: TLSConfig = TLSConfig(), - - /** EventBus host */ - val host: String, - - /** EventBus port */ - val port: Int = 5672, - - /** EventBus default exchange */ - val defaultExchange: String = "csm-exchange", - - /** EventBus default queue */ - val defaultQueue: String = "csm", - - /** EventBus default routing key */ - val defaultRoutingKey: String = "csm", - - /** EventBus listener user configuration */ - val listener: CsmEventBusUser, - - /** EventBus sender user configuration */ - val sender: CsmEventBusUser - ) { - data class CsmEventBusUser(val username: String, val password: String) - } - } data class Metrics( /** Enable Metrics service */ @@ -340,54 +269,82 @@ data class CsmPlatformProperties( ) } - data class CsmTwinCacheProperties( - /** Twin cache host */ - val host: String, + data class CsmDatabasesProperties( + val resources: CsmResourcesProperties, + val data: CsmDataIOProperties + ) { - /** Twin cache port */ - val port: String = "6379", + data class CsmResourcesProperties( + /** Host */ + val host: String, - /** Twin cache user */ - val username: String = "default", + /** Port */ + val port: String = "6379", - /** Twin cache password */ - val password: String, + /** User */ + val username: String = "default", - /** Twin cache query timeout. Kill a query after specified timeout (in millis) default 5000 */ - val queryTimeout: Long = 5000, + /** Password */ + val password: String, - /** - * After specified timeout the query bulk will be deleted from Redis (in seconds) default - * 86400 = 24h - */ - val queryBulkTTL: Long = 86400, + /** Query timeout. Kill a query after specified timeout (in millis) default 5000 */ + val queryTimeout: Long = 5000, - /** Twin cache query page information for organization */ - val organization: PageSizing = PageSizing(), + /** + * After specified timeout the query bulk will be deleted from Redis (in seconds) default + * 86400 = 24h + */ + val queryBulkTTL: Long = 86400, - /** Twin cache query page information for workspace */ - val workspace: PageSizing = PageSizing(), + /** Organization's resource query page information */ + val organization: PageSizing = PageSizing(), - /** Twin cache query page information for runner */ - val runner: PageSizing = PageSizing(), + /** Workspace's resource query page information */ + val workspace: PageSizing = PageSizing(), - /** Twin cache query page information for dataset */ - val dataset: PageSizing = PageSizing(), + /** Runner's resource query page information */ + val runner: PageSizing = PageSizing(), - /** Twin cache query page information for run */ - val run: PageSizing = PageSizing(), + /** Dataset's resource query page information */ + val dataset: PageSizing = PageSizing(), - /** Twin cache query page information for solution */ - val solution: PageSizing = PageSizing(), + /** Run's resource query page information */ + val run: PageSizing = PageSizing(), - /** TLS Platform bundle config */ - val tls: TLSConfig = TLSConfig() - ) { + /** Solution's resource query page information */ + val solution: PageSizing = PageSizing(), - data class PageSizing( - /** Max result for a single page */ - val defaultPageSize: Int = 50 - ) + /** TLS Platform bundle config */ + val tls: TLSConfig = TLSConfig() + ) { + + data class PageSizing( + /** Max result for a single page */ + val defaultPageSize: Int = 50 + ) + } + + data class CsmDataIOProperties( + /** Storage host */ + val host: String, + + /** Storage port */ + val port: Int = 5432, + + /** Storage schema */ + val schema: String = "cosmotech", + + /** Storage reader user configuration */ + val reader: CsmStorageUser, + + /** Storage writer user configuration */ + val writer: CsmStorageUser, + + /** Storage admin user configuration */ + val admin: CsmStorageUser + ) { + data class CsmStorageUser(val username: String, val password: String) + } } data class CsmRbac( diff --git a/common/src/main/kotlin/com/cosmotech/common/config/PostgresConfiguration.kt b/common/src/main/kotlin/com/cosmotech/common/config/PostgresConfiguration.kt new file mode 100644 index 000000000..20e129df8 --- /dev/null +++ b/common/src/main/kotlin/com/cosmotech/common/config/PostgresConfiguration.kt @@ -0,0 +1,75 @@ +// Copyright (c) Cosmo Tech. +// Licensed under the MIT license. +package com.cosmotech.common.config + +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.jdbc.core.JdbcTemplate +import org.springframework.jdbc.datasource.DriverManagerDataSource + +@Configuration +class PostgresConfiguration(val csmPlatformProperties: CsmPlatformProperties) { + + private val jdbcDriverClass = "org.postgresql.Driver" + + private val jdbcUrl = + "jdbc:postgresql://${csmPlatformProperties.databases.data.host}" + + ":${csmPlatformProperties.databases.data.port}" + + "/${csmPlatformProperties.databases.data.schema}" + + @Bean + fun adminUserJdbcTemplate(): JdbcTemplate { + val dataSource = + DriverManagerDataSource( + jdbcUrl, + csmPlatformProperties.databases.data.admin.username, + csmPlatformProperties.databases.data.admin.password) + dataSource.setDriverClassName(jdbcDriverClass) + return JdbcTemplate(dataSource) + } + + @Bean + fun readerJdbcTemplate(): JdbcTemplate { + val dataSource = + DriverManagerDataSource( + jdbcUrl, + csmPlatformProperties.databases.data.reader.username, + csmPlatformProperties.databases.data.reader.password) + dataSource.setDriverClassName(jdbcDriverClass) + return JdbcTemplate(dataSource) + } + + @Bean + fun writerJdbcTemplate(): JdbcTemplate { + val dataSource = + DriverManagerDataSource( + jdbcUrl, + csmPlatformProperties.databases.data.writer.username, + csmPlatformProperties.databases.data.writer.password) + dataSource.setDriverClassName(jdbcDriverClass) + return JdbcTemplate(dataSource) + } +} + +fun JdbcTemplate.existDB(name: String): Boolean { + return this.queryForList("SELECT * FROM pg_catalog.pg_database WHERE datname='$name'").size == 1 +} + +fun JdbcTemplate.existTable(name: String): Boolean { + return this.queryForList( + "SELECT * FROM information_schema.tables " + "WHERE table_name ilike '${name}'") + .size >= 1 +} + +fun String.toDataTableName(isProbeData: Boolean): String = + (if (isProbeData) "P_$this" else "CD_$this").lowercase() + +fun JdbcTemplate.createDB(name: String, comment: String? = null): String { + this.execute("CREATE DATABASE \"$name\"") + if (comment != null) this.execute("COMMENT ON DATABASE \"$name\" IS '$comment'") + return name +} + +fun JdbcTemplate.dropDB(name: String) { + if (this.existDB(name)) this.execute("DROP DATABASE \"$name\"") +} diff --git a/common/src/main/kotlin/com/cosmotech/common/postgres/PostgresConfiguration.kt b/common/src/main/kotlin/com/cosmotech/common/postgres/PostgresConfiguration.kt deleted file mode 100644 index 2f609f5d4..000000000 --- a/common/src/main/kotlin/com/cosmotech/common/postgres/PostgresConfiguration.kt +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) Cosmo Tech. -// Licensed under the MIT license. -package com.cosmotech.common.postgres - -import org.springframework.beans.factory.annotation.Value -import org.springframework.context.annotation.Bean -import org.springframework.context.annotation.Configuration -import org.springframework.jdbc.core.JdbcTemplate -import org.springframework.jdbc.datasource.DriverManagerDataSource - -@Configuration -class PostgresConfiguration { - @Value("\${csm.platform.internalResultServices.storage.admin.username}") - private lateinit var adminStorageUsername: String - @Value("\${csm.platform.internalResultServices.storage.admin.password}") - private lateinit var adminStoragePassword: String - @Value("\${csm.platform.internalResultServices.storage.reader.username}") - private lateinit var readerStorageUsername: String - @Value("\${csm.platform.internalResultServices.storage.reader.password}") - private lateinit var readerStoragePassword: String - @Value("\${csm.platform.internalResultServices.storage.writer.username}") - private lateinit var writerStorageUsername: String - @Value("\${csm.platform.internalResultServices.storage.writer.password}") - private lateinit var writerStoragePassword: String - @Value("\${csm.platform.internalResultServices.storage.host}") private lateinit var host: String - @Value("\${csm.platform.internalResultServices.storage.port}") private lateinit var port: String - @Value("\${csm.platform.internalResultServices.storage.datasets.name:cosmotech}") - private lateinit var dbName: String - - private val jdbcdriverClass = "org.postgresql.Driver" - - @Bean - fun adminJdbcTemplate(): JdbcTemplate { - val dataSource = - DriverManagerDataSource( - "jdbc:postgresql://$host:$port/$dbName", adminStorageUsername, adminStoragePassword) - dataSource.setDriverClassName(jdbcdriverClass) - return JdbcTemplate(dataSource) - } - - @Bean - fun readerJdbcTemplate(): JdbcTemplate { - val dataSource = - DriverManagerDataSource( - "jdbc:postgresql://$host:$port/$dbName", readerStorageUsername, readerStoragePassword) - dataSource.setDriverClassName(jdbcdriverClass) - return JdbcTemplate(dataSource) - } - - @Bean - fun writerJdbcTemplate(): JdbcTemplate { - val dataSource = - DriverManagerDataSource( - "jdbc:postgresql://$host:$port/$dbName", writerStorageUsername, writerStoragePassword) - dataSource.setDriverClassName(jdbcdriverClass) - return JdbcTemplate(dataSource) - } -} diff --git a/common/src/main/kotlin/com/cosmotech/common/redis/RedisConfig.kt b/common/src/main/kotlin/com/cosmotech/common/redis/RedisConfig.kt index ccb221545..78565f66d 100644 --- a/common/src/main/kotlin/com/cosmotech/common/redis/RedisConfig.kt +++ b/common/src/main/kotlin/com/cosmotech/common/redis/RedisConfig.kt @@ -20,11 +20,11 @@ open class RedisConfig { @Value("\${spring.data.redis.port}") private lateinit var twincachePort: String // This property path is not compatible with spring.data.redis used by redis-om auto configuration - @Value("\${csm.platform.twincache.tls.enabled}") private val tlsEnabled: Boolean = false + @Value("\${csm.platform.databases.resources.tls.enabled}") private val tlsEnabled: Boolean = false // This property path is not compatible with spring.data.redis used by redis-om auto configuration // It is duplicated since spring.data.redis.ssl.bundle cannot be empty - @Value("\${csm.platform.twincache.tls.bundle}") private var tlsBundle: String = "" + @Value("\${csm.platform.databases.resources.tls.bundle}") private var tlsBundle: String = "" @Value("\${spring.data.redis.password}") private lateinit var twincachePassword: String diff --git a/common/src/main/kotlin/com/cosmotech/common/tests/CsmTestBase.kt b/common/src/main/kotlin/com/cosmotech/common/tests/CsmTestBase.kt index 0cfc4f3d2..a85bf514e 100644 --- a/common/src/main/kotlin/com/cosmotech/common/tests/CsmTestBase.kt +++ b/common/src/main/kotlin/com/cosmotech/common/tests/CsmTestBase.kt @@ -71,28 +71,15 @@ open class CsmTestBase : AbstractTestcontainersRedisTestBase() { } private fun initPostgresConfiguration(registry: DynamicPropertyRegistry) { - registry.add("csm.platform.internalResultServices.storage.host") { postgres.host } - registry.add("csm.platform.internalResultServices.storage.port") { - postgres.getMappedPort(POSTGRESQL_PORT) - } - registry.add("csm.platform.internalResultServices.storage.admin.username") { - ADMIN_USER_CREDENTIALS - } - registry.add("csm.platform.internalResultServices.storage.admin.password") { - ADMIN_USER_CREDENTIALS - } - registry.add("csm.platform.internalResultServices.storage.writer.username") { - WRITER_USER_CREDENTIALS - } - registry.add("csm.platform.internalResultServices.storage.writer.password") { - WRITER_USER_CREDENTIALS - } - registry.add("csm.platform.internalResultServices.storage.reader.username") { - READER_USER_CREDENTIALS - } - registry.add("csm.platform.internalResultServices.storage.reader.password") { - READER_USER_CREDENTIALS - } + registry.add("csm.platform.databases.data.host") { postgres.host } + registry.add("csm.platform.databases.data.schema") { postgres.databaseName } + registry.add("csm.platform.databases.data.port") { postgres.getMappedPort(POSTGRESQL_PORT) } + registry.add("csm.platform.databases.data.admin.username") { ADMIN_USER_CREDENTIALS } + registry.add("csm.platform.databases.data.admin.password") { ADMIN_USER_CREDENTIALS } + registry.add("csm.platform.databases.data.writer.username") { WRITER_USER_CREDENTIALS } + registry.add("csm.platform.databases.data.writer.password") { WRITER_USER_CREDENTIALS } + registry.add("csm.platform.databases.data.reader.username") { READER_USER_CREDENTIALS } + registry.add("csm.platform.databases.data.reader.password") { READER_USER_CREDENTIALS } } } diff --git a/config/application-dev.sample.yml b/config/application-dev.sample.yml index 6ad2a5ac7..0e68a66e3 100644 --- a/config/application-dev.sample.yml +++ b/config/application-dev.sample.yml @@ -61,7 +61,6 @@ csm: bundle: "keycloak" audience: account authorizationUrl: "[fill-this-value]" # keycloak auth endpoint - code: keycloak defaultScopes: openid: OpenId Scope identity: @@ -84,41 +83,24 @@ csm: namespace: "[fill-this-value]" # NAMESPACE service-account-name: "[fill-this-value]" # e.g argo-workflows-$NAMESPACE-service-account nodePoolLabel: "" - twincache: - host: "localhost" - password: "[fill-this-value]" - port: 6379 - tls: - enabled: false - username: default - internalResultServices: - enabled: false - eventBus: - enabled: false - host: "[fill-this-value]" #e.g rabbitmq-NAMESPACE.NAMESPACE.svc.cluster.local - listener: - password: "[fill-this-value]" - username: "[fill-this-value]" - port: 5672 - sender: - password: "[fill-this-value]" - username: "[fill-this-value]" + databases: + resources: + host: "localhost" + port: 6379 + username: default + password: "[fill-this-value]" tls: enabled: false - storage: - host: "[fill-this-value]" # postgresql-NAMESPACE.NAMESPACE.svc.cluster.local - port: 5432 + data: admin: password: "[fill-this-value]" username: "[fill-this-value]" + host: "[fill-this-value]" # postgresql-NAMESPACE.NAMESPACE.svc.cluster.local + port: 5432 + schema: "[fill-this-value]" reader: password: "[fill-this-value]" username: cosmotech_api_reader writer: password: "[fill-this-value]" username: cosmotech_api_writer - datasets: - name: cosmotech - schema: datasets - - diff --git a/dataset/src/integrationTest/resources/application-dataset-test.yml b/dataset/src/integrationTest/resources/application-dataset-test.yml index 8f67776eb..37f3eb3c6 100644 --- a/dataset/src/integrationTest/resources/application-dataset-test.yml +++ b/dataset/src/integrationTest/resources/application-dataset-test.yml @@ -3,6 +3,14 @@ management: enabled-by-default: false spring: + security: + oauth2: + resource-server: + jwt: + issuer-uri: "http://localhost:9000/realms/fakeTenant" + jwk-set-uri: "http://localhost:9000/realms/fakeTenant/protocol/openid-connect/certs" + audiences: + - "cosmotech" data: redis: host: "localhost" @@ -34,7 +42,6 @@ csm: password: "test" checkSolutionImage: false identityProvider: - code: on_premise_one authorizationUrl: "http://fake_url:8080/authorize" tokenUrl: "http://fake_url:8080/token" containerScopes: @@ -89,18 +96,30 @@ csm: - ReadWriteOnce requests: storage: 1Gi - twincache: - host: "localhost" - port: "6379" - username: "default" - useGraphModule: true - # Leave it as blank as there's no auth with test container - password: - tls: - enabled: false - bundle: "" - dataset: - default-page-size: 5 + databases: + resources: + host: "localhost" + port: "6379" + username: "default" + # Leave it as blank as there's no auth with test container + password: + tls: + enabled: false + bundle: "" + connector: + default-page-size: 5 + data: + admin: + password: "password" + username: cosmotech_api_admin + host: "localhost" + port: 5432 + reader: + password: "password" + username: cosmotech_api_reader + writer: + password: "password" + username: cosmotech_api_writer rbac: enabled: true s3: @@ -109,14 +128,3 @@ csm: accessKeyId: "s3_username" secretAccessKey: "s3_password" region: "dummy" - internalResultServices: - enabled: true - eventBus: - enabled: false - host: "http://localhost:1234" - listener: - username: "fakeUser" - password: "fakePwd" - sender: - username: "fakeUser2" - password: "fakePwd2" diff --git a/dataset/src/main/kotlin/com/cosmotech/dataset/service/DatasetServiceImpl.kt b/dataset/src/main/kotlin/com/cosmotech/dataset/service/DatasetServiceImpl.kt index 86e5f6fe4..08696cdbd 100644 --- a/dataset/src/main/kotlin/com/cosmotech/dataset/service/DatasetServiceImpl.kt +++ b/dataset/src/main/kotlin/com/cosmotech/dataset/service/DatasetServiceImpl.kt @@ -218,7 +218,7 @@ class DatasetServiceImpl( size: Int? ): List { val workspace = workspaceService.getVerifiedWorkspace(organizationId, workspaceId) - val defaultPageSize = csmPlatformProperties.twincache.dataset.defaultPageSize + val defaultPageSize = csmPlatformProperties.databases.resources.dataset.defaultPageSize val pageable = constructPageRequest(page, size, defaultPageSize) val isAdmin = csmRbac.isAdmin( @@ -514,7 +514,7 @@ class DatasetServiceImpl( ): List { val dataset = getVerifiedDataset(organizationId, workspaceId, datasetId, PERMISSION_READ) - val defaultPageSize = csmPlatformProperties.twincache.dataset.defaultPageSize + val defaultPageSize = csmPlatformProperties.databases.resources.dataset.defaultPageSize val pageable = constructPageRequest(page, size, defaultPageSize) val isAdmin = csmRbac.isAdmin(dataset.security.toGenericSecurity(datasetId), getCommonRolesDefinition()) @@ -671,7 +671,7 @@ class DatasetServiceImpl( } getVerifiedDataset(organizationId, workspaceId, datasetId) - val defaultPageSize = csmPlatformProperties.twincache.dataset.defaultPageSize + val defaultPageSize = csmPlatformProperties.databases.resources.dataset.defaultPageSize val pageable = constructPageRequest(page, size, defaultPageSize) val datasetPartList = if (pageable != null) { @@ -701,7 +701,7 @@ class DatasetServiceImpl( } workspaceService.getVerifiedWorkspace(organizationId, workspaceId) - val defaultPageSize = csmPlatformProperties.twincache.dataset.defaultPageSize + val defaultPageSize = csmPlatformProperties.databases.resources.dataset.defaultPageSize val pageable = constructPageRequest(page, size, defaultPageSize) val datasetList = if (pageable != null) { diff --git a/organization/src/integrationTest/kotlin/com/cosmotech/organization/service/OrganizationServiceIntegrationTest.kt b/organization/src/integrationTest/kotlin/com/cosmotech/organization/service/OrganizationServiceIntegrationTest.kt index 4e6f4fbb0..a1189a00c 100644 --- a/organization/src/integrationTest/kotlin/com/cosmotech/organization/service/OrganizationServiceIntegrationTest.kt +++ b/organization/src/integrationTest/kotlin/com/cosmotech/organization/service/OrganizationServiceIntegrationTest.kt @@ -111,7 +111,7 @@ class OrganizationServiceIntegrationTest : CsmTestBase() { @Test fun `listOrganizations with correct values`() { val numberOfOrganizationToCreate = 20 - val defaultPageSize = csmPlatformProperties.twincache.organization.defaultPageSize + val defaultPageSize = csmPlatformProperties.databases.resources.organization.defaultPageSize batchOrganizationCreation(numberOfOrganizationToCreate) testlistOrganizations(null, null, numberOfOrganizationToCreate) @@ -1101,7 +1101,7 @@ class OrganizationServiceIntegrationTest : CsmTestBase() { @Test fun `find All Organizations with correct values`() { val numberOfOrganizationToCreate = 20 - val defaultPageSize = csmPlatformProperties.twincache.organization.defaultPageSize + val defaultPageSize = csmPlatformProperties.databases.resources.organization.defaultPageSize batchOrganizationCreation(numberOfOrganizationToCreate) testlistOrganizations(null, null, numberOfOrganizationToCreate) @@ -2175,7 +2175,7 @@ class OrganizationServiceIntegrationTest : CsmTestBase() { numberOfOrganizationCreated: Int, numberOfOrganizationReachableByTestUser: Int ) { - val defaultPageSize = csmPlatformProperties.twincache.organization.defaultPageSize + val defaultPageSize = csmPlatformProperties.databases.resources.organization.defaultPageSize testlistOrganizations(null, null, numberOfOrganizationReachableByTestUser) testlistOrganizations(0, null, defaultPageSize) diff --git a/organization/src/integrationTest/resources/application-organization-test.yml b/organization/src/integrationTest/resources/application-organization-test.yml index e7c3e8327..a0884c751 100644 --- a/organization/src/integrationTest/resources/application-organization-test.yml +++ b/organization/src/integrationTest/resources/application-organization-test.yml @@ -3,6 +3,14 @@ management: enabled-by-default: false spring: + security: + oauth2: + resource-server: + jwt: + issuer-uri: "http://localhost:9000/realms/fakeTenant" + jwk-set-uri: "http://localhost:9000/realms/fakeTenant/protocol/openid-connect/certs" + audiences: + - "cosmotech" data: redis: host: "localhost" @@ -30,7 +38,6 @@ spring: csm: platform: identityProvider: - code: on_premise_one authorizationUrl: "http://fake_url:8080/authorize" tokenUrl: "http://fake_url:8080/token" containerScopes: @@ -96,17 +103,30 @@ csm: - ReadWriteOnce requests: storage: 1Gi - twincache: - host: "localhost" - port: "6379" - username: "default" - # Leave it as blank as there's no auth with test container - password: - tls: - enabled: false - bundle: "" - organization: - default-page-size: 5 + databases: + resources: + host: "localhost" + port: "6379" + username: "default" + # Leave it as blank as there's no auth with test container + password: + tls: + enabled: false + bundle: "" + organization: + default-page-size: 5 + data: + admin: + password: "password" + username: cosmotech_api_admin + host: "localhost" + port: 5432 + reader: + password: "password" + username: cosmotech_api_reader + writer: + password: "password" + username: cosmotech_api_writer rbac: enabled: true s3: @@ -115,14 +135,3 @@ csm: accessKeyId: "s3_username" secretAccessKey: "s3_password" region: "dummy" - internalResultServices: - enabled: true - eventBus: - enabled: false - host: "http://localhost:1234" - listener: - username: "fakeUser" - password: "fakePwd" - sender: - username: "fakeUser2" - password: "fakePwd2" diff --git a/organization/src/main/kotlin/com/cosmotech/organization/service/OrganizationServiceImpl.kt b/organization/src/main/kotlin/com/cosmotech/organization/service/OrganizationServiceImpl.kt index 876b58368..0c8438456 100644 --- a/organization/src/main/kotlin/com/cosmotech/organization/service/OrganizationServiceImpl.kt +++ b/organization/src/main/kotlin/com/cosmotech/organization/service/OrganizationServiceImpl.kt @@ -43,7 +43,7 @@ class OrganizationServiceImpl( ) : CsmPhoenixService(), OrganizationApiServiceInterface { override fun listOrganizations(page: Int?, size: Int?): List { - val defaultPageSize = csmPlatformProperties.twincache.organization.defaultPageSize + val defaultPageSize = csmPlatformProperties.databases.resources.organization.defaultPageSize val pageable = constructPageRequest(page, size, defaultPageSize) val isAdmin = csmAdmin.verifyCurrentRolesAdmin() val result: MutableList diff --git a/run/build.gradle.kts b/run/build.gradle.kts index b404123e9..e319304c8 100644 --- a/run/build.gradle.kts +++ b/run/build.gradle.kts @@ -7,8 +7,6 @@ plugins { id("org.jetbrains.kotlinx.kover") } val argoClientJavaVersion = "v3.5.11" val retroFitVersion = "2.11.0" val okHttpBom = "4.12.0" -val testContainersRabbitMQVersion = "1.20.6" -val springRabbitMQTestVersion = "3.2.4" dependencies { implementation(projects.cosmotechDatasetApi) @@ -24,10 +22,6 @@ dependencies { implementation(platform("com.squareup.okhttp3:okhttp-bom:$okHttpBom")) implementation("com.squareup.okhttp3:okhttp") implementation("com.squareup.okhttp3:logging-interceptor") - implementation("org.springframework.boot:spring-boot-starter-amqp") - - testImplementation("org.testcontainers:rabbitmq:$testContainersRabbitMQVersion") - testImplementation("org.springframework.amqp:spring-rabbit-test:$springRabbitMQTestVersion") } tasks.withType { additionalProperties.put("modelMutable", false) } diff --git a/run/src/integrationTest/kotlin/com/cosmotech/run/service/RunServiceIntegrationTest.kt b/run/src/integrationTest/kotlin/com/cosmotech/run/service/RunServiceIntegrationTest.kt index dd6082915..c9f598667 100644 --- a/run/src/integrationTest/kotlin/com/cosmotech/run/service/RunServiceIntegrationTest.kt +++ b/run/src/integrationTest/kotlin/com/cosmotech/run/service/RunServiceIntegrationTest.kt @@ -3,6 +3,9 @@ package com.cosmotech.run.service import com.cosmotech.common.config.CsmPlatformProperties +import com.cosmotech.common.config.existDB +import com.cosmotech.common.config.existTable +import com.cosmotech.common.config.toDataTableName import com.cosmotech.common.events.RunDeleted import com.cosmotech.common.events.RunStart import com.cosmotech.common.rbac.ROLE_ADMIN @@ -21,9 +24,6 @@ import com.cosmotech.organization.domain.OrganizationCreateRequest import com.cosmotech.organization.domain.OrganizationSecurity import com.cosmotech.run.RunApiServiceInterface import com.cosmotech.run.RunContainerFactory -import com.cosmotech.run.config.existDB -import com.cosmotech.run.config.existTable -import com.cosmotech.run.config.toDataTableName import com.cosmotech.run.domain.Run import com.cosmotech.run.domain.RunDataQuery import com.cosmotech.run.domain.RunEditInfo @@ -112,7 +112,7 @@ class RunServiceIntegrationTest : CsmTestBase() { @SpykBean @Autowired lateinit var runApiService: RunApiServiceInterface @Autowired lateinit var eventPublisher: com.cosmotech.common.events.CsmEventPublisher - @Autowired lateinit var adminRunStorageTemplate: JdbcTemplate + @Autowired lateinit var adminUserJdbcTemplate: JdbcTemplate lateinit var dataset: DatasetCreateRequest lateinit var solution: SolutionCreateRequest @@ -329,7 +329,7 @@ class RunServiceIntegrationTest : CsmTestBase() { @Test fun `test find All Runs with different pagination params`() { val numberOfRuns = 20 - val defaultPageSize = csmPlatformProperties.twincache.run.defaultPageSize + val defaultPageSize = csmPlatformProperties.databases.resources.run.defaultPageSize val expectedSize = 15 IntRange(1, numberOfRuns).forEach { @@ -463,15 +463,14 @@ class RunServiceIntegrationTest : CsmTestBase() { fun setUp() { runSavedId = mockStartRun(organizationSaved.id, workspaceSaved.id, runnerSaved.id, solutionSaved.id) - assertTrue(adminRunStorageTemplate.existDB(runSavedId)) + assertTrue(adminUserJdbcTemplate.existDB(runSavedId)) - val internalResultServices = csmPlatformProperties.internalResultServices!! + val databaseIO = csmPlatformProperties.databases.data val runtimeDS = DriverManagerDataSource( - "jdbc:postgresql://${internalResultServices.storage.host}:${internalResultServices.storage.port}" + - "/$runSavedId", - internalResultServices.storage.reader.username, - internalResultServices.storage.reader.password) + "jdbc:postgresql://${databaseIO.host}:${databaseIO.port}" + "/$runSavedId", + databaseIO.reader.username, + databaseIO.reader.password) runtimeDS.setDriverClassName("org.postgresql.Driver") readerRunStorageTemplate = JdbcTemplate(runtimeDS) } @@ -479,7 +478,7 @@ class RunServiceIntegrationTest : CsmTestBase() { @Test fun `test deleteRun should remove the database`() { runApiService.deleteRun(organizationSaved.id, workspaceSaved.id, runnerSaved.id, runSavedId) - assertFalse(adminRunStorageTemplate.existDB(runSavedId)) + assertFalse(adminUserJdbcTemplate.existDB(runSavedId)) } @Test diff --git a/run/src/integrationTest/resources/application-run-test.yml b/run/src/integrationTest/resources/application-run-test.yml index 91f3cfda5..cedc71b8f 100644 --- a/run/src/integrationTest/resources/application-run-test.yml +++ b/run/src/integrationTest/resources/application-run-test.yml @@ -1,9 +1,12 @@ spring: - rabbitmq: - host: "localhost" - port: 5672 - username: "user_rabbit" - password: "pwd_rabbit" + security: + oauth2: + resource-server: + jwt: + issuer-uri: "http://localhost:9000/realms/fakeTenant" + jwk-set-uri: "http://localhost:9000/realms/fakeTenant/protocol/openid-connect/certs" + audiences: + - "cosmotech" data: redis: host: "localhost" @@ -41,7 +44,6 @@ management: csm: platform: identityProvider: - code: on_premise_one authorizationUrl: "http://fake_url:8080/authorize" tokenUrl: "http://fake_url:8080/token" containerScopes: @@ -107,45 +109,31 @@ csm: - ReadWriteOnce requests: storage: 1Gi - twincache: - host: "localhost" - port: "6379" - username: "default" - password: "my-wonderful-password" - useGraphModule: true - tls: - enabled: false - bundle: "" - run: - default-page-size: 5 - internalResultServices: - enabled: true - storage: + databases: + resources: host: "localhost" - port: "5432" - reader: - username: "readusertest" - password: "readusertest" - writer: - username: "writeusertest" - password: "writeusertest" - admin: - username: "adminusertest" - password: "adminusertest" - eventBus: - host: "localhost" - default-exchange: "csm-exchange" - default-queue: "csm" - default-routing-key: "csm" - sender: - username: "user3" - password: "pwd3" - listener: - username: "user4" - password: "pwd4" + port: "6379" + username: "default" + # Leave it as blank as there's no auth with test container + password: tls: enabled: false bundle: "" + run: + default-page-size: 5 + data: + admin: + password: "password" + username: cosmotech_api_admin + host: "localhost" + schema: "postgres" + port: 5432 + reader: + password: "password" + username: cosmotech_api_reader + writer: + password: "password" + username: cosmotech_api_writer rbac: enabled: true containerRegistry: diff --git a/run/src/main/kotlin/com/cosmotech/run/RunContainerFactory.kt b/run/src/main/kotlin/com/cosmotech/run/RunContainerFactory.kt index 7503ad0cb..8194e1c37 100644 --- a/run/src/main/kotlin/com/cosmotech/run/RunContainerFactory.kt +++ b/run/src/main/kotlin/com/cosmotech/run/RunContainerFactory.kt @@ -26,7 +26,6 @@ import com.cosmotech.workspace.domain.Workspace import org.springframework.stereotype.Component const val CONTAINER_CSM_ORC = "csmorchestrator" -internal const val IDENTITY_PROVIDER = "IDENTITY_PROVIDER" internal const val IDP_CLIENT_ID = "IDP_CLIENT_ID" internal const val IDP_CLIENT_SECRET = "IDP_CLIENT_SECRET" internal const val IDP_BASE_URL = "IDP_BASE_URL" @@ -53,7 +52,6 @@ private const val PARAMETERS_RUN_VAR = "CSM_RUN_ID" private const val RUN_TEMPLATE_ID_VAR = "CSM_RUN_TEMPLATE_ID" private const val ENTRYPOINT_NAME = "entrypoint.py" -private const val EVENT_HUB_MEASURES_VAR = "CSM_PROBES_MEASURES_TOPIC" private const val NODE_PARAM_NONE = "%NONE%" const val NODE_LABEL_DEFAULT = "basic" private const val NODE_LABEL_HIGH_CPU = "highcpu" @@ -61,9 +59,6 @@ private const val NODE_LABEL_HIGH_MEMORY = "highmemory" const val NODE_LABEL_SUFFIX = "pool" private const val GENERATE_NAME_SUFFIX = "-" -internal const val CSM_AMQPCONSUMER_USER_ENV_VAR = "CSM_AMQPCONSUMER_USER" -internal const val CSM_AMQPCONSUMER_PASSWORD_ENV_VAR = "CSM_AMQPCONSUMER_PASSWORD" - internal const val CSM_JOB_ID_LABEL_KEY = "cosmotech.com/job_id" internal const val WORKFLOW_TYPE_LABEL = "cosmotech.com/workflowtype" internal const val ORGANIZATION_ID_LABEL = "cosmotech.com/organizationId" @@ -174,8 +169,6 @@ class RunContainerFactory( envVars[RUN_TEMPLATE_ID_VAR] = runTemplateId - envVars.putAll(getEventBusEnvVars(workspace)) - val customSizing = runSizing ?: (runTemplateSizing ?: defaultSizing) containers.add( @@ -278,25 +271,6 @@ class RunContainerFactory( "runTemplateId $runTemplateId not found in Solution ${solution.id}") } - private fun getEventBusEnvVars(workspace: Workspace): Map { - - val envVars: MutableMap = mutableMapOf() - - if (csmPlatformProperties.internalResultServices?.enabled == true) { - val internalServiceInfo = csmPlatformProperties.internalResultServices!! - val eventHubUri = - "amqp://" + - "${internalServiceInfo.eventBus.host}:${internalServiceInfo.eventBus.port}/${workspace.id}" - envVars.putAll( - mapOf( - CSM_AMQPCONSUMER_USER_ENV_VAR to internalServiceInfo.eventBus.sender.username, - CSM_AMQPCONSUMER_PASSWORD_ENV_VAR to internalServiceInfo.eventBus.sender.password, - EVENT_HUB_MEASURES_VAR to eventHubUri)) - } - - return envVars.toMap() - } - private fun getImageName(registry: String, repository: String, version: String? = null): String { var imageRef = if (registry.isNotEmpty()) "$registry/" else "" imageRef += repository @@ -319,7 +293,7 @@ internal fun getMinimalCommonEnvVars( ): Map { val twinCacheEnvVars: MutableMap = mutableMapOf() - val twinCacheInfo = csmPlatformProperties.twincache + val twinCacheInfo = csmPlatformProperties.databases.resources twinCacheEnvVars.putAll( mapOf( TWIN_CACHE_HOST to (twinCacheInfo.host), diff --git a/run/src/main/kotlin/com/cosmotech/run/config/RabbitMqConfig.kt b/run/src/main/kotlin/com/cosmotech/run/config/RabbitMqConfig.kt deleted file mode 100644 index 9eb7e0f14..000000000 --- a/run/src/main/kotlin/com/cosmotech/run/config/RabbitMqConfig.kt +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) Cosmo Tech. -// Licensed under the MIT license. -package com.cosmotech.run.config - -import org.springframework.amqp.core.Binding -import org.springframework.amqp.core.BindingBuilder -import org.springframework.amqp.core.Queue -import org.springframework.amqp.core.TopicExchange -import org.springframework.amqp.rabbit.annotation.RabbitListenerConfigurer -import org.springframework.amqp.rabbit.config.SimpleRabbitListenerContainerFactory -import org.springframework.amqp.rabbit.connection.ConnectionFactory -import org.springframework.amqp.rabbit.core.RabbitAdmin -import org.springframework.amqp.rabbit.core.RabbitTemplate -import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistrar -import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression -import org.springframework.context.annotation.Bean -import org.springframework.context.annotation.Configuration -import org.springframework.messaging.handler.annotation.support.DefaultMessageHandlerMethodFactory - -@Configuration -@ConditionalOnExpression( - "'\${csm.platform.internalResultServices.enabled}' == 'true' " + - "and '\${csm.platform.internalResultServices.eventBus.enabled}' == 'true'") -class RabbitMqConfig( - val rabbitMqConfigModel: RabbitMqConfigModel, - val connectionFactory: ConnectionFactory -) : RabbitListenerConfigurer { - - @Bean - fun defaultQueue(): Queue { - return Queue(rabbitMqConfigModel.queue, true) - } - - @Bean - fun defaultExchange(): TopicExchange { - return TopicExchange(rabbitMqConfigModel.exchange) - } - - @Bean - fun defaultBinding(): Binding { - return BindingBuilder.bind(defaultQueue()) - .to(defaultExchange()) - .with(rabbitMqConfigModel.routingKey) - } - - @Bean fun rabbitListenerEndpointRegistry() = RabbitListenerEndpointRegistry() - - @Bean fun messageHandlerMethodFactory() = DefaultMessageHandlerMethodFactory() - - @Bean fun rabbitTemplate() = RabbitTemplate(connectionFactory) - - @Bean fun rabbitAdmin() = RabbitAdmin(connectionFactory) - - override fun configureRabbitListeners(registrar: RabbitListenerEndpointRegistrar) { - val factory = SimpleRabbitListenerContainerFactory() - factory.setPrefetchCount(1) - factory.setConsecutiveActiveTrigger(1) - factory.setConsecutiveIdleTrigger(1) - factory.setConnectionFactory(connectionFactory) - registrar.setContainerFactory(factory) - registrar.setEndpointRegistry(rabbitListenerEndpointRegistry()) - registrar.messageHandlerMethodFactory = messageHandlerMethodFactory() - } -} diff --git a/run/src/main/kotlin/com/cosmotech/run/config/RabbitMqConfigModel.kt b/run/src/main/kotlin/com/cosmotech/run/config/RabbitMqConfigModel.kt deleted file mode 100644 index 5817bf1dd..000000000 --- a/run/src/main/kotlin/com/cosmotech/run/config/RabbitMqConfigModel.kt +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) Cosmo Tech. -// Licensed under the MIT license. -package com.cosmotech.run.config - -import org.springframework.beans.factory.annotation.Value -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression -import org.springframework.context.annotation.Configuration - -@Configuration -@ConditionalOnExpression( - "'\${csm.platform.internalResultServices.enabled}' == 'true' " + - "and '\${csm.platform.internalResultServices.eventBus.enabled}' == 'true'") -class RabbitMqConfigModel { - - @Value("\${csm.platform.internalResultServices.eventBus.default-exchange}") - lateinit var exchange: String - - @Value("\${csm.platform.internalResultServices.eventBus.default-queue}") - lateinit var queue: String - - @Value("\${csm.platform.internalResultServices.eventBus.default-routing-key}") - lateinit var routingKey: String -} diff --git a/run/src/main/kotlin/com/cosmotech/run/config/RunStorageConfig.kt b/run/src/main/kotlin/com/cosmotech/run/config/RunStorageConfig.kt deleted file mode 100644 index ee6591b06..000000000 --- a/run/src/main/kotlin/com/cosmotech/run/config/RunStorageConfig.kt +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) Cosmo Tech. -// Licensed under the MIT license. -package com.cosmotech.run.config - -import javax.sql.DataSource -import org.springframework.beans.factory.annotation.Qualifier -import org.springframework.beans.factory.annotation.Value -import org.springframework.context.annotation.Bean -import org.springframework.context.annotation.Configuration -import org.springframework.jdbc.core.JdbcTemplate -import org.springframework.jdbc.datasource.DriverManagerDataSource - -@Configuration -class RunStorageConfig { - - @Value("\${csm.platform.internalResultServices.storage.admin.username}") - private lateinit var adminStorageUsername: String - - @Value("\${csm.platform.internalResultServices.storage.admin.password}") - private lateinit var adminStoragePassword: String - - @Value("\${csm.platform.internalResultServices.storage.host}") private lateinit var host: String - @Value("\${csm.platform.internalResultServices.storage.port}") private lateinit var port: String - - private val jdbcdriverClass = "org.postgresql.Driver" - - @Bean - fun adminRunStorageDatasource(): DriverManagerDataSource { - val dataSource = - DriverManagerDataSource( - "jdbc:postgresql://$host:$port/postgres", adminStorageUsername, adminStoragePassword) - dataSource.setDriverClassName(jdbcdriverClass) - return dataSource - } - - @Bean - fun adminRunStorageTemplate( - @Qualifier("adminRunStorageDatasource") dataSource: DataSource - ): JdbcTemplate { - return JdbcTemplate(dataSource) - } -} - -fun JdbcTemplate.existDB(name: String): Boolean { - return this.queryForList("SELECT * FROM pg_catalog.pg_database WHERE datname='$name'").size == 1 -} - -fun JdbcTemplate.existTable(name: String): Boolean { - return this.queryForList( - "SELECT * FROM information_schema.tables " + "WHERE table_name ilike '${name}'") - .size >= 1 -} - -fun String.toDataTableName(isProbeData: Boolean): String = - (if (isProbeData) "P_$this" else "CD_$this").lowercase() - -fun JdbcTemplate.createDB(name: String, comment: String? = null): String { - this.execute("CREATE DATABASE \"$name\"") - if (comment != null) this.execute("COMMENT ON DATABASE \"$name\" IS '$comment'") - return name -} - -fun JdbcTemplate.dropDB(name: String) { - if (this.existDB(name)) this.execute("DROP DATABASE \"$name\"") -} diff --git a/run/src/main/kotlin/com/cosmotech/run/service/RunServiceImpl.kt b/run/src/main/kotlin/com/cosmotech/run/service/RunServiceImpl.kt index 857ca4085..91a114706 100644 --- a/run/src/main/kotlin/com/cosmotech/run/service/RunServiceImpl.kt +++ b/run/src/main/kotlin/com/cosmotech/run/service/RunServiceImpl.kt @@ -3,6 +3,10 @@ package com.cosmotech.run.service import com.cosmotech.common.CsmPhoenixService +import com.cosmotech.common.config.createDB +import com.cosmotech.common.config.dropDB +import com.cosmotech.common.config.existTable +import com.cosmotech.common.config.toDataTableName import com.cosmotech.common.events.RunDeleted import com.cosmotech.common.events.RunStart import com.cosmotech.common.events.RunStop @@ -17,10 +21,6 @@ import com.cosmotech.common.utils.constructPageRequest import com.cosmotech.common.utils.getCurrentAccountIdentifier import com.cosmotech.run.RunApiServiceInterface import com.cosmotech.run.RunContainerFactory -import com.cosmotech.run.config.createDB -import com.cosmotech.run.config.dropDB -import com.cosmotech.run.config.existTable -import com.cosmotech.run.config.toDataTableName import com.cosmotech.run.container.StartInfo import com.cosmotech.run.domain.QueryResult import com.cosmotech.run.domain.Run @@ -43,7 +43,6 @@ import com.google.gson.JsonParser import com.google.gson.reflect.TypeToken import java.sql.SQLException import java.time.Instant -import org.apache.commons.lang3.NotImplementedException import org.postgresql.util.PGobject import org.springframework.beans.factory.annotation.Value import org.springframework.context.event.EventListener @@ -99,34 +98,29 @@ class RunServiceImpl( private val runnerApiService: RunnerApiServiceInterface, private val runRepository: RunRepository, private val csmRbac: CsmRbac, - private val adminRunStorageTemplate: JdbcTemplate + private val adminUserJdbcTemplate: JdbcTemplate ) : CsmPhoenixService(), RunApiServiceInterface { - @Value("\${csm.platform.internalResultServices.storage.admin.username}") + @Value("\${csm.platform.databases.data.admin.username}") private lateinit var adminStorageUsername: String - @Value("\${csm.platform.internalResultServices.storage.admin.password}") + @Value("\${csm.platform.databases.data.admin.password}") private lateinit var adminStoragePassword: String - @Value("\${csm.platform.internalResultServices.storage.writer.username}") + @Value("\${csm.platform.databases.data.writer.username}") private lateinit var writerStorageUsername: String - @Value("\${csm.platform.internalResultServices.storage.writer.password}") + @Value("\${csm.platform.databases.data.writer.password}") private lateinit var writerStoragePassword: String - @Value("\${csm.platform.internalResultServices.storage.reader.username}") + @Value("\${csm.platform.databases.data.reader.username}") private lateinit var readerStorageUsername: String - @Value("\${csm.platform.internalResultServices.storage.reader.password}") + @Value("\${csm.platform.databases.data.reader.password}") private lateinit var readerStoragePassword: String - @Value("\${csm.platform.internalResultServices.storage.host}") private lateinit var host: String - @Value("\${csm.platform.internalResultServices.storage.port}") private lateinit var port: String - - private val notImplementedExceptionMessage = - "The API is configured to use the external result data service. " + - "This endpoint is deactivated so, use scenario/scenariorun endpoints instead. " + - "To change that, set the API configuration entry 'csm.platform.use-internal-result-services' to true" + @Value("\${csm.platform.databases.data.host}") private lateinit var host: String + @Value("\${csm.platform.databases.data.port}") private lateinit var port: String override fun listRuns( organizationId: String, @@ -138,7 +132,7 @@ class RunServiceImpl( // This call verify the user read authorization in the Runner runnerApiService.getRunner(organizationId, workspaceId, runnerId) - val defaultPageSize = csmPlatformProperties.twincache.run.defaultPageSize + val defaultPageSize = csmPlatformProperties.databases.resources.run.defaultPageSize val pageable = constructPageRequest(page, size, defaultPageSize) ?: PageRequest.of(0, defaultPageSize) @@ -156,7 +150,7 @@ class RunServiceImpl( // This call verify the user read authorization in the Runner runnerApiService.getRunner(organizationId, workspaceId, runnerId) - val defaultPageSize = csmPlatformProperties.twincache.run.defaultPageSize + val defaultPageSize = csmPlatformProperties.databases.resources.run.defaultPageSize var pageRequest: Pageable = PageRequest.ofSize(defaultPageSize) val runs = mutableListOf() @@ -318,7 +312,6 @@ class RunServiceImpl( sendRunDataRequest: SendRunDataRequest ): RunData { - checkInternalResultDataServiceConfiguration() val run = getRun(organizationId, workspaceId, runnerId, runId) run.hasPermission(PERMISSION_WRITE) @@ -335,7 +328,6 @@ class RunServiceImpl( runDataQuery: RunDataQuery ): QueryResult { - checkInternalResultDataServiceConfiguration() getRun(organizationId, workspaceId, runnerId, runId) val runtimeDS = @@ -452,7 +444,7 @@ class RunServiceImpl( workflowService.stopWorkflow(run) - val defaultPageSize = csmPlatformProperties.twincache.run.defaultPageSize + val defaultPageSize = csmPlatformProperties.databases.resources.run.defaultPageSize val pageRequest = PageRequest.ofSize(defaultPageSize) val runs = runRepository @@ -470,8 +462,7 @@ class RunServiceImpl( RunDeleted(this, run.organizationId, run.workspaceId, run.runnerId, run.id!!, lastRun) this.eventPublisher.publishEvent(runDeleted) - if (csmPlatformProperties.internalResultServices?.enabled == true) - adminRunStorageTemplate.dropDB(run.id) + adminUserJdbcTemplate.dropDB(run.id) runRepository.delete(run) } catch (exception: IllegalStateException) { @@ -512,46 +503,44 @@ class RunServiceImpl( val runner = runStartRequest.runnerData as Runner val runId = idGenerator.generate("run", prependPrefix = "run-") - if (csmPlatformProperties.internalResultServices?.enabled == true) { - val dbComment = - "organizationId=${runner.organizationId}, workspaceId=${runner.workspaceId}, runnerId=${runner.id}" - adminRunStorageTemplate.createDB(runId, dbComment) - - val runtimeDS = - DriverManagerDataSource( - "jdbc:postgresql://$host:$port/$runId", adminStorageUsername, adminStoragePassword) - runtimeDS.setDriverClassName(POSTGRESQL_DRIVER_CLASS_NAME) - val runDBJdbcTemplate = JdbcTemplate(runtimeDS) - val connection = runDBJdbcTemplate.dataSource!!.connection - try { - connection.autoCommit = false - connection - .prepareStatement("GRANT CONNECT ON DATABASE \"$runId\" TO $readerStorageUsername") - .executeUpdate() - connection - .prepareStatement("GRANT CONNECT ON DATABASE \"$runId\" TO $writerStorageUsername") - .executeUpdate() - - connection - .prepareStatement("GRANT CREATE ON SCHEMA public to $writerStorageUsername") - .executeUpdate() - - connection - .prepareStatement("GRANT USAGE ON SCHEMA public to $writerStorageUsername") - .executeUpdate() + val dbComment = + "organizationId=${runner.organizationId}, workspaceId=${runner.workspaceId}, runnerId=${runner.id}" + adminUserJdbcTemplate.createDB(runId, dbComment) - connection - .prepareStatement( - "ALTER DEFAULT PRIVILEGES FOR USER $writerStorageUsername IN SCHEMA public " + - "GRANT SELECT ON TABLES TO $readerStorageUsername;") - .executeUpdate() - connection.commit() - } catch (e: SQLException) { - connection.rollback() - throw e - } finally { - connection.close() - } + val runtimeDS = + DriverManagerDataSource( + "jdbc:postgresql://$host:$port/$runId", adminStorageUsername, adminStoragePassword) + runtimeDS.setDriverClassName(POSTGRESQL_DRIVER_CLASS_NAME) + val runDBJdbcTemplate = JdbcTemplate(runtimeDS) + val connection = runDBJdbcTemplate.dataSource!!.connection + try { + connection.autoCommit = false + connection + .prepareStatement("GRANT CONNECT ON DATABASE \"$runId\" TO $readerStorageUsername") + .executeUpdate() + connection + .prepareStatement("GRANT CONNECT ON DATABASE \"$runId\" TO $writerStorageUsername") + .executeUpdate() + + connection + .prepareStatement("GRANT CREATE ON SCHEMA public to $writerStorageUsername") + .executeUpdate() + + connection + .prepareStatement("GRANT USAGE ON SCHEMA public to $writerStorageUsername") + .executeUpdate() + + connection + .prepareStatement( + "ALTER DEFAULT PRIVILEGES FOR USER $writerStorageUsername IN SCHEMA public " + + "GRANT SELECT ON TABLES TO $readerStorageUsername;") + .executeUpdate() + connection.commit() + } catch (e: SQLException) { + connection.rollback() + throw e + } finally { + connection.close() } val startInfo = @@ -653,12 +642,6 @@ class RunServiceImpl( csmRbac.verify(runner.getRbac(), permission, getRunnerRolesDefinition()) } - internal fun checkInternalResultDataServiceConfiguration() { - if (csmPlatformProperties.internalResultServices?.enabled != true) { - throw NotImplementedException(notImplementedExceptionMessage) - } - } - @EventListener(RunnerDeleted::class) fun onRunnerDeleted(runnerDeleted: RunnerDeleted) { listAllRuns(runnerDeleted.organizationId, runnerDeleted.workspaceId, runnerDeleted.runnerId) diff --git a/run/src/main/kotlin/com/cosmotech/run/service/amqp/AddQueueOnRunStartListener.kt b/run/src/main/kotlin/com/cosmotech/run/service/amqp/AddQueueOnRunStartListener.kt deleted file mode 100644 index d8aa52af1..000000000 --- a/run/src/main/kotlin/com/cosmotech/run/service/amqp/AddQueueOnRunStartListener.kt +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) Cosmo Tech. -// Licensed under the MIT license. -package com.cosmotech.run.service.amqp - -import com.cosmotech.common.events.RunStart -import com.cosmotech.run.config.RabbitMqConfigModel -import com.cosmotech.runner.domain.Runner -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression -import org.springframework.context.ApplicationListener -import org.springframework.stereotype.Component - -@Component -@ConditionalOnExpression( - "'\${csm.platform.internalResultServices.enabled}' == 'true' " + - "and '\${csm.platform.internalResultServices.eventBus.enabled}' == 'true'") -class AddQueueOnRunStartListener( - private val rabbitMqConfigModel: RabbitMqConfigModel, - private val amqpClientServiceImpl: AmqpClientServiceImpl, -) : ApplicationListener { - - override fun onApplicationEvent(event: RunStart) { - val exchange = rabbitMqConfigModel.exchange - amqpClientServiceImpl.addNewQueue(exchange, (event.runnerData as Runner).workspaceId) - } -} diff --git a/run/src/main/kotlin/com/cosmotech/run/service/amqp/AmqpClientServiceImpl.kt b/run/src/main/kotlin/com/cosmotech/run/service/amqp/AmqpClientServiceImpl.kt deleted file mode 100644 index 96b853504..000000000 --- a/run/src/main/kotlin/com/cosmotech/run/service/amqp/AmqpClientServiceImpl.kt +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (c) Cosmo Tech. -// Licensed under the MIT license. -package com.cosmotech.run.service.amqp - -import com.cosmotech.run.service.RunServiceImpl -import com.google.gson.Gson -import com.google.gson.reflect.TypeToken -import java.io.BufferedReader -import java.io.ByteArrayInputStream -import java.io.InputStreamReader -import java.util.zip.GZIPInputStream -import org.slf4j.LoggerFactory -import org.springframework.amqp.core.Binding -import org.springframework.amqp.core.Message -import org.springframework.amqp.core.Queue -import org.springframework.amqp.rabbit.annotation.RabbitListener -import org.springframework.amqp.rabbit.core.RabbitAdmin -import org.springframework.amqp.rabbit.listener.AbstractMessageListenerContainer -import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression -import org.springframework.stereotype.Service - -@Suppress("ConstructorParameterNaming") -data class ProbeMessage( - val simulation: Map, - val probe: Map, - val factsCommon: Map, - val facts: List> -) - -@Service -@ConditionalOnExpression( - "'\${csm.platform.internalResultServices.enabled}' == 'true' " + - "and '\${csm.platform.internalResultServices.eventBus.enabled}' == 'true'") -class AmqpClientServiceImpl( - private val rabbitAdmin: RabbitAdmin, - private val rabbitListenerEndpointRegistry: RabbitListenerEndpointRegistry, - private val runServiceImpl: RunServiceImpl -) : AmqpClientServiceInterface { - - private val logger = LoggerFactory.getLogger(AmqpClientServiceImpl::class.java) - - @RabbitListener( - id = "\${csm.platform.internalResultServices.eventBus.default-exchange}", - queues = ["\${csm.platform.internalResultServices.eventBus.default-queue}"], - concurrency = "5") - fun receive(message: Message) { - logger.debug("Message received...") - val gson = Gson() - val mapAdapter = gson.getAdapter(object : TypeToken() {}) - val messageRead = - mapAdapter.fromJson( - ByteArrayInputStream(message.body).use { bais -> - GZIPInputStream(bais).use { gis -> - InputStreamReader(gis, Charsets.UTF_8).use { isr -> - BufferedReader(isr).use { bfr -> bfr.readText() } - } - } - }) - val data = mutableListOf>() - messageRead.facts.forEach { - val row = (it + messageRead.factsCommon).toMutableMap() - row["probe_name"] = messageRead.probe["name"].toString() - row["probe_run"] = messageRead.probe["run"]!! - data.add(row) - } - val runId = messageRead.simulation["run"].toString() - val tableName = messageRead.probe["type"].toString() - runServiceImpl.sendDataToStorage(runId, tableName, data, true) - } - - override fun addNewQueue(exchangeName: String, queueName: String) { - logger.debug("Adding Queue $exchangeName/$queueName to broker") - // TODO pass durable to true (in case of broker restart) => adapt Proton client - val queue = Queue(queueName, false, false, false) - val binding = Binding(queueName, Binding.DestinationType.QUEUE, exchangeName, queueName, null) - rabbitAdmin.declareQueue(queue) - rabbitAdmin.declareBinding(binding) - logger.debug("Queue $exchangeName/$queueName added to broker") - this.addQueueToListener(exchangeName, queueName) - } - - private fun addQueueToListener(exchangeName: String, queueName: String) { - logger.debug("Trying to add Queue $exchangeName/$queueName to listener") - if (!checkQueueExistOnListener(exchangeName, queueName)) { - logger.debug("Queue $exchangeName/$queueName is not handled by a listener...") - val listenerContainer = this.getMessageListenerContainerById(exchangeName) - if (listenerContainer != null) { - listenerContainer.addQueueNames(queueName) - logger.debug("Queue $exchangeName/$queueName added to listener") - } - } - } - - private fun checkQueueExistOnListener(exchangeName: String, queueName: String): Boolean { - val listenerContainer = this.getMessageListenerContainerById(exchangeName) - if (listenerContainer != null) { - val existingQueueNames = listenerContainer.queueNames - return existingQueueNames.contains(queueName) - } - logger.debug("No listener defined for couple exchange/queueName $exchangeName/$queueName") - return false - } - - private fun getMessageListenerContainerById(exchangeName: String) = - this.rabbitListenerEndpointRegistry.getListenerContainer(exchangeName) - as AbstractMessageListenerContainer? - - override fun removeQueue(exchangeName: String, queueName: String) { - logger.debug("Trying to remove Queue $exchangeName/$queueName to listener") - if (checkQueueExistOnListener(exchangeName, queueName)) { - logger.debug("Queue $exchangeName/$queueName is handled by a listener") - getMessageListenerContainerById(exchangeName)?.removeQueueNames(queueName) - } - this.rabbitAdmin.deleteQueue(queueName) - } -} diff --git a/run/src/main/kotlin/com/cosmotech/run/service/amqp/AmqpClientServiceInterface.kt b/run/src/main/kotlin/com/cosmotech/run/service/amqp/AmqpClientServiceInterface.kt deleted file mode 100644 index f141e12c5..000000000 --- a/run/src/main/kotlin/com/cosmotech/run/service/amqp/AmqpClientServiceInterface.kt +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) Cosmo Tech. -// Licensed under the MIT license. -package com.cosmotech.run.service.amqp - -interface AmqpClientServiceInterface { - - fun addNewQueue(exchangeName: String, queueName: String) - - fun removeQueue(exchangeName: String, queueName: String) -} diff --git a/run/src/main/kotlin/com/cosmotech/run/service/amqp/RemoveQueueOnWorkspaceDeletedListener.kt b/run/src/main/kotlin/com/cosmotech/run/service/amqp/RemoveQueueOnWorkspaceDeletedListener.kt deleted file mode 100644 index 288732929..000000000 --- a/run/src/main/kotlin/com/cosmotech/run/service/amqp/RemoveQueueOnWorkspaceDeletedListener.kt +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) Cosmo Tech. -// Licensed under the MIT license. -package com.cosmotech.run.service.amqp - -import com.cosmotech.common.events.WorkspaceDeleted -import com.cosmotech.run.config.RabbitMqConfigModel -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression -import org.springframework.context.ApplicationListener -import org.springframework.stereotype.Component - -@Component -@ConditionalOnExpression( - "'\${csm.platform.internalResultServices.enabled}' == 'true' " + - "and '\${csm.platform.internalResultServices.eventBus.enabled}' == 'true'") -class RemoveQueueOnWorkspaceDeletedListener( - private val rabbitMqConfigModel: RabbitMqConfigModel, - private val amqpClientServiceImpl: AmqpClientServiceImpl -) : ApplicationListener { - - override fun onApplicationEvent(event: WorkspaceDeleted) { - val exchange = rabbitMqConfigModel.exchange - amqpClientServiceImpl.removeQueue(exchange, event.workspaceId) - } -} diff --git a/run/src/test/kotlin/com/cosmotech/run/ContainerFactoryTests.kt b/run/src/test/kotlin/com/cosmotech/run/ContainerFactoryTests.kt index a0e5fd4da..722ce6f7f 100644 --- a/run/src/test/kotlin/com/cosmotech/run/ContainerFactoryTests.kt +++ b/run/src/test/kotlin/com/cosmotech/run/ContainerFactoryTests.kt @@ -87,8 +87,8 @@ class ContainerFactoryTests { tenantId = "my_tenant_id", clientId = "my_client_id", clientSecret = "my_client_secret")) - every { csmPlatformProperties.twincache } returns - CsmPlatformProperties.CsmTwinCacheProperties( + every { csmPlatformProperties.databases.resources } returns + CsmPlatformProperties.CsmDatabasesProperties.CsmResourcesProperties( host = "this_is_a_host", port = "6973", password = "this_is_a_password", @@ -101,32 +101,19 @@ class ContainerFactoryTests { password = "password", username = "username") - every { csmPlatformProperties.internalResultServices } returns - CsmPlatformProperties.CsmServiceResult( - enabled = true, - eventBus = - CsmPlatformProperties.CsmServiceResult.CsmEventBus( - host = "localhost", - port = 6379, - listener = - CsmPlatformProperties.CsmServiceResult.CsmEventBus.CsmEventBusUser( - password = "password", username = "username"), - sender = - CsmPlatformProperties.CsmServiceResult.CsmEventBus.CsmEventBusUser( - password = "password", username = "username")), - storage = - CsmPlatformProperties.CsmServiceResult.CsmStorage( - host = "localhost", - port = 5432, - admin = - CsmPlatformProperties.CsmServiceResult.CsmStorage.CsmStorageUser( - password = "password", username = "username"), - reader = - CsmPlatformProperties.CsmServiceResult.CsmStorage.CsmStorageUser( - username = "username", password = "password"), - writer = - CsmPlatformProperties.CsmServiceResult.CsmStorage.CsmStorageUser( - username = "username", password = "password"))) + every { csmPlatformProperties.databases.data } returns + CsmPlatformProperties.CsmDatabasesProperties.CsmDataIOProperties( + host = "localhost", + port = 5432, + admin = + CsmPlatformProperties.CsmDatabasesProperties.CsmDataIOProperties.CsmStorageUser( + password = "password", username = "username"), + reader = + CsmPlatformProperties.CsmDatabasesProperties.CsmDataIOProperties.CsmStorageUser( + username = "username", password = "password"), + writer = + CsmPlatformProperties.CsmDatabasesProperties.CsmDataIOProperties.CsmStorageUser( + username = "username", password = "password")) factory = RunContainerFactory( @@ -173,12 +160,6 @@ class ContainerFactoryTests { runId: String ): RunContainer { - val eventHubUri = - "amqp://" + - "${csmPlatformProperties.internalResultServices?.eventBus?.host}:" + - "${csmPlatformProperties.internalResultServices?.eventBus?.port}/" + - workspace.id - return RunContainer( name = CONTAINER_CSM_ORC, image = "twinengines.azurecr.io/" + solution.repository + ":" + solution.version, @@ -190,10 +171,10 @@ class ContainerFactoryTests { "CSM_PARAMETERS_ABSOLUTE_PATH" to "/mnt/scenariorun-parameters", "CSM_OUTPUT_ABSOLUTE_PATH" to "/pkg/share/Simulation/Output", "CSM_TEMP_ABSOLUTE_PATH" to "/usr/tmp", - "TWIN_CACHE_HOST" to csmPlatformProperties.twincache.host, - "TWIN_CACHE_PORT" to csmPlatformProperties.twincache.port, - "TWIN_CACHE_PASSWORD" to csmPlatformProperties.twincache.password, - "TWIN_CACHE_USERNAME" to csmPlatformProperties.twincache.username, + "TWIN_CACHE_HOST" to csmPlatformProperties.databases.resources.host, + "TWIN_CACHE_PORT" to csmPlatformProperties.databases.resources.port, + "TWIN_CACHE_PASSWORD" to csmPlatformProperties.databases.resources.password, + "TWIN_CACHE_USERNAME" to csmPlatformProperties.databases.resources.username, "IDP_CLIENT_ID" to csmPlatformProperties.identityProvider.identity.clientId, "IDP_CLIENT_SECRET" to csmPlatformProperties.identityProvider.identity.clientSecret, "IDP_BASE_URL" to csmPlatformProperties.identityProvider.serverBaseUrl, @@ -203,11 +184,7 @@ class ContainerFactoryTests { "CSM_WORKSPACE_ID" to workspace.id, "CSM_RUNNER_ID" to runner.id, "CSM_RUN_ID" to runId, - "CSM_RUN_TEMPLATE_ID" to CSM_RUN_TEMPLATE_ID, - "CSM_PROBES_MEASURES_TOPIC" to eventHubUri, - "CSM_AMQPCONSUMER_USER" to "username", - "CSM_AMQPCONSUMER_PASSWORD" to "password", - ), + "CSM_RUN_TEMPLATE_ID" to CSM_RUN_TEMPLATE_ID), entrypoint = "entrypoint.py", nodeLabel = runTemplate.computeSize!!.removeSuffix("pool"), runSizing = diff --git a/runner/src/integrationTest/kotlin/com/cosmotech/runner/service/RunnerServiceIntegrationTest.kt b/runner/src/integrationTest/kotlin/com/cosmotech/runner/service/RunnerServiceIntegrationTest.kt index 84e377b7c..77896c860 100644 --- a/runner/src/integrationTest/kotlin/com/cosmotech/runner/service/RunnerServiceIntegrationTest.kt +++ b/runner/src/integrationTest/kotlin/com/cosmotech/runner/service/RunnerServiceIntegrationTest.kt @@ -337,7 +337,7 @@ class RunnerServiceIntegrationTest : CsmTestBase() { // We create more runner than there can be on one page of default size to assert // deleteAllRunners still works with high quantities of runners - repeat(csmPlatformProperties.twincache.runner.defaultPageSize + 1) { + repeat(csmPlatformProperties.databases.resources.runner.defaultPageSize + 1) { runnerApiService.createRunner( organizationSaved.id, workspaceSaved.id, makeRunnerCreateRequest()) } @@ -346,7 +346,7 @@ class RunnerServiceIntegrationTest : CsmTestBase() { @Test fun `test find All Runners with different pagination params`() { val numberOfRunners = 20 - val defaultPageSize = csmPlatformProperties.twincache.runner.defaultPageSize + val defaultPageSize = csmPlatformProperties.databases.resources.runner.defaultPageSize val expectedSize = 15 IntRange(1, numberOfRunners - 1).forEach { val runner = diff --git a/runner/src/integrationTest/resources/application-runner-test.yml b/runner/src/integrationTest/resources/application-runner-test.yml index a83a1991c..0a8bae36c 100644 --- a/runner/src/integrationTest/resources/application-runner-test.yml +++ b/runner/src/integrationTest/resources/application-runner-test.yml @@ -1,4 +1,12 @@ spring: + security: + oauth2: + resource-server: + jwt: + issuer-uri: "http://localhost:9000/realms/fakeTenant" + jwk-set-uri: "http://localhost:9000/realms/fakeTenant/protocol/openid-connect/certs" + audiences: + - "cosmotech" data: redis: host: "localhost" @@ -30,7 +38,6 @@ management: csm: platform: identityProvider: - code: on_premise_one authorizationUrl: "http://fake_url:8080/authorize" tokenUrl: "http://fake_url:8080/token" containerScopes: @@ -92,17 +99,30 @@ csm: - ReadWriteOnce requests: storage: 1Gi - twincache: - host: "localhost" - port: "6379" - username: "default" - password: "my-wonderful-password" - useGraphModule: true - tls: - enabled: false - bundle: "" - runner: - default-page-size: 20 + databases: + resources: + host: "localhost" + port: "6379" + username: "default" + # Leave it as blank as there's no auth with test container + password: + tls: + enabled: false + bundle: "" + runner: + default-page-size: 20 + data: + admin: + password: "password" + username: cosmotech_api_admin + host: "localhost" + port: 5432 + reader: + password: "password" + username: cosmotech_api_reader + writer: + password: "password" + username: cosmotech_api_writer rbac: enabled: true s3: @@ -111,14 +131,3 @@ csm: accessKeyId: "s3_username" secretAccessKey: "s3_password" region: "dummy" - internalResultServices: - enabled: true - eventBus: - enabled: false - host: "http://localhost:1234" - listener: - username: "fakeUser" - password: "fakePwd" - sender: - username: "fakeUser2" - password: "fakePwd2" diff --git a/runner/src/main/kotlin/com/cosmotech/runner/service/RunnerApiServiceImpl.kt b/runner/src/main/kotlin/com/cosmotech/runner/service/RunnerApiServiceImpl.kt index d2115083b..63830b148 100644 --- a/runner/src/main/kotlin/com/cosmotech/runner/service/RunnerApiServiceImpl.kt +++ b/runner/src/main/kotlin/com/cosmotech/runner/service/RunnerApiServiceImpl.kt @@ -99,7 +99,7 @@ internal class RunnerApiServiceImpl( ): List { val runnerService = getRunnerService().inOrganization(organizationId).inWorkspace(workspaceId) - val defaultPageSize = csmPlatformProperties.twincache.runner.defaultPageSize + val defaultPageSize = csmPlatformProperties.databases.resources.runner.defaultPageSize val pageRequest = constructPageRequest(page, size, defaultPageSize) ?: PageRequest.of(0, defaultPageSize) return runnerService.listInstances(pageRequest) diff --git a/runner/src/main/kotlin/com/cosmotech/runner/service/RunnerService.kt b/runner/src/main/kotlin/com/cosmotech/runner/service/RunnerService.kt index f69a3422f..6985b0367 100644 --- a/runner/src/main/kotlin/com/cosmotech/runner/service/RunnerService.kt +++ b/runner/src/main/kotlin/com/cosmotech/runner/service/RunnerService.kt @@ -151,7 +151,7 @@ class RunnerService( workspaceId: String, parentId: String ): List { - val defaultPageSize = csmPlatformProperties.twincache.runner.defaultPageSize + val defaultPageSize = csmPlatformProperties.databases.resources.runner.defaultPageSize var pageRequest: Pageable = PageRequest.ofSize(defaultPageSize) val runners = mutableListOf() diff --git a/solution/src/integrationTest/kotlin/com/cosmotech/solution/service/SolutionServiceIntegrationTest.kt b/solution/src/integrationTest/kotlin/com/cosmotech/solution/service/SolutionServiceIntegrationTest.kt index 91718ccee..4e7fef720 100644 --- a/solution/src/integrationTest/kotlin/com/cosmotech/solution/service/SolutionServiceIntegrationTest.kt +++ b/solution/src/integrationTest/kotlin/com/cosmotech/solution/service/SolutionServiceIntegrationTest.kt @@ -707,7 +707,7 @@ class SolutionServiceIntegrationTest : CsmTestBase() { @Test fun `test find All Solutions with different pagination params`() { val numberOfSolutions = 20 - val defaultPageSize = csmPlatformProperties.twincache.solution.defaultPageSize + val defaultPageSize = csmPlatformProperties.databases.resources.solution.defaultPageSize val expectedSize = 15 IntRange(1, numberOfSolutions - 1).forEach { solutionApiService.createSolution( diff --git a/solution/src/integrationTest/resources/application-solution-test.yml b/solution/src/integrationTest/resources/application-solution-test.yml index 6c2d787d8..96a1f1e72 100644 --- a/solution/src/integrationTest/resources/application-solution-test.yml +++ b/solution/src/integrationTest/resources/application-solution-test.yml @@ -3,6 +3,14 @@ management: enabled-by-default: false spring: + security: + oauth2: + resource-server: + jwt: + issuer-uri: "http://localhost:9000/realms/fakeTenant" + jwk-set-uri: "http://localhost:9000/realms/fakeTenant/protocol/openid-connect/certs" + audiences: + - "cosmotech" data: redis: host: "localhost" @@ -30,7 +38,6 @@ spring: csm: platform: identityProvider: - code: on_premise_one authorizationUrl: "http://fake_url:8080/authorize" tokenUrl: "http://fake_url:8080/token" containerScopes: @@ -83,17 +90,30 @@ csm: - ReadWriteOnce requests: storage: 1Gi - twincache: - host: "localhost" - port: "6379" - username: "default" - # Leave it as blank as there's no auth with test container - password: - tls: - enabled: false - bundle: "" - solution: - default-page-size: 20 + databases: + resources: + host: "localhost" + port: "6379" + username: "default" + # Leave it as blank as there's no auth with test container + password: + solution: + default-page-size: 20 + tls: + enabled: false + bundle: "" + data: + admin: + password: "password" + username: cosmotech_api_admin + host: "localhost" + port: 5432 + reader: + password: "password" + username: cosmotech_api_reader + writer: + password: "password" + username: cosmotech_api_writer rbac: enabled: true s3: @@ -102,14 +122,3 @@ csm: accessKeyId: "s3_username" secretAccessKey: "s3_password" region: "dummy" - internalResultServices: - enabled: true - eventBus: - enabled: false - host: "http://localhost:1234" - listener: - username: "fakeUser" - password: "fakePwd" - sender: - username: "fakeUser2" - password: "fakePwd2" diff --git a/solution/src/main/kotlin/com/cosmotech/solution/service/SolutionServiceImpl.kt b/solution/src/main/kotlin/com/cosmotech/solution/service/SolutionServiceImpl.kt index 193d2792d..409251e71 100644 --- a/solution/src/main/kotlin/com/cosmotech/solution/service/SolutionServiceImpl.kt +++ b/solution/src/main/kotlin/com/cosmotech/solution/service/SolutionServiceImpl.kt @@ -60,7 +60,7 @@ class SolutionServiceImpl( override fun listSolutions(organizationId: String, page: Int?, size: Int?): List { organizationApiService.getVerifiedOrganization(organizationId) - val defaultPageSize = csmPlatformProperties.twincache.solution.defaultPageSize + val defaultPageSize = csmPlatformProperties.databases.resources.solution.defaultPageSize val pageable = constructPageRequest(page, size, defaultPageSize) val isAdmin = csmAdmin.verifyCurrentRolesAdmin() val result: MutableList @@ -308,7 +308,7 @@ class SolutionServiceImpl( @Async("csm-in-process-event-executor") fun onOrganizationUnregistered(organizationUnregistered: OrganizationUnregistered) { val pageable: Pageable = - Pageable.ofSize(csmPlatformProperties.twincache.solution.defaultPageSize) + Pageable.ofSize(csmPlatformProperties.databases.resources.solution.defaultPageSize) val solutions = solutionRepository .findByOrganizationId(organizationUnregistered.organizationId, pageable) diff --git a/workspace/src/integrationTest/kotlin/com/cosmotech/workspace/service/WorkspaceServiceIntegrationTest.kt b/workspace/src/integrationTest/kotlin/com/cosmotech/workspace/service/WorkspaceServiceIntegrationTest.kt index 93b0fe503..e7cc1367e 100644 --- a/workspace/src/integrationTest/kotlin/com/cosmotech/workspace/service/WorkspaceServiceIntegrationTest.kt +++ b/workspace/src/integrationTest/kotlin/com/cosmotech/workspace/service/WorkspaceServiceIntegrationTest.kt @@ -287,7 +287,7 @@ class WorkspaceServiceIntegrationTest : CsmTestBase() { fun `test find All Workspaces with different pagination params`() { val workspaceNumber = 20 - val defaultPageSize = csmPlatformProperties.twincache.workspace.defaultPageSize + val defaultPageSize = csmPlatformProperties.databases.resources.workspace.defaultPageSize val expectedSize = 15 IntRange(1, workspaceNumber - 1).forEach { val workspace = makeWorkspaceCreateRequest(solutionSaved.id, "w-workspace-$it") diff --git a/workspace/src/integrationTest/resources/application-workspace-test.yml b/workspace/src/integrationTest/resources/application-workspace-test.yml index bdd39ec09..c045b0788 100644 --- a/workspace/src/integrationTest/resources/application-workspace-test.yml +++ b/workspace/src/integrationTest/resources/application-workspace-test.yml @@ -3,6 +3,14 @@ management: enabled-by-default: false spring: + security: + oauth2: + resource-server: + jwt: + issuer-uri: "http://localhost:9000/realms/fakeTenant" + jwk-set-uri: "http://localhost:9000/realms/fakeTenant/protocol/openid-connect/certs" + audiences: + - "cosmotech" data: redis: host: "localhost" @@ -104,18 +112,30 @@ csm: - ReadWriteOnce requests: storage: 1Gi - twincache: - host: "localhost" - port: "6379" - username: "default" - # Leave it as blank as there's no auth with test container - password: - useGraphModule: true - tls: - enabled: false - bundle: "" - workspace: - default-page-size: 5 + databases: + resources: + host: "localhost" + port: "6379" + username: "default" + # Leave it as blank as there's no auth with test container + password: + workspace: + default-page-size: 5 + tls: + enabled: false + bundle: "" + data: + admin: + password: "password" + username: cosmotech_api_admin + host: "localhost" + port: 5432 + reader: + password: "password" + username: cosmotech_api_reader + writer: + password: "password" + username: cosmotech_api_writer rbac: enabled: true s3: @@ -124,14 +144,3 @@ csm: accessKeyId: "s3_username" secretAccessKey: "s3_password" region: "dummy" - internalResultServices: - enabled: true - eventBus: - enabled: false - host: "http://localhost:1234" - listener: - username: "fakeUser" - password: "fakePwd" - sender: - username: "fakeUser2" - password: "fakePwd2" diff --git a/workspace/src/main/kotlin/com/cosmotech/workspace/service/WorkspaceServiceImpl.kt b/workspace/src/main/kotlin/com/cosmotech/workspace/service/WorkspaceServiceImpl.kt index 9417060c5..7d8c186e8 100644 --- a/workspace/src/main/kotlin/com/cosmotech/workspace/service/WorkspaceServiceImpl.kt +++ b/workspace/src/main/kotlin/com/cosmotech/workspace/service/WorkspaceServiceImpl.kt @@ -76,7 +76,7 @@ internal class WorkspaceServiceImpl( val isAdmin = csmRbac.isAdmin( organization.security.toGenericSecurity(organizationId), getCommonRolesDefinition()) - val defaultPageSize = csmPlatformProperties.twincache.workspace.defaultPageSize + val defaultPageSize = csmPlatformProperties.databases.resources.workspace.defaultPageSize var result: List var pageable = constructPageRequest(page, size, defaultPageSize) @@ -307,14 +307,12 @@ internal class WorkspaceServiceImpl( fun onOrganizationUnregistered(organizationUnregistered: OrganizationUnregistered) { val organizationId = organizationUnregistered.organizationId val pageable: Pageable = - Pageable.ofSize(csmPlatformProperties.twincache.workspace.defaultPageSize) + Pageable.ofSize(csmPlatformProperties.databases.resources.workspace.defaultPageSize) val workspaces = workspaceRepository.findByOrganizationId(organizationId, pageable).toList() workspaces.forEach { deleteAllS3WorkspaceObjects(organizationId, it) workspaceRepository.delete(it) - if (csmPlatformProperties.internalResultServices?.enabled == true) { - this.eventPublisher.publishEvent(WorkspaceDeleted(this, organizationId, it.id)) - } + this.eventPublisher.publishEvent(WorkspaceDeleted(this, organizationId, it.id)) } } From 5495be391b621bcb0debe87a7cb2ce1b71341d16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Reynard?= Date: Mon, 6 Oct 2025 15:37:45 +0200 Subject: [PATCH 5/5] Remove RDS and Postgres admin role in API configuration As we remove the PostgreSQL admin role in API configuration (not needed afterward), we also remove the RDS initiative: - it was not used - it has to be maintained for nothing - outputs will not be part of CoAL / csm-orc libraries --- .../cosmotech/api/home/ControllerTestBase.kt | 2 - .../api/home/run/RunControllerTests.kt | 103 ------ .../resources/application-test.yml | 7 +- api/src/integrationTest/resources/init-db.sql | 1 - .../common/config/CsmPlatformProperties.kt | 9 +- .../common/config/PostgresConfiguration.kt | 13 +- .../com/cosmotech/common/tests/CsmTestBase.kt | 4 - .../resources/application-dataset-test.yml | 5 - .../src/integrationTest/resources/init-db.sql | 1 - doc/.openapi-generator/FILES | 4 - doc/Apis/RunApi.md | 60 ---- doc/Models/QueryResult.md | 9 - doc/Models/RunData.md | 11 - doc/Models/RunDataQuery.md | 9 - doc/Models/SendRunDataRequest.md | 10 - doc/README.md | 6 - openapi/plantuml/schemas.plantuml | 19 -- .../application-organization-test.yml | 3 - .../src/integrationTest/resources/init-db.sql | 1 - .../run/service/RunServiceIntegrationTest.kt | 274 --------------- .../resources/application-run-test.yml | 5 +- run/src/integrationTest/resources/init-db.sql | 1 - .../cosmotech/run/service/RunServiceImpl.kt | 322 +----------------- run/src/main/openapi/run.yaml | 189 ---------- .../cosmotech/run/ContainerFactoryTests.kt | 4 +- .../resources/application-runner-test.yml | 3 - .../src/integrationTest/resources/init-db.sql | 1 - .../resources/application-solution-test.yml | 3 - .../src/integrationTest/resources/init-db.sql | 1 - .../resources/application-workspace-test.yml | 4 - .../src/integrationTest/resources/init-db.sql | 1 - 31 files changed, 8 insertions(+), 1077 deletions(-) delete mode 100644 doc/Models/QueryResult.md delete mode 100644 doc/Models/RunData.md delete mode 100644 doc/Models/RunDataQuery.md delete mode 100644 doc/Models/SendRunDataRequest.md diff --git a/api/src/integrationTest/kotlin/com/cosmotech/api/home/ControllerTestBase.kt b/api/src/integrationTest/kotlin/com/cosmotech/api/home/ControllerTestBase.kt index 533a03c2a..2109f2207 100644 --- a/api/src/integrationTest/kotlin/com/cosmotech/api/home/ControllerTestBase.kt +++ b/api/src/integrationTest/kotlin/com/cosmotech/api/home/ControllerTestBase.kt @@ -152,8 +152,6 @@ abstract class ControllerTestBase : AbstractTestcontainersRedisTestBase() { private fun initPostgresConfiguration(registry: DynamicPropertyRegistry) { registry.add("csm.platform.databases.data.host") { postgres.host } registry.add("csm.platform.databases.data.port") { postgres.getMappedPort(POSTGRESQL_PORT) } - registry.add("csm.platform.databases.data.admin.username") { ADMIN_USER_CREDENTIALS } - registry.add("csm.platform.databases.data.admin.password") { ADMIN_USER_CREDENTIALS } registry.add("csm.platform.databases.data.writer.username") { WRITER_USER_CREDENTIALS } registry.add("csm.platform.databases.data.writer.password") { WRITER_USER_CREDENTIALS } registry.add("csm.platform.databases.data.reader.username") { READER_USER_CREDENTIALS } diff --git a/api/src/integrationTest/kotlin/com/cosmotech/api/home/run/RunControllerTests.kt b/api/src/integrationTest/kotlin/com/cosmotech/api/home/run/RunControllerTests.kt index 053b3ef25..abc1a5a9d 100644 --- a/api/src/integrationTest/kotlin/com/cosmotech/api/home/run/RunControllerTests.kt +++ b/api/src/integrationTest/kotlin/com/cosmotech/api/home/run/RunControllerTests.kt @@ -23,8 +23,6 @@ import com.cosmotech.api.home.run.RunConstants.RequestContent.CONTAINER_NAME import com.cosmotech.api.home.run.RunConstants.RequestContent.CONTAINER_NODE_LABEL import com.cosmotech.api.home.run.RunConstants.RequestContent.CONTAINER_RUN_ARGS import com.cosmotech.api.home.run.RunConstants.RequestContent.CSM_SIMULATION_RUN -import com.cosmotech.api.home.run.RunConstants.RequestContent.CUSTOM_DATA_QUERY -import com.cosmotech.api.home.run.RunConstants.RequestContent.CUSTOM_DATA_TABLE_NAME import com.cosmotech.api.home.run.RunConstants.RequestContent.DATASET_LIST import com.cosmotech.api.home.run.RunConstants.RequestContent.DESCRIPTION import com.cosmotech.api.home.run.RunConstants.RequestContent.HOST_NODE_NAME @@ -277,107 +275,6 @@ class RunControllerTests : ControllerTestBase() { "organizations/{organization_id}/workspaces/{workspace_id}/runners/{runner_id}/runs/{run_id}/DELETE")) } - @Test - @WithMockOauth2User - fun send_data_run() { - - every { eventPublisher.publishEvent(any()) } answers - { - firstArg().response = "Running" - } - - val dataToSend = - """{ - "id": "my_table", - "data": [ - { - "additionalProp1": {}, - "additionalProp2": "test", - "additionalProp3": 100 - }, - { - "additionalProp1": {}, - "additionalProp2": "test", - "additionalProp4": 1000 - } - ] - }""" - mvc.perform( - post( - "/organizations/$organizationId/workspaces/$workspaceId/runners/$runnerId}/runs/$runId/data/send") - .contentType(MediaType.APPLICATION_JSON) - .content(dataToSend) - .accept(MediaType.APPLICATION_JSON) - .with(csrf())) - .andExpect(status().is2xxSuccessful) - .andExpect(jsonPath("$.database_name").value(runId)) - .andExpect(jsonPath("$.table_name").value(CUSTOM_DATA_TABLE_NAME)) - .andExpect(jsonPath("$.data[0].additionalProp1").value("")) - .andExpect(jsonPath("$.data[0].additionalProp2").value("test")) - .andExpect(jsonPath("$.data[0].additionalProp3").value(100)) - .andExpect(jsonPath("$.data[1].additionalProp1").value("")) - .andExpect(jsonPath("$.data[1].additionalProp2").value("test")) - .andExpect(jsonPath("$.data[1].additionalProp4").value(1000)) - .andDo(MockMvcResultHandlers.print()) - .andDo( - document( - "organizations/{organization_id}/workspaces/{workspace_id}/runners/{runner_id}/runs/{run_id}/data/send/POST")) - } - - @Test - @WithMockOauth2User - fun query_data_run() { - - every { eventPublisher.publishEvent(any()) } answers - { - firstArg().response = "Running" - } - - val dataToSend = - """{ - "id": "my_table", - "data": [ - { - "additionalProp1": {}, - "additionalProp2": "test", - "additionalProp3": 100 - }, - { - "additionalProp1": {}, - "additionalProp2": "test", - "additionalProp4": 1000 - } - ] - }""" - mvc.perform( - post( - "/organizations/$organizationId/workspaces/$workspaceId/runners/$runnerId}/runs/$runId/data/send") - .contentType(MediaType.APPLICATION_JSON) - .content(dataToSend) - .accept(MediaType.APPLICATION_JSON) - .with(csrf())) - .andExpect(status().is2xxSuccessful) - - mvc.perform( - post( - "/organizations/$organizationId/workspaces/$workspaceId/runners/$runnerId}/runs/$runId/data/query") - .contentType(MediaType.APPLICATION_JSON) - .content(CUSTOM_DATA_QUERY) - .accept(MediaType.APPLICATION_JSON) - .with(csrf())) - .andExpect(status().is2xxSuccessful) - .andExpect(jsonPath("$.result[0].additionalprop1").value("")) - .andExpect(jsonPath("$.result[0].additionalprop2").value("test")) - .andExpect(jsonPath("$.result[0].additionalprop3").value(100)) - .andExpect(jsonPath("$.result[1].additionalprop1").value("")) - .andExpect(jsonPath("$.result[1].additionalprop2").value("test")) - .andExpect(jsonPath("$.result[1].additionalprop4").value(1000)) - .andDo(MockMvcResultHandlers.print()) - .andDo( - document( - "organizations/{organization_id}/workspaces/{workspace_id}/runners/{runner_id}/runs/{run_id}/data/query/POST")) - } - @Test @WithMockOauth2User fun get_run_logs() { diff --git a/api/src/integrationTest/resources/application-test.yml b/api/src/integrationTest/resources/application-test.yml index 0c9fe92b0..7f4fe0a1c 100644 --- a/api/src/integrationTest/resources/application-test.yml +++ b/api/src/integrationTest/resources/application-test.yml @@ -107,13 +107,8 @@ csm: tls: enabled: false bundle: "" - connector: - default-page-size: 5 data: - schema: "postgres" - admin: - password: "password" - username: cosmotech_api_admin + database: "postgres" host: "localhost" port: 5432 reader: diff --git a/api/src/integrationTest/resources/init-db.sql b/api/src/integrationTest/resources/init-db.sql index 18a77f012..783692013 100644 --- a/api/src/integrationTest/resources/init-db.sql +++ b/api/src/integrationTest/resources/init-db.sql @@ -1,3 +1,2 @@ CREATE USER readusertest WITH PASSWORD 'readusertest'; -CREATE USER adminusertest WITH SUPERUSER PASSWORD 'adminusertest'; CREATE USER writeusertest WITH PASSWORD 'writeusertest'; \ No newline at end of file diff --git a/common/src/main/kotlin/com/cosmotech/common/config/CsmPlatformProperties.kt b/common/src/main/kotlin/com/cosmotech/common/config/CsmPlatformProperties.kt index f3e88a3dc..9f9f52cf5 100644 --- a/common/src/main/kotlin/com/cosmotech/common/config/CsmPlatformProperties.kt +++ b/common/src/main/kotlin/com/cosmotech/common/config/CsmPlatformProperties.kt @@ -331,17 +331,14 @@ data class CsmPlatformProperties( /** Storage port */ val port: Int = 5432, - /** Storage schema */ - val schema: String = "cosmotech", + /** Storage database */ + val database: String = "cosmotech", /** Storage reader user configuration */ val reader: CsmStorageUser, /** Storage writer user configuration */ - val writer: CsmStorageUser, - - /** Storage admin user configuration */ - val admin: CsmStorageUser + val writer: CsmStorageUser ) { data class CsmStorageUser(val username: String, val password: String) } diff --git a/common/src/main/kotlin/com/cosmotech/common/config/PostgresConfiguration.kt b/common/src/main/kotlin/com/cosmotech/common/config/PostgresConfiguration.kt index 20e129df8..e9ec226aa 100644 --- a/common/src/main/kotlin/com/cosmotech/common/config/PostgresConfiguration.kt +++ b/common/src/main/kotlin/com/cosmotech/common/config/PostgresConfiguration.kt @@ -15,18 +15,7 @@ class PostgresConfiguration(val csmPlatformProperties: CsmPlatformProperties) { private val jdbcUrl = "jdbc:postgresql://${csmPlatformProperties.databases.data.host}" + ":${csmPlatformProperties.databases.data.port}" + - "/${csmPlatformProperties.databases.data.schema}" - - @Bean - fun adminUserJdbcTemplate(): JdbcTemplate { - val dataSource = - DriverManagerDataSource( - jdbcUrl, - csmPlatformProperties.databases.data.admin.username, - csmPlatformProperties.databases.data.admin.password) - dataSource.setDriverClassName(jdbcDriverClass) - return JdbcTemplate(dataSource) - } + "/${csmPlatformProperties.databases.data.database}" @Bean fun readerJdbcTemplate(): JdbcTemplate { diff --git a/common/src/main/kotlin/com/cosmotech/common/tests/CsmTestBase.kt b/common/src/main/kotlin/com/cosmotech/common/tests/CsmTestBase.kt index a85bf514e..994292c79 100644 --- a/common/src/main/kotlin/com/cosmotech/common/tests/CsmTestBase.kt +++ b/common/src/main/kotlin/com/cosmotech/common/tests/CsmTestBase.kt @@ -20,7 +20,6 @@ import org.testcontainers.utility.MountableFile open class CsmTestBase : AbstractTestcontainersRedisTestBase() { companion object { - private const val ADMIN_USER_CREDENTIALS = "adminusertest" private const val READER_USER_CREDENTIALS = "readusertest" private const val WRITER_USER_CREDENTIALS = "writeusertest" private const val DEFAULT_REDIS_PORT = 6379 @@ -72,10 +71,7 @@ open class CsmTestBase : AbstractTestcontainersRedisTestBase() { private fun initPostgresConfiguration(registry: DynamicPropertyRegistry) { registry.add("csm.platform.databases.data.host") { postgres.host } - registry.add("csm.platform.databases.data.schema") { postgres.databaseName } registry.add("csm.platform.databases.data.port") { postgres.getMappedPort(POSTGRESQL_PORT) } - registry.add("csm.platform.databases.data.admin.username") { ADMIN_USER_CREDENTIALS } - registry.add("csm.platform.databases.data.admin.password") { ADMIN_USER_CREDENTIALS } registry.add("csm.platform.databases.data.writer.username") { WRITER_USER_CREDENTIALS } registry.add("csm.platform.databases.data.writer.password") { WRITER_USER_CREDENTIALS } registry.add("csm.platform.databases.data.reader.username") { READER_USER_CREDENTIALS } diff --git a/dataset/src/integrationTest/resources/application-dataset-test.yml b/dataset/src/integrationTest/resources/application-dataset-test.yml index 37f3eb3c6..e6a8974fd 100644 --- a/dataset/src/integrationTest/resources/application-dataset-test.yml +++ b/dataset/src/integrationTest/resources/application-dataset-test.yml @@ -106,12 +106,7 @@ csm: tls: enabled: false bundle: "" - connector: - default-page-size: 5 data: - admin: - password: "password" - username: cosmotech_api_admin host: "localhost" port: 5432 reader: diff --git a/dataset/src/integrationTest/resources/init-db.sql b/dataset/src/integrationTest/resources/init-db.sql index 18a77f012..783692013 100644 --- a/dataset/src/integrationTest/resources/init-db.sql +++ b/dataset/src/integrationTest/resources/init-db.sql @@ -1,3 +1,2 @@ CREATE USER readusertest WITH PASSWORD 'readusertest'; -CREATE USER adminusertest WITH SUPERUSER PASSWORD 'adminusertest'; CREATE USER writeusertest WITH PASSWORD 'writeusertest'; \ No newline at end of file diff --git a/doc/.openapi-generator/FILES b/doc/.openapi-generator/FILES index 4091cafee..ed44fd68f 100644 --- a/doc/.openapi-generator/FILES +++ b/doc/.openapi-generator/FILES @@ -31,12 +31,9 @@ Models/OrganizationEditInfo.md Models/OrganizationRole.md Models/OrganizationSecurity.md Models/OrganizationUpdateRequest.md -Models/QueryResult.md Models/ResourceSizeInfo.md Models/Run.md Models/RunContainer.md -Models/RunData.md -Models/RunDataQuery.md Models/RunEditInfo.md Models/RunResourceRequested.md Models/RunState.md @@ -64,7 +61,6 @@ Models/RunnerRunTemplateParameterValue.md Models/RunnerSecurity.md Models/RunnerUpdateRequest.md Models/RunnerValidationStatus.md -Models/SendRunDataRequest.md Models/Solution.md Models/SolutionAccessControl.md Models/SolutionCreateRequest.md diff --git a/doc/Apis/RunApi.md b/doc/Apis/RunApi.md index 830a58dc5..d73f2321e 100644 --- a/doc/Apis/RunApi.md +++ b/doc/Apis/RunApi.md @@ -9,8 +9,6 @@ All URIs are relative to *http://localhost:8080* | [**getRunLogs**](RunApi.md#getRunLogs) | **GET** /organizations/{organization_id}/workspaces/{workspace_id}/runners/{runner_id}/runs/{run_id}/logs | get the logs for the Run | | [**getRunStatus**](RunApi.md#getRunStatus) | **GET** /organizations/{organization_id}/workspaces/{workspace_id}/runners/{runner_id}/runs/{run_id}/status | get the status for the Run | | [**listRuns**](RunApi.md#listRuns) | **GET** /organizations/{organization_id}/workspaces/{workspace_id}/runners/{runner_id}/runs | get the list of Runs for the Runner | -| [**queryRunData**](RunApi.md#queryRunData) | **POST** /organizations/{organization_id}/workspaces/{workspace_id}/runners/{runner_id}/runs/{run_id}/data/query | query the run data | -| [**sendRunData**](RunApi.md#sendRunData) | **POST** /organizations/{organization_id}/workspaces/{workspace_id}/runners/{runner_id}/runs/{run_id}/data/send | Send data associated to a run | @@ -154,61 +152,3 @@ get the list of Runs for the Runner - **Content-Type**: Not defined - **Accept**: application/json, application/yaml - -# **queryRunData** -> QueryResult queryRunData(organization\_id, workspace\_id, runner\_id, run\_id, RunDataQuery) - -query the run data - -### Parameters - -|Name | Type | Description | Notes | -|------------- | ------------- | ------------- | -------------| -| **organization\_id** | **String**| the Organization identifier | [default to null] | -| **workspace\_id** | **String**| the Workspace identifier | [default to null] | -| **runner\_id** | **String**| the Runner identifier | [default to null] | -| **run\_id** | **String**| the Run identifier | [default to null] | -| **RunDataQuery** | [**RunDataQuery**](../Models/RunDataQuery.md)| the query to run | | - -### Return type - -[**QueryResult**](../Models/QueryResult.md) - -### Authorization - -[oAuth2AuthCode](../README.md#oAuth2AuthCode) - -### HTTP request headers - -- **Content-Type**: application/json, application/yaml -- **Accept**: application/json, application/yaml - - -# **sendRunData** -> RunData sendRunData(organization\_id, workspace\_id, runner\_id, run\_id, SendRunDataRequest) - -Send data associated to a run - -### Parameters - -|Name | Type | Description | Notes | -|------------- | ------------- | ------------- | -------------| -| **organization\_id** | **String**| the Organization identifier | [default to null] | -| **workspace\_id** | **String**| the Workspace identifier | [default to null] | -| **runner\_id** | **String**| the Runner identifier | [default to null] | -| **run\_id** | **String**| the Run identifier | [default to null] | -| **SendRunDataRequest** | [**SendRunDataRequest**](../Models/SendRunDataRequest.md)| Custom data to register | | - -### Return type - -[**RunData**](../Models/RunData.md) - -### Authorization - -[oAuth2AuthCode](../README.md#oAuth2AuthCode) - -### HTTP request headers - -- **Content-Type**: application/json, application/yaml -- **Accept**: application/json, application/yaml - diff --git a/doc/Models/QueryResult.md b/doc/Models/QueryResult.md deleted file mode 100644 index 0f2019688..000000000 --- a/doc/Models/QueryResult.md +++ /dev/null @@ -1,9 +0,0 @@ -# QueryResult -## Properties - -| Name | Type | Description | Notes | -|------------ | ------------- | ------------- | -------------| -| **result** | [**List**](map.md) | the list of results | [optional] [default to null] | - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - diff --git a/doc/Models/RunData.md b/doc/Models/RunData.md deleted file mode 100644 index e0ce1335f..000000000 --- a/doc/Models/RunData.md +++ /dev/null @@ -1,11 +0,0 @@ -# RunData -## Properties - -| Name | Type | Description | Notes | -|------------ | ------------- | ------------- | -------------| -| **database\_name** | **String** | Database name | [optional] [default to null] | -| **table\_name** | **String** | Table name | [optional] [default to null] | -| **data** | [**List**](map.md) | | [optional] [default to null] | - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - diff --git a/doc/Models/RunDataQuery.md b/doc/Models/RunDataQuery.md deleted file mode 100644 index 62c42d561..000000000 --- a/doc/Models/RunDataQuery.md +++ /dev/null @@ -1,9 +0,0 @@ -# RunDataQuery -## Properties - -| Name | Type | Description | Notes | -|------------ | ------------- | ------------- | -------------| -| **query** | **String** | the query in SQL | [default to null] | - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - diff --git a/doc/Models/SendRunDataRequest.md b/doc/Models/SendRunDataRequest.md deleted file mode 100644 index ceab57c50..000000000 --- a/doc/Models/SendRunDataRequest.md +++ /dev/null @@ -1,10 +0,0 @@ -# SendRunDataRequest -## Properties - -| Name | Type | Description | Notes | -|------------ | ------------- | ------------- | -------------| -| **id** | **String** | | [default to null] | -| **data** | [**List**](map.md) | | [default to null] | - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - diff --git a/doc/README.md b/doc/README.md index 82aaca893..8fb46f6ca 100644 --- a/doc/README.md +++ b/doc/README.md @@ -48,8 +48,6 @@ All URIs are relative to *http://localhost:8080* *RunApi* | [**getRunLogs**](Apis/RunApi.md#getrunlogs) | **GET** /organizations/{organization_id}/workspaces/{workspace_id}/runners/{runner_id}/runs/{run_id}/logs | get the logs for the Run | *RunApi* | [**getRunStatus**](Apis/RunApi.md#getrunstatus) | **GET** /organizations/{organization_id}/workspaces/{workspace_id}/runners/{runner_id}/runs/{run_id}/status | get the status for the Run | *RunApi* | [**listRuns**](Apis/RunApi.md#listruns) | **GET** /organizations/{organization_id}/workspaces/{workspace_id}/runners/{runner_id}/runs | get the list of Runs for the Runner | -*RunApi* | [**queryRunData**](Apis/RunApi.md#queryrundata) | **POST** /organizations/{organization_id}/workspaces/{workspace_id}/runners/{runner_id}/runs/{run_id}/data/query | query the run data | -*RunApi* | [**sendRunData**](Apis/RunApi.md#sendrundata) | **POST** /organizations/{organization_id}/workspaces/{workspace_id}/runners/{runner_id}/runs/{run_id}/data/send | Send data associated to a run | | *RunnerApi* | [**createRunner**](Apis/RunnerApi.md#createrunner) | **POST** /organizations/{organization_id}/workspaces/{workspace_id}/runners | Create a new Runner | *RunnerApi* | [**createRunnerAccessControl**](Apis/RunnerApi.md#createrunneraccesscontrol) | **POST** /organizations/{organization_id}/workspaces/{workspace_id}/runners/{runner_id}/security/access | Add a control access to the Runner | *RunnerApi* | [**deleteRunner**](Apis/RunnerApi.md#deleterunner) | **DELETE** /organizations/{organization_id}/workspaces/{workspace_id}/runners/{runner_id} | Delete a runner | @@ -141,12 +139,9 @@ All URIs are relative to *http://localhost:8080* - [OrganizationRole](./Models/OrganizationRole.md) - [OrganizationSecurity](./Models/OrganizationSecurity.md) - [OrganizationUpdateRequest](./Models/OrganizationUpdateRequest.md) - - [QueryResult](./Models/QueryResult.md) - [ResourceSizeInfo](./Models/ResourceSizeInfo.md) - [Run](./Models/Run.md) - [RunContainer](./Models/RunContainer.md) - - [RunData](./Models/RunData.md) - - [RunDataQuery](./Models/RunDataQuery.md) - [RunEditInfo](./Models/RunEditInfo.md) - [RunResourceRequested](./Models/RunResourceRequested.md) - [RunState](./Models/RunState.md) @@ -174,7 +169,6 @@ All URIs are relative to *http://localhost:8080* - [RunnerSecurity](./Models/RunnerSecurity.md) - [RunnerUpdateRequest](./Models/RunnerUpdateRequest.md) - [RunnerValidationStatus](./Models/RunnerValidationStatus.md) - - [SendRunDataRequest](./Models/SendRunDataRequest.md) - [Solution](./Models/Solution.md) - [SolutionAccessControl](./Models/SolutionAccessControl.md) - [SolutionCreateRequest](./Models/SolutionCreateRequest.md) diff --git a/openapi/plantuml/schemas.plantuml b/openapi/plantuml/schemas.plantuml index 5116812e1..2630885cb 100644 --- a/openapi/plantuml/schemas.plantuml +++ b/openapi/plantuml/schemas.plantuml @@ -162,10 +162,6 @@ entity OrganizationUpdateRequest { name: String } -entity QueryResult { - result: List -} - entity ResourceSizeInfo { * cpu: String * memory: String @@ -206,16 +202,6 @@ entity RunContainer { runSizing: ContainerResourceSizing } -entity RunData { - database_name: String - table_name: String - data: List -} - -entity RunDataQuery { - * query: String -} - entity RunEditInfo { * timestamp: Long * userId: String @@ -465,11 +451,6 @@ entity RunnerUpdateRequest { entity RunnerValidationStatus { } -entity SendRunDataRequest { - * id: String - * data: List -} - entity Solution { * id: String * organizationId: String diff --git a/organization/src/integrationTest/resources/application-organization-test.yml b/organization/src/integrationTest/resources/application-organization-test.yml index a0884c751..af83d50be 100644 --- a/organization/src/integrationTest/resources/application-organization-test.yml +++ b/organization/src/integrationTest/resources/application-organization-test.yml @@ -116,9 +116,6 @@ csm: organization: default-page-size: 5 data: - admin: - password: "password" - username: cosmotech_api_admin host: "localhost" port: 5432 reader: diff --git a/organization/src/integrationTest/resources/init-db.sql b/organization/src/integrationTest/resources/init-db.sql index 18a77f012..783692013 100644 --- a/organization/src/integrationTest/resources/init-db.sql +++ b/organization/src/integrationTest/resources/init-db.sql @@ -1,3 +1,2 @@ CREATE USER readusertest WITH PASSWORD 'readusertest'; -CREATE USER adminusertest WITH SUPERUSER PASSWORD 'adminusertest'; CREATE USER writeusertest WITH PASSWORD 'writeusertest'; \ No newline at end of file diff --git a/run/src/integrationTest/kotlin/com/cosmotech/run/service/RunServiceIntegrationTest.kt b/run/src/integrationTest/kotlin/com/cosmotech/run/service/RunServiceIntegrationTest.kt index c9f598667..ebc4d0ef0 100644 --- a/run/src/integrationTest/kotlin/com/cosmotech/run/service/RunServiceIntegrationTest.kt +++ b/run/src/integrationTest/kotlin/com/cosmotech/run/service/RunServiceIntegrationTest.kt @@ -3,9 +3,6 @@ package com.cosmotech.run.service import com.cosmotech.common.config.CsmPlatformProperties -import com.cosmotech.common.config.existDB -import com.cosmotech.common.config.existTable -import com.cosmotech.common.config.toDataTableName import com.cosmotech.common.events.RunDeleted import com.cosmotech.common.events.RunStart import com.cosmotech.common.rbac.ROLE_ADMIN @@ -25,10 +22,8 @@ import com.cosmotech.organization.domain.OrganizationSecurity import com.cosmotech.run.RunApiServiceInterface import com.cosmotech.run.RunContainerFactory import com.cosmotech.run.domain.Run -import com.cosmotech.run.domain.RunDataQuery import com.cosmotech.run.domain.RunEditInfo import com.cosmotech.run.domain.RunStatus -import com.cosmotech.run.domain.SendRunDataRequest import com.cosmotech.run.workflow.WorkflowService import com.cosmotech.runner.RunnerApiServiceInterface import com.cosmotech.runner.domain.LastRunInfo @@ -55,31 +50,20 @@ import io.mockk.impl.annotations.MockK import io.mockk.junit5.MockKExtension import io.mockk.mockk import io.mockk.mockkStatic -import java.math.BigDecimal -import java.sql.SQLException import java.time.Instant import java.util.* -import kotlin.test.assertContentEquals import kotlin.test.assertEquals -import kotlin.test.assertFailsWith -import kotlin.test.assertFalse import kotlin.test.assertNotEquals -import kotlin.test.assertTrue -import org.json.JSONObject import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.Nested import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertDoesNotThrow import org.junit.jupiter.api.assertThrows import org.junit.jupiter.api.extension.ExtendWith import org.junit.runner.RunWith -import org.postgresql.util.PSQLException import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.http.HttpStatus -import org.springframework.jdbc.core.JdbcTemplate -import org.springframework.jdbc.datasource.DriverManagerDataSource import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.junit.jupiter.SpringExtension import org.springframework.test.context.junit4.SpringRunner @@ -112,8 +96,6 @@ class RunServiceIntegrationTest : CsmTestBase() { @SpykBean @Autowired lateinit var runApiService: RunApiServiceInterface @Autowired lateinit var eventPublisher: com.cosmotech.common.events.CsmEventPublisher - @Autowired lateinit var adminUserJdbcTemplate: JdbcTemplate - lateinit var dataset: DatasetCreateRequest lateinit var solution: SolutionCreateRequest lateinit var organization: OrganizationCreateRequest @@ -453,260 +435,4 @@ class RunServiceIntegrationTest : CsmTestBase() { runApiService.getRunLogs(organizationSaved.id, workspaceSaved.id, runnerSaved.id, runSavedId) } } - - @Nested - inner class RunServicePostgresIntegrationTest { - - lateinit var readerRunStorageTemplate: JdbcTemplate - - @BeforeEach - fun setUp() { - runSavedId = - mockStartRun(organizationSaved.id, workspaceSaved.id, runnerSaved.id, solutionSaved.id) - assertTrue(adminUserJdbcTemplate.existDB(runSavedId)) - - val databaseIO = csmPlatformProperties.databases.data - val runtimeDS = - DriverManagerDataSource( - "jdbc:postgresql://${databaseIO.host}:${databaseIO.port}" + "/$runSavedId", - databaseIO.reader.username, - databaseIO.reader.password) - runtimeDS.setDriverClassName("org.postgresql.Driver") - readerRunStorageTemplate = JdbcTemplate(runtimeDS) - } - - @Test - fun `test deleteRun should remove the database`() { - runApiService.deleteRun(organizationSaved.id, workspaceSaved.id, runnerSaved.id, runSavedId) - assertFalse(adminUserJdbcTemplate.existDB(runSavedId)) - } - - @Test - fun `test sendRunData must create table available to reader user`() { - val tableName = "MyCustomData" - val data = - listOf( - mapOf("param1" to "value1"), - mapOf("param2" to 2), - mapOf("param3" to JSONObject(mapOf("param4" to "value4")))) - val requestBody = SendRunDataRequest(id = tableName, data = data) - val runDataResult = - runApiService.sendRunData( - organizationSaved.id, workspaceSaved.id, runnerSaved.id, runSavedId, requestBody) - - assertEquals(tableName.toDataTableName(false), runDataResult.tableName) - - assertTrue(readerRunStorageTemplate.existTable(tableName.toDataTableName(false))) - - val rows = - readerRunStorageTemplate.queryForList( - "SELECT * FROM \"${tableName.toDataTableName(false)}\"") - - assertEquals(data.size, rows.size) - } - - @Test - fun `test sendRunData must create different tables when id change`() { - val tableName = "MyCustomData" - val data = - listOf( - mapOf("param1" to "value1"), - mapOf("param2" to 2), - mapOf("param3" to JSONObject(mapOf("param4" to "value4")))) - val requestBody = SendRunDataRequest(id = tableName, data = data) - runApiService.sendRunData( - organizationSaved.id, workspaceSaved.id, runnerSaved.id, runSavedId, requestBody) - - assertTrue(readerRunStorageTemplate.existTable(tableName.toDataTableName(false))) - - val rows = - readerRunStorageTemplate.queryForList( - "SELECT * FROM \"${tableName.toDataTableName(false)}\"") - - assertEquals(data.size, rows.size) - - val tableName2 = "MyCustomData2" - val data2 = listOf(mapOf("param1" to "value1"), mapOf("param2" to 2)) - val requestBody2 = SendRunDataRequest(id = tableName2, data = data2) - runApiService.sendRunData( - organizationSaved.id, workspaceSaved.id, runnerSaved.id, runSavedId, requestBody2) - - assertTrue(readerRunStorageTemplate.existTable(tableName2.toDataTableName(false))) - - val rows2 = - readerRunStorageTemplate.queryForList( - "SELECT * FROM \"${tableName2.toDataTableName(false)}\"") - - assertEquals(data2.size, rows2.size) - } - - @Test - fun `test multiple sendRunData with incompatible schema does not work`() { - val tableName = "MyCustomData" - val data = listOf(mapOf("parameter" to "stringValue")) - val data2 = listOf(mapOf("parameter" to JSONObject(mapOf("key" to "value")))) - val requestBody = SendRunDataRequest(id = tableName, data = data) - val requestBody2 = SendRunDataRequest(id = tableName, data = data2) - runApiService.sendRunData( - organizationSaved.id, workspaceSaved.id, runnerSaved.id, runSavedId, requestBody) - - assertFailsWith(SQLException::class, "Schema should have been rejected") { - runApiService.sendRunData( - organizationSaved.id, workspaceSaved.id, runnerSaved.id, runSavedId, requestBody2) - } - } - - @Test - fun `test multiple sendRunData with new columns works`() { - val tableName = "MyCustomData" - val data = listOf(mapOf("parameter" to "stringValue")) - val data2 = listOf(mapOf("parameter2" to JSONObject(mapOf("key" to "value")))) - val requestBody = SendRunDataRequest(id = tableName, data = data) - val requestBody2 = SendRunDataRequest(id = tableName, data = data2) - runApiService.sendRunData( - organizationSaved.id, workspaceSaved.id, runnerSaved.id, runSavedId, requestBody) - - runApiService.sendRunData( - organizationSaved.id, workspaceSaved.id, runnerSaved.id, runSavedId, requestBody2) - - val rows = - readerRunStorageTemplate.queryForList( - "SELECT * FROM \"${tableName.toDataTableName(false)}\"") - - assertEquals( - data.size + data2.size, - rows.size, - "Multiple send of data with new columns should add new rows in table") - } - - @Test - fun `test sendRunData with no data fails`() { - val tableName = "MyCustomData" - val data = listOf>() - val requestBody = SendRunDataRequest(id = tableName, data = data) - assertFailsWith( - IllegalArgumentException::class, "sendRunData must fail if data is an empty list") { - runApiService.sendRunData( - organizationSaved.id, workspaceSaved.id, runnerSaved.id, runSavedId, requestBody) - } - } - - @Test - fun `should get all entries`() { - val data = - listOf( - mapOf("param1" to "value1"), - mapOf("param2" to 2), - mapOf("param3" to mapOf("param4" to "value4"))) - val customDataId = "CustomData" - val requestBody = SendRunDataRequest(id = customDataId, data = data) - runApiService.sendRunData( - organizationSaved.id, workspaceSaved.id, runnerSaved.id, runSavedId, requestBody) - val queryResult = - runApiService.queryRunData( - organizationSaved.id, - workspaceSaved.id, - runnerSaved.id, - runSavedId, - RunDataQuery("SELECT * FROM ${customDataId.toDataTableName(false)}")) - val expectedResult = - listOf( - mapOf("param1" to "value1"), - mapOf("param2" to BigDecimal(2)), - mapOf("param3" to mapOf("param4" to "value4"))) - assertContentEquals(expectedResult, queryResult.result!!) - } - - @Test - fun `should throw table do not exist`() { - val data = - listOf( - mapOf("param1" to "value1"), - mapOf("param2" to 2), - mapOf("param3" to mapOf("param4" to "value4"))) - val customDataId = "CustomData" - val requestBody = SendRunDataRequest(id = customDataId, data = data) - runApiService.sendRunData( - organizationSaved.id, workspaceSaved.id, runnerSaved.id, runSavedId, requestBody) - val exception = - assertThrows { - runApiService.queryRunData( - organizationSaved.id, - workspaceSaved.id, - runnerSaved.id, - runSavedId, - RunDataQuery("SELECT * FROM ${customDataId.toDataTableName(false)}2")) - } - assertEquals( - "ERROR: relation \"${customDataId.toDataTableName(false)}2\" does not exist\n Position: 15", - exception.message) - } - - @Test - fun `should not allow command other than select`() { - val data = - listOf( - mapOf("param1" to "value1"), - mapOf("param2" to 2), - mapOf("param3" to mapOf("param4" to "value4"))) - val customDataId = "CustomData" - val requestBody = SendRunDataRequest(id = customDataId, data = data) - runApiService.sendRunData( - organizationSaved.id, workspaceSaved.id, runnerSaved.id, runSavedId, requestBody) - var e = - assertThrows { - runApiService.queryRunData( - organizationSaved.id, - workspaceSaved.id, - runnerSaved.id, - runSavedId, - RunDataQuery("DROP TABLE ${customDataId.toDataTableName(false)}")) - } - assertEquals( - "ERROR: must be owner of table ${customDataId.toDataTableName(false)}", e.message) - e = - assertThrows { - runApiService.queryRunData( - organizationSaved.id, - workspaceSaved.id, - runnerSaved.id, - runSavedId, - RunDataQuery( - "CREATE TABLE ${customDataId.toDataTableName(false)} (id VARCHAR(100))")) - } - assertEquals("ERROR: permission denied for schema public\n" + " Position: 14", e.message) - } - - @Test - fun `should get all tables in dB`() { - val data = - listOf( - mapOf("param1" to "value1"), - mapOf("param2" to 2), - mapOf("param3" to mapOf("param4" to "value4"))) - var requestBody = SendRunDataRequest(id = "table1", data = data) - runApiService.sendRunData( - organizationSaved.id, workspaceSaved.id, runnerSaved.id, runSavedId, requestBody) - requestBody = SendRunDataRequest(id = "table2", data = data) - runApiService.sendRunData( - organizationSaved.id, workspaceSaved.id, runnerSaved.id, runSavedId, requestBody) - requestBody = SendRunDataRequest(id = "table3", data = data) - runApiService.sendRunData( - organizationSaved.id, workspaceSaved.id, runnerSaved.id, runSavedId, requestBody) - val queryResult = - runApiService.queryRunData( - organizationSaved.id, - workspaceSaved.id, - runnerSaved.id, - runSavedId, - RunDataQuery( - "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public';")) - val expectedResult = - listOf( - mapOf("table_name" to ("table1").toDataTableName(false)), - mapOf("table_name" to ("table2").toDataTableName(false)), - mapOf("table_name" to ("table3").toDataTableName(false))) - assertEquals(expectedResult, queryResult.result) - } - } } diff --git a/run/src/integrationTest/resources/application-run-test.yml b/run/src/integrationTest/resources/application-run-test.yml index cedc71b8f..0855ea3e7 100644 --- a/run/src/integrationTest/resources/application-run-test.yml +++ b/run/src/integrationTest/resources/application-run-test.yml @@ -122,11 +122,8 @@ csm: run: default-page-size: 5 data: - admin: - password: "password" - username: cosmotech_api_admin host: "localhost" - schema: "postgres" + database: "postgres" port: 5432 reader: password: "password" diff --git a/run/src/integrationTest/resources/init-db.sql b/run/src/integrationTest/resources/init-db.sql index 18a77f012..783692013 100644 --- a/run/src/integrationTest/resources/init-db.sql +++ b/run/src/integrationTest/resources/init-db.sql @@ -1,3 +1,2 @@ CREATE USER readusertest WITH PASSWORD 'readusertest'; -CREATE USER adminusertest WITH SUPERUSER PASSWORD 'adminusertest'; CREATE USER writeusertest WITH PASSWORD 'writeusertest'; \ No newline at end of file diff --git a/run/src/main/kotlin/com/cosmotech/run/service/RunServiceImpl.kt b/run/src/main/kotlin/com/cosmotech/run/service/RunServiceImpl.kt index 91a114706..e1ae431aa 100644 --- a/run/src/main/kotlin/com/cosmotech/run/service/RunServiceImpl.kt +++ b/run/src/main/kotlin/com/cosmotech/run/service/RunServiceImpl.kt @@ -3,10 +3,6 @@ package com.cosmotech.run.service import com.cosmotech.common.CsmPhoenixService -import com.cosmotech.common.config.createDB -import com.cosmotech.common.config.dropDB -import com.cosmotech.common.config.existTable -import com.cosmotech.common.config.toDataTableName import com.cosmotech.common.events.RunDeleted import com.cosmotech.common.events.RunStart import com.cosmotech.common.events.RunStop @@ -22,15 +18,11 @@ import com.cosmotech.common.utils.getCurrentAccountIdentifier import com.cosmotech.run.RunApiServiceInterface import com.cosmotech.run.RunContainerFactory import com.cosmotech.run.container.StartInfo -import com.cosmotech.run.domain.QueryResult import com.cosmotech.run.domain.Run -import com.cosmotech.run.domain.RunData -import com.cosmotech.run.domain.RunDataQuery import com.cosmotech.run.domain.RunEditInfo import com.cosmotech.run.domain.RunState import com.cosmotech.run.domain.RunStatus import com.cosmotech.run.domain.RunTemplateParameterValue -import com.cosmotech.run.domain.SendRunDataRequest import com.cosmotech.run.repository.RunRepository import com.cosmotech.run.utils.isTerminal import com.cosmotech.run.utils.withoutSensitiveData @@ -38,58 +30,16 @@ import com.cosmotech.run.workflow.WorkflowService import com.cosmotech.runner.RunnerApiServiceInterface import com.cosmotech.runner.domain.Runner import com.cosmotech.runner.service.getRbac -import com.google.gson.Gson -import com.google.gson.JsonParser -import com.google.gson.reflect.TypeToken -import java.sql.SQLException import java.time.Instant -import org.postgresql.util.PGobject -import org.springframework.beans.factory.annotation.Value import org.springframework.context.event.EventListener import org.springframework.data.domain.PageRequest import org.springframework.data.domain.Pageable import org.springframework.http.HttpStatus -import org.springframework.jdbc.core.JdbcTemplate -import org.springframework.jdbc.datasource.DriverManagerDataSource import org.springframework.stereotype.Service import org.springframework.web.client.RestClientResponseException internal const val WORKFLOW_TYPE_RUN = "container-run" -internal const val BOOLEAN_KEY_WEIGHT = 0 -internal const val BOOLEAN_POSTGRESQL_TYPE = "BOOLEAN" -internal const val NUMERIC_KEY_WEIGHT = 1 -internal const val NUMERIC_POSTGRESQL_TYPE = "NUMERIC" -internal const val TEXT_KEY_WEIGHT = 2 -internal const val TEXT_POSTGRESQL_TYPE = "TEXT" -internal const val JSON_KEY_WEIGHT = 3 -internal const val JSON_POSTGRESQL_TYPE = "JSONB" -internal const val CONFLICT_KEY_WEIGHT = JSON_KEY_WEIGHT -internal const val POSTGRESQL_DRIVER_CLASS_NAME = "org.postgresql.Driver" - -internal val jsonTypeMapWeight = - mapOf( - Boolean::class.simpleName to BOOLEAN_KEY_WEIGHT, - Int::class.simpleName to NUMERIC_KEY_WEIGHT, - Double::class.simpleName to NUMERIC_KEY_WEIGHT, - String::class.simpleName to TEXT_KEY_WEIGHT, - ArrayList::class.simpleName to JSON_KEY_WEIGHT, - LinkedHashMap::class.simpleName to JSON_KEY_WEIGHT) - -internal val postgresTypeFromWeight = - mapOf( - BOOLEAN_KEY_WEIGHT to BOOLEAN_POSTGRESQL_TYPE, - NUMERIC_KEY_WEIGHT to NUMERIC_POSTGRESQL_TYPE, - TEXT_KEY_WEIGHT to TEXT_POSTGRESQL_TYPE, - JSON_KEY_WEIGHT to JSON_POSTGRESQL_TYPE) - -internal val postgresTypeToWeight = - mapOf( - BOOLEAN_POSTGRESQL_TYPE to BOOLEAN_KEY_WEIGHT, - NUMERIC_POSTGRESQL_TYPE to NUMERIC_KEY_WEIGHT, - TEXT_POSTGRESQL_TYPE to TEXT_KEY_WEIGHT, - JSON_POSTGRESQL_TYPE to JSON_KEY_WEIGHT) - @Service @Suppress("TooManyFunctions") class RunServiceImpl( @@ -97,31 +47,9 @@ class RunServiceImpl( private val workflowService: WorkflowService, private val runnerApiService: RunnerApiServiceInterface, private val runRepository: RunRepository, - private val csmRbac: CsmRbac, - private val adminUserJdbcTemplate: JdbcTemplate + private val csmRbac: CsmRbac ) : CsmPhoenixService(), RunApiServiceInterface { - @Value("\${csm.platform.databases.data.admin.username}") - private lateinit var adminStorageUsername: String - - @Value("\${csm.platform.databases.data.admin.password}") - private lateinit var adminStoragePassword: String - - @Value("\${csm.platform.databases.data.writer.username}") - private lateinit var writerStorageUsername: String - - @Value("\${csm.platform.databases.data.writer.password}") - private lateinit var writerStoragePassword: String - - @Value("\${csm.platform.databases.data.reader.username}") - private lateinit var readerStorageUsername: String - - @Value("\${csm.platform.databases.data.reader.password}") - private lateinit var readerStoragePassword: String - - @Value("\${csm.platform.databases.data.host}") private lateinit var host: String - @Value("\${csm.platform.databases.data.port}") private lateinit var port: String - override fun listRuns( organizationId: String, workspaceId: String, @@ -165,212 +93,6 @@ class RunServiceImpl( return runs } - @Suppress("LongMethod", "ThrowsCount") - fun sendDataToStorage( - runId: String, - tableName: String, - data: List>, - isProbeData: Boolean = false - ): RunData { - val dataTableName = tableName.toDataTableName(isProbeData) - - if (!dataTableName.matches(Regex("\\w+"))) { - throw SQLException("Table name \"$dataTableName\" is not a valid SQL identifier") - } - - // Start by looking through the data for postgresql column type inference - // Make use of a "weight" system for each data type to ensure most specific type when executed - val dataKeyWeight = mutableMapOf() - - val treatedData = mutableListOf>() - data.forEach { dataLine -> - val newDataLine = mutableMapOf() - dataLine.keys.forEach { key -> newDataLine[key.lowercase()] = dataLine[key]!! } - treatedData.add(newDataLine) - } - - treatedData.forEach { dataLine -> - dataLine.keys.forEach { key -> - // Get weight for a given column - if (!key.matches(Regex("\\w+"))) { - throw SQLException("Column name \"$key\" is not a valid SQL identifier") - } - val keyWeight = - jsonTypeMapWeight.getOrDefault(dataLine[key]!!::class.simpleName, CONFLICT_KEY_WEIGHT) - if (!dataKeyWeight.containsKey(key)) dataKeyWeight[key] = keyWeight - // If a conflict exists between 2 values in a same column use default value instead - else if (dataKeyWeight[key] != keyWeight) dataKeyWeight[key] = CONFLICT_KEY_WEIGHT - } - } - - val dataKeyType = dataKeyWeight.mapValues { postgresTypeFromWeight[it.value]!! }.toMutableMap() - val dataKeys = dataKeyType.keys - - val gson = Gson() - - val runtimeDS = - DriverManagerDataSource( - "jdbc:postgresql://$host:$port/$runId", writerStorageUsername, writerStoragePassword) - - runtimeDS.setDriverClassName(POSTGRESQL_DRIVER_CLASS_NAME) - val runDBJdbcTemplate = JdbcTemplate(runtimeDS) - val connection = runDBJdbcTemplate.dataSource!!.connection - - try { - // Start revert-able code - connection.autoCommit = false - if (!runDBJdbcTemplate.existTable(dataTableName)) { - // Table does not exist - connection - .prepareStatement( - "CREATE TABLE $dataTableName " + - "( ${dataKeys.joinToString(separator = ", ") { "$it ${dataKeyType[it]!!}" }} ) ") - .executeUpdate() - - logger.debug("Creating new table $dataTableName for run $runId") - } else { - // The table exist already - - // Start by getting the table schema (column name + type) - val listColumnsPreparedStatement = - connection.prepareStatement( - "SELECT column_name, upper(data_type::text) as column_type " + - "FROM information_schema.columns WHERE table_name ilike ?") - listColumnsPreparedStatement.setString(1, dataTableName) - val postgresTableKeyResultSet = listColumnsPreparedStatement.executeQuery() - - val postgresTableKeysType = mutableMapOf() - - // Each line contains "column_name" and "column_type" - while (postgresTableKeyResultSet.next()) postgresTableKeysType[ - postgresTableKeyResultSet.getString("column_name")] = - postgresTableKeyResultSet.getString("column_type") - - val postgresTableKeys: Set = postgresTableKeysType.keys - val postgresTableKeyWeight = - postgresTableKeysType.mapValues { postgresTypeToWeight[it.value]!! } - - val missingKeys = dataKeys - postgresTableKeys - val commonKeys = dataKeys - missingKeys - - // For each column both in data and existing table make a type check - commonKeys.forEach { commonKey -> - val columnKeyWeight = postgresTableKeyWeight[commonKey]!! - // If the existing type in the database is not compatible with the type of the data throw - // an error - if (columnKeyWeight != CONFLICT_KEY_WEIGHT && - dataKeyWeight[commonKey]!! != columnKeyWeight) - throw SQLException( - "Column $commonKey can not be converted to ${postgresTableKeysType[commonKey]!!}") - else dataKeyType[commonKey] = postgresTableKeysType[commonKey]!! - } - missingKeys.forEach { missingKey -> - // Alter the table for each missing key to add the column missing - logger.debug("Adding COLUMN $missingKey on table $dataTableName for run $runId") - connection - .prepareStatement( - "ALTER TABLE $dataTableName ADD COLUMN $missingKey ${dataKeyType[missingKey]!!}") - .executeUpdate() - } - } - treatedData.forEach { dataLine -> - // Insertion of data using prepared statements - // for each key a parameter is created in the SQL query - // that is then replaced by a string representation of the data that is then cast to the - // correct type - val insertPreparedStatement = - connection.prepareStatement( - "INSERT INTO $dataTableName ( ${dataLine.keys.joinToString(separator = ", ") {"$it"}} ) " + - "VALUES ( ${dataLine.keys.joinToString(separator = ", ") { "?::${dataKeyType[it]!!}" }} )") - // insert all values as pure data into the statement ensuring no SQL can be executed - // inside the query - dataLine.keys.forEachIndexed { index, key -> - if (dataKeyType[key] == JSON_POSTGRESQL_TYPE) - insertPreparedStatement.setString(index + 1, gson.toJson(dataLine[key])) - else insertPreparedStatement.setObject(index + 1, dataLine[key]) - } - insertPreparedStatement.executeUpdate() - } - - logger.debug("Inserted ${data.size} rows in table $dataTableName for run $runId") - connection.commit() - } catch (e: SQLException) { - connection.rollback() - throw e - } finally { - connection.close() - } - return RunData( - databaseName = runId, tableName = tableName.toDataTableName(isProbeData), data = data) - } - - override fun sendRunData( - organizationId: String, - workspaceId: String, - runnerId: String, - runId: String, - sendRunDataRequest: SendRunDataRequest - ): RunData { - - val run = getRun(organizationId, workspaceId, runnerId, runId) - run.hasPermission(PERMISSION_WRITE) - - require(sendRunDataRequest.data.isNotEmpty()) { "Data field cannot be empty" } - - return this.sendDataToStorage(runId, sendRunDataRequest.id, sendRunDataRequest.data) - } - - override fun queryRunData( - organizationId: String, - workspaceId: String, - runnerId: String, - runId: String, - runDataQuery: RunDataQuery - ): QueryResult { - - getRun(organizationId, workspaceId, runnerId, runId) - - val runtimeDS = - DriverManagerDataSource( - "jdbc:postgresql://$host:$port/$runId", readerStorageUsername, readerStoragePassword) - runtimeDS.setDriverClassName(POSTGRESQL_DRIVER_CLASS_NAME) - - val runDBJdbcTemplate = JdbcTemplate(runtimeDS) - val connection = runDBJdbcTemplate.dataSource!!.connection - - val preparedStatement = connection.prepareStatement(runDataQuery.query) - val queryResults = preparedStatement.executeQuery() - - val results = mutableListOf>() - val gson = Gson() - val mapAdapter = gson.getAdapter(object : TypeToken>() {}) - val arrayAdapter = gson.getAdapter(object : TypeToken>() {}) - while (queryResults.next()) { - val row = mutableMapOf() - for (i in 1..queryResults.metaData.columnCount) { - if (queryResults.getObject(i) == null) continue - - val resultValue = queryResults.getObject(i) - if (resultValue is PGobject) { - val parsedValue = JsonParser.parseString(resultValue.value) - - if (parsedValue.isJsonObject) - row[queryResults.metaData.getColumnName(i)] = mapAdapter.fromJson(resultValue.value) - else if (parsedValue.isJsonArray) - row[queryResults.metaData.getColumnName(i)] = arrayAdapter.fromJson(resultValue.value) - else if (parsedValue.asJsonPrimitive.isBoolean) - row[queryResults.metaData.getColumnName(i)] = parsedValue.asBoolean - else if (parsedValue.asJsonPrimitive.isNumber) - row[queryResults.metaData.getColumnName(i)] = parsedValue.asNumber - else row[queryResults.metaData.getColumnName(i)] = parsedValue.asString - } else row[queryResults.metaData.getColumnName(i)] = resultValue - } - results.add(row) - } - - return QueryResult(results) - } - private fun Run.withStateInformation(): Run { if (this.state?.isTerminal() == true) { return this @@ -462,8 +184,6 @@ class RunServiceImpl( RunDeleted(this, run.organizationId, run.workspaceId, run.runnerId, run.id!!, lastRun) this.eventPublisher.publishEvent(runDeleted) - adminUserJdbcTemplate.dropDB(run.id) - runRepository.delete(run) } catch (exception: IllegalStateException) { logger.debug( @@ -503,46 +223,6 @@ class RunServiceImpl( val runner = runStartRequest.runnerData as Runner val runId = idGenerator.generate("run", prependPrefix = "run-") - val dbComment = - "organizationId=${runner.organizationId}, workspaceId=${runner.workspaceId}, runnerId=${runner.id}" - adminUserJdbcTemplate.createDB(runId, dbComment) - - val runtimeDS = - DriverManagerDataSource( - "jdbc:postgresql://$host:$port/$runId", adminStorageUsername, adminStoragePassword) - runtimeDS.setDriverClassName(POSTGRESQL_DRIVER_CLASS_NAME) - val runDBJdbcTemplate = JdbcTemplate(runtimeDS) - val connection = runDBJdbcTemplate.dataSource!!.connection - try { - connection.autoCommit = false - connection - .prepareStatement("GRANT CONNECT ON DATABASE \"$runId\" TO $readerStorageUsername") - .executeUpdate() - connection - .prepareStatement("GRANT CONNECT ON DATABASE \"$runId\" TO $writerStorageUsername") - .executeUpdate() - - connection - .prepareStatement("GRANT CREATE ON SCHEMA public to $writerStorageUsername") - .executeUpdate() - - connection - .prepareStatement("GRANT USAGE ON SCHEMA public to $writerStorageUsername") - .executeUpdate() - - connection - .prepareStatement( - "ALTER DEFAULT PRIVILEGES FOR USER $writerStorageUsername IN SCHEMA public " + - "GRANT SELECT ON TABLES TO $readerStorageUsername;") - .executeUpdate() - connection.commit() - } catch (e: SQLException) { - connection.rollback() - throw e - } finally { - connection.close() - } - val startInfo = containerFactory.getStartInfo( runner.organizationId, runner.workspaceId, runner.id, WORKFLOW_TYPE_RUN, runId) diff --git a/run/src/main/openapi/run.yaml b/run/src/main/openapi/run.yaml index 4b1d05648..467aba33a 100644 --- a/run/src/main/openapi/run.yaml +++ b/run/src/main/openapi/run.yaml @@ -161,128 +161,6 @@ paths: examples: Run: $ref: '#/components/examples/BreweryRunLogs' - /organizations/{organization_id}/workspaces/{workspace_id}/runners/{runner_id}/runs/{run_id}/data/send: - parameters: - - name: organization_id - in: path - description: the Organization identifier - required: true - schema: - type: string - - name: workspace_id - in: path - description: the Workspace identifier - required: true - schema: - type: string - - name: runner_id - in: path - description: the Runner identifier - required: true - schema: - type: string - - name: run_id - in: path - description: the Run identifier - required: true - schema: - type: string - post: - operationId: sendRunData - tags: - - run - summary: Send data associated to a run - requestBody: - description: Custom data to register - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/SendRunDataRequest' - application/yaml: - schema: - $ref: '#/components/schemas/SendRunDataRequest' - responses: - "201": - description: Data stored - content: - application/json: - schema: - $ref: '#/components/schemas/RunData' - examples: - Run: - $ref: '#/components/examples/DefaultRunData' - application/yaml: - schema: - $ref: '#/components/schemas/RunData' - examples: - Run: - $ref: '#/components/examples/DefaultRunData' - "400": - description: Data sent format is malformed - /organizations/{organization_id}/workspaces/{workspace_id}/runners/{runner_id}/runs/{run_id}/data/query: - parameters: - - name: organization_id - in: path - description: the Organization identifier - required: true - schema: - type: string - - name: workspace_id - in: path - description: the Workspace identifier - required: true - schema: - type: string - - name: runner_id - in: path - description: the Runner identifier - required: true - schema: - type: string - - name: run_id - in: path - description: the Run identifier - required: true - schema: - type: string - post: - operationId: queryRunData - tags: - - run - summary: query the run data - requestBody: - description: the query to run - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/RunDataQuery' - examples: - GetAllRunDataInTable: - $ref: '#/components/examples/RunDataQueryExample' - application/yaml: - schema: - $ref: '#/components/schemas/RunDataQuery' - examples: - GetAllRunDataInTable: - $ref: '#/components/examples/RunDataQueryExample' - responses: - "200": - description: the query response - content: - application/json: - schema: - $ref: '#/components/schemas/QueryResult' - examples: - MultipleResults: - $ref: '#/components/examples/QueryResultExample' - application/yaml: - schema: - $ref: '#/components/schemas/QueryResult' - examples: - MultipleResults: - $ref: '#/components/examples/QueryResultExample' /organizations/{organization_id}/workspaces/{workspace_id}/runners/{runner_id}/runs: parameters: - name: organization_id @@ -353,35 +231,6 @@ components: tokenUrl: "https://example.com/token" scopes: {} schemas: - RunData: - type: object - description: Run Data stored - properties: - database_name: - type: string - description: Database name - table_name: - type: string - description: Table name - data: - type: array - items: - type: object - additionalProperties: true - SendRunDataRequest: - type: object - description: Run Data to send - properties: - id: - type: string - data: - type: array - items: - type: object - additionalProperties: true - required: - - id - - data RunStatus: type: object description: a Run status @@ -675,25 +524,6 @@ components: required: - cpu - memory - RunDataQuery: - type: object - description: a data result query in SQL - properties: - query: - type: string - description: the query in SQL - required: - - query - QueryResult: - type: object - description: the result of a SQL Query - properties: - result: - type: array - description: the list of results - items: - type: object - additionalProperties: true RunEditInfo: type: object properties: @@ -709,11 +539,6 @@ components: - userId examples: - DefaultRunData: - summary: Run Data exemple - description: Run Data exemple - value: - name: test BreweryRun: summary: Brewery Run example description: Brewery Run example @@ -864,17 +689,3 @@ components: progress: 1/1 startTime: "2021-05-18T16:15:53.000Z" endTime: "2021-05-18T16:16:03.000Z" - RunDataQueryExample: - summary: Get all runData in a table - description: Get all runData in a table - value: - query: "SELECT * FROM cd_mycustomdata" - QueryResultExample: - summary: Result of SQL Query - description: Result of SQL Query - value: - result: - - key1: value1 - key2: value2 - - key1: value3 - key2: value4 diff --git a/run/src/test/kotlin/com/cosmotech/run/ContainerFactoryTests.kt b/run/src/test/kotlin/com/cosmotech/run/ContainerFactoryTests.kt index 722ce6f7f..0c5303b76 100644 --- a/run/src/test/kotlin/com/cosmotech/run/ContainerFactoryTests.kt +++ b/run/src/test/kotlin/com/cosmotech/run/ContainerFactoryTests.kt @@ -105,9 +105,7 @@ class ContainerFactoryTests { CsmPlatformProperties.CsmDatabasesProperties.CsmDataIOProperties( host = "localhost", port = 5432, - admin = - CsmPlatformProperties.CsmDatabasesProperties.CsmDataIOProperties.CsmStorageUser( - password = "password", username = "username"), + database = "cosmotech", reader = CsmPlatformProperties.CsmDatabasesProperties.CsmDataIOProperties.CsmStorageUser( username = "username", password = "password"), diff --git a/runner/src/integrationTest/resources/application-runner-test.yml b/runner/src/integrationTest/resources/application-runner-test.yml index 0a8bae36c..ad1951d0e 100644 --- a/runner/src/integrationTest/resources/application-runner-test.yml +++ b/runner/src/integrationTest/resources/application-runner-test.yml @@ -112,9 +112,6 @@ csm: runner: default-page-size: 20 data: - admin: - password: "password" - username: cosmotech_api_admin host: "localhost" port: 5432 reader: diff --git a/runner/src/integrationTest/resources/init-db.sql b/runner/src/integrationTest/resources/init-db.sql index 18a77f012..783692013 100644 --- a/runner/src/integrationTest/resources/init-db.sql +++ b/runner/src/integrationTest/resources/init-db.sql @@ -1,3 +1,2 @@ CREATE USER readusertest WITH PASSWORD 'readusertest'; -CREATE USER adminusertest WITH SUPERUSER PASSWORD 'adminusertest'; CREATE USER writeusertest WITH PASSWORD 'writeusertest'; \ No newline at end of file diff --git a/solution/src/integrationTest/resources/application-solution-test.yml b/solution/src/integrationTest/resources/application-solution-test.yml index 96a1f1e72..e772ee3c5 100644 --- a/solution/src/integrationTest/resources/application-solution-test.yml +++ b/solution/src/integrationTest/resources/application-solution-test.yml @@ -103,9 +103,6 @@ csm: enabled: false bundle: "" data: - admin: - password: "password" - username: cosmotech_api_admin host: "localhost" port: 5432 reader: diff --git a/solution/src/integrationTest/resources/init-db.sql b/solution/src/integrationTest/resources/init-db.sql index 18a77f012..783692013 100644 --- a/solution/src/integrationTest/resources/init-db.sql +++ b/solution/src/integrationTest/resources/init-db.sql @@ -1,3 +1,2 @@ CREATE USER readusertest WITH PASSWORD 'readusertest'; -CREATE USER adminusertest WITH SUPERUSER PASSWORD 'adminusertest'; CREATE USER writeusertest WITH PASSWORD 'writeusertest'; \ No newline at end of file diff --git a/workspace/src/integrationTest/resources/application-workspace-test.yml b/workspace/src/integrationTest/resources/application-workspace-test.yml index c045b0788..4f7f8ccbf 100644 --- a/workspace/src/integrationTest/resources/application-workspace-test.yml +++ b/workspace/src/integrationTest/resources/application-workspace-test.yml @@ -42,7 +42,6 @@ csm: password: "test" checkSolutionImage: false identityProvider: - code: on_premise_one authorizationUrl: "http://fake_url:8080/authorize" tokenUrl: "http://fake_url:8080/token" containerScopes: @@ -125,9 +124,6 @@ csm: enabled: false bundle: "" data: - admin: - password: "password" - username: cosmotech_api_admin host: "localhost" port: 5432 reader: diff --git a/workspace/src/integrationTest/resources/init-db.sql b/workspace/src/integrationTest/resources/init-db.sql index 18a77f012..783692013 100644 --- a/workspace/src/integrationTest/resources/init-db.sql +++ b/workspace/src/integrationTest/resources/init-db.sql @@ -1,3 +1,2 @@ CREATE USER readusertest WITH PASSWORD 'readusertest'; -CREATE USER adminusertest WITH SUPERUSER PASSWORD 'adminusertest'; CREATE USER writeusertest WITH PASSWORD 'writeusertest'; \ No newline at end of file