diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 9d03e09b8..2265130c1 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,11 +1,18 @@ { "name": "spi-base", "build": { "dockerfile": "Dockerfile" }, - "extensions": [ - "sswg.swift-lang" - ], - "settings": { - "lldb.library": "/usr/lib/liblldb.so" + "forwardPorts": [8080], + "customizations": { + "vscode": { + "extensions": [ + "sswg.swift-lang" + ], + "settings": { + "lldb.library": "/usr/lib/liblldb.so" + } + } }, - "forwardPorts": [8080] + "runArgs": [ + "--network=spi_test" + ] } diff --git a/.env.testing.template b/.env.testing.template index a2b58f4f5..5e32d7161 100644 --- a/.env.testing.template +++ b/.env.testing.template @@ -1,5 +1,8 @@ -DATABASE_HOST=localhost -DATABASE_PORT=5432 +# DATABASE_HOST - localhost +# DATABASE_PORT - unused +# CI uses a hard-coded range 6000-6007 of ports and corresponding hosts, +# locally it discovers running containers +# and launches dbs as needed. DATABASE_NAME=spi_test DATABASE_USERNAME=spi_test DATABASE_PASSWORD=xxx diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4595d956f..a914772b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,20 +31,6 @@ jobs: container: image: registry.gitlab.com/finestructure/spi-base:1.2.0 options: --privileged - services: - postgres: - image: postgres:16-alpine - env: - POSTGRES_DB: spi_test - POSTGRES_USER: spi_test - POSTGRES_PASSWORD: xxx - ports: - - '5432:5432' - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 steps: - name: GH Runner bug workaround run: sysctl -w vm.mmap_rnd_bits=28 @@ -57,8 +43,95 @@ jobs: run: cp .env.testing.template .env.testing && make test env: COLLECTION_SIGNING_PRIVATE_KEY: ${{ secrets.COLLECTION_SIGNING_PRIVATE_KEY }} - DATABASE_HOST: postgres - DATABASE_PORT: '5432' + services: + spi_test_0: + image: postgres:16-alpine + env: + POSTGRES_DB: spi_test + POSTGRES_USER: spi_test + POSTGRES_PASSWORD: xxx + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + spi_test_1: + image: postgres:16-alpine + env: + POSTGRES_DB: spi_test + POSTGRES_USER: spi_test + POSTGRES_PASSWORD: xxx + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + spi_test_2: + image: postgres:16-alpine + env: + POSTGRES_DB: spi_test + POSTGRES_USER: spi_test + POSTGRES_PASSWORD: xxx + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + spi_test_3: + image: postgres:16-alpine + env: + POSTGRES_DB: spi_test + POSTGRES_USER: spi_test + POSTGRES_PASSWORD: xxx + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + spi_test_4: + image: postgres:16-alpine + env: + POSTGRES_DB: spi_test + POSTGRES_USER: spi_test + POSTGRES_PASSWORD: xxx + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + spi_test_5: + image: postgres:16-alpine + env: + POSTGRES_DB: spi_test + POSTGRES_USER: spi_test + POSTGRES_PASSWORD: xxx + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + spi_test_6: + image: postgres:16-alpine + env: + POSTGRES_DB: spi_test + POSTGRES_USER: spi_test + POSTGRES_PASSWORD: xxx + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + spi_test_7: + image: postgres:16-alpine + env: + POSTGRES_DB: spi_test + POSTGRES_USER: spi_test + POSTGRES_PASSWORD: xxx + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 release-build-linux: name: Release build @@ -93,8 +166,6 @@ jobs: # POSTGRES_DB: spi_dev # POSTGRES_USER: spi_dev # POSTGRES_PASSWORD: xxx - # ports: - # - 5432:5432 # options: >- # --health-cmd pg_isready # --health-interval 10s diff --git a/.readme-images/test-plan-options.png b/.readme-images/test-plan-options.png index b70139ecf..210e81df6 100644 Binary files a/.readme-images/test-plan-options.png and b/.readme-images/test-plan-options.png differ diff --git a/LOCAL_DEVELOPMENT_SETUP.md b/LOCAL_DEVELOPMENT_SETUP.md index be3a562f2..7c2541439 100644 --- a/LOCAL_DEVELOPMENT_SETUP.md +++ b/LOCAL_DEVELOPMENT_SETUP.md @@ -14,7 +14,7 @@ Once you have the project cloned locally, the `Makefile` defines a set of useful You'll need some environment variables configured before you can run the project. There are template files in the repository as `.env.testing.template` and `.env.development.template` and your first step should be to copy these files as `.env.testing` and `.env.development` and review their content in case your setup deviates from the default. -Then, to create Postgres databases in Docker for your development and test environments, run: +Then, to create Postgres databases in Docker for your development environment, run: ``` make db-up @@ -38,15 +38,17 @@ Close the scheme editor and run the application by selecting "Run" from the Xcod [ NOTICE ] Server starting on http://127.0.0.1:8080 [component: server] ``` -When Xcode opens the `Package.swift` file, it will auto-create a test plan based on all tests in the project. This works for most cases, but we need to tell Xcode to run our tests sequentially, not in parallel. The first thing to do is to persist the autocreated test plan. From the Product menu, select "Test Plan" then "Manage Test Plans...", then click the small arrow button: +When working locally, it's helpful to have a database with pre-populated data from the live system. [Talk to us on Discord](https://discord.gg/vQRb6KkYRw), and we'll supply you with a recent database dump that you can load with `./scripts/load-db.sh`. -![A screenshot of Xcode's scheme editor showing a small arrow next to 'SPI-Server-Package (Autocreated)'.](.readme-images/manage-test-plans.png) +### Running the tests -Once you open the autocreated test plan, you will be asked if you would like to persist the test plan. Click "Save" and accept the default location in the `.swiftpm` directory. Then, for each item in the test plan, click the "Options" and select "Disabled" for the "Paralellization" setting. +The suite is capabale of running the tests in parallel against a database pool of configurable size. The default is 8 databases and it can be changed via the environment variable `DATABASEPOOL_SIZE`. -![A screenshot of Xcode's test plan editor showing the parallelization options.](.readme-images/test-plan-options.png) +The docker test databases will be launched and configured automatically when the tests are run. They will remain active after the tests have completed. If you prefer to have the database containers removed when the tests finish running, set the environment variable `DATABASEPOOL_TEARDOWN` to `true` or `1`. -When working locally, it's helpful to have a database with pre-populated data from the live system. [Talk to us on Discord](https://discord.gg/vQRb6KkYRw), and we'll supply you with a recent database dump that you can load with `./scripts/load-db.sh`. +If you have an existing server project, make sure parallel testing is enable for the test target via its `Options...`. + +![A screenshot of Xcode's test plan editor showing the parallelization options.](.readme-images/test-plan-options.png) ### Setup the Front End diff --git a/Makefile b/Makefile index 692dfdea9..5304330dd 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ run: test: xcbeautify set -o pipefail \ - && swift test --disable-automatic-resolution --sanitize=thread --no-parallel \ + && swift test --disable-automatic-resolution \ 2>&1 | ./xcbeautify --renderer github-actions test-query-performance: xcbeautify @@ -98,32 +98,21 @@ redis-up-dev: redis-down-dev: docker rm -f spi_redis -db-up: db-up-dev db-up-test redis-up-dev +db-up: db-up-dev redis-up-dev db-up-dev: docker run --name spi_dev -e POSTGRES_DB=spi_dev -e POSTGRES_USER=spi_dev -e POSTGRES_PASSWORD=xxx -p 6432:5432 -d postgres:16-alpine -# Keep test db on postgres:13 for now, to make local testing faster. See -# https://github.com/SwiftPackageIndex/SwiftPackageIndex-Server/issues/3360#issuecomment-2331103211 -# for details -db-up-test: - docker run --name spi_test \ - -e POSTGRES_DB=spi_test \ - -e POSTGRES_USER=spi_test \ - -e POSTGRES_PASSWORD=xxx \ - -e PGDATA=/pgdata \ - --tmpfs /pgdata:rw,noexec,nosuid,size=1024m \ - -p 5432:5432 \ - -d \ - postgres:13-alpine - -db-down: db-down-dev db-down-test redis-down-dev +db-up-ci: + ./scripts/start-ci-dbs.sh + +db-down: db-down-dev redis-down-dev db-down-dev: docker rm -f spi_dev -db-down-test: - docker rm -f spi_test +db-down-ci: + ./scripts/stop-ci-dbs.sh db-reset: db-down db-up migrate diff --git a/Sources/App/Core/Retry.swift b/Sources/App/Core/Retry.swift new file mode 100644 index 000000000..832b33873 --- /dev/null +++ b/Sources/App/Core/Retry.swift @@ -0,0 +1,56 @@ +// Copyright Dave Verwer, Sven A. Schmidt, and other contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +public enum Retry { + public enum Error: Swift.Error { + case maxAttemptsExceeded + } + + public enum BackoffStrategy { + case constant(Duration) + + func delay(attempt: Int) async throws { + switch self { + case .constant(let duration): + try await Task.sleep(for: duration) + } + } + } +} + + +@discardableResult +public func run( + maxAttempts: Int = 3, + backoff: Retry.BackoffStrategy = .constant(.milliseconds(100)), + operation: (_ attempt: Int) async throws -> T, + errorLogger logError: ((Error) -> Void) = { print("\($0)") } +) async throws -> T { + var attemptsLeft = maxAttempts + while attemptsLeft > 0 { + let attempt = maxAttempts - attemptsLeft + 1 + do { + return try await operation(attempt) + } catch { + logError(error) + if attemptsLeft != maxAttempts { + try? await backoff.delay(attempt: attempt) + } + attemptsLeft -= 1 + } + } + throw Retry.Error.maxAttemptsExceeded +} + diff --git a/Sources/App/configure.swift b/Sources/App/configure.swift index 2d442bb8d..8afbccf51 100644 --- a/Sources/App/configure.swift +++ b/Sources/App/configure.swift @@ -18,8 +18,7 @@ import FluentPostgresDriver import Vapor -@discardableResult -public func configure(_ app: Application, databasePort: Int? = nil) async throws -> String { +public func configure(_ app: Application, databaseHost: String? = nil, databasePort: Int? = nil) async throws { #if DEBUG && os(macOS) // The bundle is only loaded if /Applications/InjectionIII.app exists on the local development machine. // Requires InjectionIII 4.7.3 or higher to be loaded for compatibility with Package.swift files. @@ -52,7 +51,7 @@ public func configure(_ app: Application, databasePort: Int? = nil) async throws // Setup database connection guard - let host = Environment.get("DATABASE_HOST"), + let host = databaseHost ?? Environment.get("DATABASE_HOST"), let port = databasePort ?? Environment.get("DATABASE_PORT").flatMap(Int.init), let username = Environment.get("DATABASE_USERNAME"), let password = Environment.get("DATABASE_PASSWORD"), @@ -367,6 +366,4 @@ public func configure(_ app: Application, databasePort: Int? = nil) async throws // bootstrap app metrics @Dependency(\.metricsSystem) var metricsSystem metricsSystem.bootstrap() - - return host } diff --git a/Tests/AppTests/AllTests.swift b/Tests/AppTests/AllTests.swift index ca659459e..18ea2281b 100644 --- a/Tests/AppTests/AllTests.swift +++ b/Tests/AppTests/AllTests.swift @@ -20,12 +20,14 @@ import Testing @Suite( + .setupDatabasePool, .dependency(\.date.now, .t0), .dependency(\.metricsSystem, .mock), .snapshots(record: .failed) ) struct AllTests { } + extension AllTests { @Suite struct AlertingTests { } @Suite struct AnalyzerTests { } diff --git a/Tests/AppTests/Helpers/DatabasePool.swift b/Tests/AppTests/Helpers/DatabasePool.swift new file mode 100644 index 000000000..1fc1c0830 --- /dev/null +++ b/Tests/AppTests/Helpers/DatabasePool.swift @@ -0,0 +1,339 @@ +// Copyright Dave Verwer, Sven A. Schmidt, and other contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +@testable import App + +import PostgresNIO +import ShellOut +import Vapor + + + +actor DatabasePool { + struct Database: Hashable { + let index: Int + let connectionDetails: ConnectionDetails + + init(index: Int) { + self.index = index + self.connectionDetails = .init(index: index) + } + + var host: String { connectionDetails.host } + var port: Int { connectionDetails.port } + } + + static let shared = DatabasePool(maxCount: Environment.databasePoolSize) + + var maxCount: Int + + init(maxCount: Int) { + self.maxCount = maxCount + } + + var availableDatabases: Set = .init() + + func setUp() async throws { + // Call DotEnvFile.load once to ensure env variables are set + await DotEnvFile.load(for: .testing, fileio: .init(threadPool: .singleton)) + + let runningDbs = try await runningDatabases() + + if isRunningInCI() { + // In CI, running dbs are new and need to be set up + try await withThrowingTaskGroup(of: Database.self) { group in + for db in runningDbs { + group.addTask { + try await db.setup(for: .testing) + return db + } + } + for try await db in group { + availableDatabases.insert(db) + } + } + } else { + // Re-use up to maxCount running dbs + for db in runningDbs.prefix(maxCount) { + availableDatabases.insert(db) + } + + do { // Delete overprovisioned dbs + let overprovisioned = runningDbs.dropFirst(maxCount) + try await tearDown(databases: overprovisioned) + } + + do { // Create missing dbs + let underprovisionedCount = max(maxCount - availableDatabases.count, 0) + try await withThrowingTaskGroup(of: Database.self) { group in + for _ in (0..) async throws { + guard Environment.databasePoolTearDown else { return } + try await withThrowingTaskGroup { group in + for db in databases { + group.addTask { + try await self.removeDB(database: db) + } + } + try await group.waitForAll() + } + } + + func withDatabase(_ operation: @Sendable (Database) async throws -> Void) async throws { + let db = try await retainDatabase() + do { + try await operation(db) + try await releaseDatabase(database: db) + } catch { + try await releaseDatabase(database: db) + throw error + } + } + + private func runningDatabases() async throws -> [Database] { + if isRunningInCI() { + // We don't have docker available in CI to probe for running dbs. + // Instead, we have a hard-coded list of dbs we launch in the GH workflow + // file and correspondingly, we hard-code their ports here. + return (0.. Database { + var database = availableDatabases.randomElement() + while database == nil { + try await Task.sleep(for: .milliseconds(10)) + database = availableDatabases.randomElement() + } + guard let database else { fatalError("database cannot be nil here") } + availableDatabases.remove(database) + return database + } + + private func releaseDatabase(database: Database) async throws { + availableDatabases.insert(database) + } + + private func launchDB(maxAttempts: Int = 3) async throws -> Database { + let port = Int.random(in: 10_000...65_000) + _ = try? await ShellOut.shellOut(to: .removeDB(port: port)) + try await run(maxAttempts: 3) { attempt in + print("⚠️ Launching DB on port \(port) (attempt: \(attempt))") + try await ShellOut.shellOut(to: .launchDB(port: port)) + } + return .init(index: port) + } + + private func removeDB(database: Database, maxAttempts: Int = 3) async throws { + try await run(maxAttempts: 3) { attempt in + // print("⚠️ Removing DB on port \(database.port) (attempt: \(attempt))") + try await ShellOut.shellOut(to: .removeDB(port: database.index)) + } + } +} + + +extension DatabasePool.Database { + + struct ConnectionDetails: Hashable { + var host: String + var port: Int + var username: String + var password: String + + init(index: Int) { + // Ensure DATABASE_HOST is from a restricted set db hostnames and nothing else. + // This is safeguard against accidental inheritance of setup in QueryPerformanceTests + // and to ensure the database resetting cannot impact any other network hosts. + if isRunningInCI() { + self.host = "spi_test_\(index)" + self.port = 5432 + } else { + self.host = Environment.get("DATABASE_HOST")! + precondition(["localhost", "postgres", "host.docker.internal"].contains(host), + "DATABASE_HOST must be a local db, was: \(host)") + self.port = index + } + self.username = Environment.get("DATABASE_USERNAME")! + self.password = Environment.get("DATABASE_PASSWORD")! + } + } + + func setup(for environment: Environment) async throws { + // Create initial db snapshot + try await createSchema(environment) + try await createSnapshot() + } + + func createSchema(_ environment: Environment) async throws { + do { + try await _withDatabase("postgres", details: connectionDetails, timeout: .seconds(10)) { // Connect to `postgres` db in order to reset the test db + let databaseName = Environment.get("DATABASE_NAME")! + try await $0.query(PostgresQuery(unsafeSQL: "DROP DATABASE IF EXISTS \(databaseName) WITH (FORCE)")) + try await $0.query(PostgresQuery(unsafeSQL: "CREATE DATABASE \(databaseName)")) + } + + do { // Use autoMigrate to spin up the schema + let app = try await Application.make(environment) + app.logger = .init(label: "noop") { _ in SwiftLogNoOpLogHandler() } + try await configure(app, databaseHost: connectionDetails.host, databasePort: connectionDetails.port) + try await app.autoMigrate() + try await app.asyncShutdown() + } + } catch { + print("Create schema failed with error: ", String(reflecting: error)) + throw error + } + } + + func createSnapshot() async throws { + let original = Environment.get("DATABASE_NAME")! + let snapshot = original + "_snapshot" + do { + try await _withDatabase("postgres", details: connectionDetails, timeout: .seconds(10)) { client in + try await client.query(PostgresQuery(unsafeSQL: "DROP DATABASE IF EXISTS \(snapshot) WITH (FORCE)")) + try await client.query(PostgresQuery(unsafeSQL: "CREATE DATABASE \(snapshot) TEMPLATE \(original)")) + } + } catch { + print("Create snapshot failed with error: ", String(reflecting: error)) + throw error + } + } + + func restoreSnapshot(details: ConnectionDetails) async throws { + let original = Environment.get("DATABASE_NAME")! + let snapshot = original + "_snapshot" + // delete db and re-create from snapshot + do { + try await _withDatabase("postgres", details: details, timeout: .seconds(10)) { client in + try await client.query(PostgresQuery(unsafeSQL: "DROP DATABASE IF EXISTS \(original) WITH (FORCE)")) + try await client.query(PostgresQuery(unsafeSQL: "CREATE DATABASE \(original) TEMPLATE \(snapshot)")) + } + } catch { + print("Restore snapshot failed with error: ", String(reflecting: error)) + throw error + } + } + +} + + +private func connect(to databaseName: String, details: DatabasePool.Database.ConnectionDetails) -> PostgresClient { + let config = PostgresClient.Configuration( + host: details.host, + port: details.port, + username: details.username, + password: details.password, + database: databaseName, + tls: .disable + ) + return .init(configuration: config) +} + + +private func _withDatabase(_ databaseName: String, + details: DatabasePool.Database.ConnectionDetails, + timeout: Duration, + _ query: @Sendable @escaping (PostgresClient) async throws -> Void) async throws { + let client = connect(to: databaseName, details: details) + try await run(timeout: timeout) { + try await withThrowingTaskGroup { taskGroup in + taskGroup.addTask { await client.run() } + + taskGroup.addTask { try await query(client) } + + try await taskGroup.next() + taskGroup.cancelAll() + } + } +} + + +extension Environment { + static var databasePoolSize: Int { + if isRunningInCI() { + 8 + } else { + Environment.get("DATABASEPOOL_SIZE").flatMap(Int.init) ?? 8 + } + } + + static var databasePoolTearDown: Bool { + if isRunningInCI() { + false + } else { + Environment.get("DATABASEPOOL_TEARDOWN").flatMap(\.asBool) ?? true + } + } +} + + +private enum TimeoutError: Error { + case timeout + case noResult +} + + +private func run(timeout: Duration, operation: @escaping @Sendable () async throws -> Void) async throws { + try await withThrowingTaskGroup(of: Bool.self) { group in + group.addTask { + try? await Task.sleep(for: timeout) + return false + } + group.addTask { + try await operation() + return true + } + let res = await group.nextResult() + group.cancelAll() + switch res { + case .success(false): + throw TimeoutError.timeout + case .success(true): + break + case .failure(let error): + throw error + case .none: + throw TimeoutError.noResult + } + } +} diff --git a/Tests/AppTests/Helpers/DatabasePoolSetupTrait.swift b/Tests/AppTests/Helpers/DatabasePoolSetupTrait.swift new file mode 100644 index 000000000..baf7ec43e --- /dev/null +++ b/Tests/AppTests/Helpers/DatabasePoolSetupTrait.swift @@ -0,0 +1,31 @@ +// Copyright Dave Verwer, Sven A. Schmidt, and other contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Testing + + +struct DatabasePoolSetupTrait: SuiteTrait, TestScoping { + func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws { + try await DatabasePool.shared.setUp() + try await function() + try await DatabasePool.shared.tearDown() + } +} + + +extension SuiteTrait where Self == DatabasePoolSetupTrait { + static var setupDatabasePool: Self { + Self() + } +} diff --git a/Tests/AppTests/Helpers/ShellOutCommand+ext.swift b/Tests/AppTests/Helpers/ShellOutCommand+ext.swift new file mode 100644 index 000000000..ebb9b9c09 --- /dev/null +++ b/Tests/AppTests/Helpers/ShellOutCommand+ext.swift @@ -0,0 +1,58 @@ +// Copyright Dave Verwer, Sven A. Schmidt, and other contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import ShellOut + + +extension ShellOutCommand { + static func launchDB(port: Int) -> ShellOutCommand { + .init(command: .docker, arguments: [ + "run", "--name", "spi_test_\(port)", + "-e", "POSTGRES_DB=spi_test", + "-e", "POSTGRES_USER=spi_test", + "-e", "POSTGRES_PASSWORD=xxx", + "-e", "PGDATA=/pgdata", + "--tmpfs", "/pgdata:rw,noexec,nosuid,size=1024m", + "-p", "\(port):5432", + "-d", + "postgres:13-alpine" + ]) + } + + static func removeDB(port: Int) -> ShellOutCommand { + .init(command: .docker, arguments: [ + "rm", "-f", "spi_test_\(port)" + ]) + } + + static var getContainerNames: ShellOutCommand { + .init(command: .docker, arguments: [ + "ps", "--format", "{{.Names}}" + ]) + } +} + + +private extension String { + static var docker: Self { +#if os(macOS) + // Starting from macOS 15.4.1 Xcode does not have `/usr/local/bin` in its path anymore. + // Therefore, we have to explicitly set it (or require project users to fiddle with Xcode + // to re-introduce it, which is not desirable). + "/usr/local/bin/docker" +#else + "docker" +#endif + } +} diff --git a/Tests/AppTests/Helpers/TestSupport.swift b/Tests/AppTests/Helpers/TestSupport.swift index c13b862a3..f902660ce 100644 --- a/Tests/AppTests/Helpers/TestSupport.swift +++ b/Tests/AppTests/Helpers/TestSupport.swift @@ -29,147 +29,23 @@ func withSPIApp( $0.logger = .noop } - try await TestSupport.setupDb(environment) - let app = try await TestSupport.setupApp(environment) - - return try await run { - try await setup(app) - try await withDependencies(updateValuesForOperation) { - try await test(app) - } - } defer: { - try await app.asyncShutdown() - } -} - - -func isRunningInCI() -> Bool { - ProcessInfo.processInfo.environment.keys.contains("GITHUB_WORKFLOW") -} - - -enum TestSupport { - - static func setupApp(_ environment: Environment, databasePort: Int? = nil) async throws -> Application { + try await DatabasePool.shared.withDatabase { database in + try await database.restoreSnapshot(details: database.connectionDetails) let app = try await Application.make(environment) - try await configure(app, databasePort: databasePort) - return app - } - - - static func setupDb(_ environment: Environment, databasePort: Int? = nil) async throws { - await DotEnvFile.load(for: environment, fileio: .init(threadPool: .singleton)) - - // Ensure DATABASE_HOST is from a restricted set db hostnames and nothing else. - // This is safeguard against accidental inheritance of setup in QueryPerformanceTests - // and to ensure the database resetting cannot impact any other network hosts. - let host = Environment.get("DATABASE_HOST") - let databasePort = databasePort ?? Environment.get("DATABASE_PORT").flatMap(Int.init)! - precondition(["localhost", "postgres", "host.docker.internal"].contains(host), - "DATABASE_HOST must be a local db, was: \(host)") - - let testDbName = Environment.get("DATABASE_NAME")! - let snapshotName = testDbName + "_snapshot" - - // Create initial db snapshot on first run - try await snapshotCreated.withValue { snapshotCreated in - if !snapshotCreated { - try await createSchema(environment, databaseName: testDbName, databasePort: databasePort) - try await createSnapshot(original: testDbName, snapshot: snapshotName, databasePort: databasePort, environment: environment) - snapshotCreated = true - } - } - - try await restoreSnapshot(original: testDbName, snapshot: snapshotName, databasePort: databasePort, environment: environment) - } - - - static func createSchema(_ environment: Environment, - databaseName: String, - databasePort: Int) async throws { - do { - try await withDatabase("postgres", port: databasePort, environment) { // Connect to `postgres` db in order to reset the test db - try await $0.query(PostgresQuery(unsafeSQL: "DROP DATABASE IF EXISTS \(databaseName) WITH (FORCE)")) - try await $0.query(PostgresQuery(unsafeSQL: "CREATE DATABASE \(databaseName)")) - } + try await configure(app, databaseHost: database.host, databasePort: database.port) - do { // Use autoMigrate to spin up the schema - let app = try await Application.make(environment) - app.logger = .init(label: "noop") { _ in SwiftLogNoOpLogHandler() } - try await configure(app, databasePort: databasePort) - try await app.autoMigrate() - try await app.asyncShutdown() + return try await run { + try await setup(app) + try await withDependencies(updateValuesForOperation) { + try await test(app) } - } catch { - print("Create schema failed with error: ", String(reflecting: error)) - throw error + } defer: { + try await app.asyncShutdown() } } - - - static func createSnapshot(original: String, - snapshot: String, - databasePort: Int, - environment: Environment) async throws { - do { - try await withDatabase("postgres", port: databasePort, environment) { client in - try await client.query(PostgresQuery(unsafeSQL: "DROP DATABASE IF EXISTS \(snapshot) WITH (FORCE)")) - try await client.query(PostgresQuery(unsafeSQL: "CREATE DATABASE \(snapshot) TEMPLATE \(original)")) - } - } catch { - print("Create snapshot failed with error: ", String(reflecting: error)) - throw error - } - } - - - static func restoreSnapshot(original: String, - snapshot: String, - databasePort: Int, - environment: Environment) async throws { - // delete db and re-create from snapshot - do { - try await withDatabase("postgres", port: databasePort, environment) { client in - try await client.query(PostgresQuery(unsafeSQL: "DROP DATABASE IF EXISTS \(original) WITH (FORCE)")) - try await client.query(PostgresQuery(unsafeSQL: "CREATE DATABASE \(original) TEMPLATE \(snapshot)")) - } - } catch { - print("Restore snapshot failed with error: ", String(reflecting: error)) - throw error - } - } - - - static let snapshotCreated = ActorIsolated(false) - -} - - -private func connect(to databaseName: String, - port: Int, - _ environment: Environment) async throws -> PostgresClient { - await DotEnvFile.load(for: environment, fileio: .init(threadPool: .singleton)) - let host = Environment.get("DATABASE_HOST")! - let username = Environment.get("DATABASE_USERNAME")! - let password = Environment.get("DATABASE_PASSWORD")! - - let config = PostgresClient.Configuration(host: host, port: port, username: username, password: password, database: databaseName, tls: .disable) - - return .init(configuration: config) } -private func withDatabase(_ databaseName: String, - port: Int, - _ environment: Environment, - _ query: @escaping (PostgresClient) async throws -> Void) async throws { - let client = try await connect(to: databaseName, port: port, environment) - try await withThrowingTaskGroup(of: Void.self) { taskGroup in - taskGroup.addTask { await client.run() } - - try await query(client) - - taskGroup.cancelAll() - } +func isRunningInCI() -> Bool { + ProcessInfo.processInfo.environment.keys.contains("GITHUB_WORKFLOW") } - diff --git a/Tests/AppTests/QueryPerformanceTests.swift b/Tests/AppTests/QueryPerformanceTests.swift index 78a1f2298..9d3d5d06e 100644 --- a/Tests/AppTests/QueryPerformanceTests.swift +++ b/Tests/AppTests/QueryPerformanceTests.swift @@ -20,17 +20,12 @@ import Testing import Vapor -extension AllTests { - @Suite( - .serialized, - .tags(.performance), - .disabled(if: !runQueryPerformanceTests()) - ) - struct QueryPerformanceTests { } -} - - -extension AllTests.QueryPerformanceTests { +@Suite( + .serialized, + .tags(.performance), + .disabled(if: !runQueryPerformanceTests()) +) +struct QueryPerformanceTests { // Set this to true when running locally to convert warnings to test failures for easier updating of values. static let failOnWarning = false @@ -38,13 +33,17 @@ extension AllTests.QueryPerformanceTests { // Update db settings for CI runs in // https://github.com/SwiftPackageIndex/SwiftPackageIndex-Server/settings/secrets/actions // or in `.env.staging` for local runs. - let app = try await Application.make(.staging) + let environment = Environment.staging + + // Make sure we're connecting to the correct database + let host = try await DotEnvFile.databaseHost(for: environment) + try #require(host.hasPrefix("spi-dev-db")) + try #require(host.hasSuffix("postgres.database.azure.com")) + + let app = try await Application.make(environment) app.logger.logLevel = Environment.get("LOG_LEVEL") .flatMap(Logger.Level.init(rawValue:)) ?? .warning - let host = try await configure(app) - - try #require(host.hasPrefix("spi-dev-db"), "was: \(host)") - try #require(host.hasSuffix("postgres.database.azure.com"), "was: \(host)") + try await configure(app) return try await run { try await test(app) @@ -181,7 +180,7 @@ extension AllTests.QueryPerformanceTests { JOIN versions v ON v.package_id = p.id WHERE v.reference ->> 'branch' = r.default_branch """) - try await assertQueryPerformance(query, expectedCost: 132_000, variation: 5000) + try await assertQueryPerformance(query, expectedCost: 150_000, variation: 5000) } } @@ -225,7 +224,7 @@ extension SQLQueryBuilder { } -private extension AllTests.QueryPerformanceTests { +private extension QueryPerformanceTests { func assertQueryPerformance(_ query: SQLQueryBuilder, expectedCost: Double, @@ -291,3 +290,30 @@ private extension AllTests.QueryPerformanceTests { } } + + +private struct TestError: Error { + var description: String + init(_ description: String) { + self.description = description + } +} + + +private extension [DotEnvFile.Line] { + subscript(key: String) -> String? { + first(where: { $0.key == key })?.value + } +} + + +private extension DotEnvFile { + static func databaseHost(for environment: Environment) async throws -> String { + let envFile = try await DotEnvFile.read(path: ".env.\(environment.name)", + fileio: .init(threadPool: .singleton)) + guard let host = envFile.lines["DATABASE_HOST"] else { + throw TestError("DATABASE_HOST not found in settings file .env.\(environment.name)") + } + return host + } +} diff --git a/scripts/start-ci-dbs.sh b/scripts/start-ci-dbs.sh new file mode 100755 index 000000000..6c0fc2d55 --- /dev/null +++ b/scripts/start-ci-dbs.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Sets up the databases the same way we do in ci.yml +# - no ports exposed +# - connected to a bridge network +# Tests need to be run from a container attached to the same network. + +docker network create -d bridge spi_test 2> /dev/null + +for port in {0..7}; do + docker run --name "spi_test_$port" \ + -e POSTGRES_DB=spi_test \ + -e POSTGRES_USER=spi_test \ + -e POSTGRES_PASSWORD=xxx \ + -e PGDATA=/pgdata \ + --tmpfs /pgdata:rw,noexec,nosuid,size=1024m \ + --network spi_test \ + -d \ + postgres:13-alpine +done diff --git a/scripts/stop-ci-dbs.sh b/scripts/stop-ci-dbs.sh new file mode 100755 index 000000000..4ec833d14 --- /dev/null +++ b/scripts/stop-ci-dbs.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +for c in $(docker ps --all --format "{{.Names}}" | grep spi_test_); do + docker rm -f "$c" +done + +docker network rm spi_test 2> /dev/null || true