Parallelise tests

.devcontainer/devcontainer.json

@@ -1,11 +1,18 @@
 {
     "name": "spi-base",
     "build": { "dockerfile": "Dockerfile" },
-    "extensions": [
-      "sswg.swift-lang"
-    ],
-    "settings": {
-      "lldb.library": "/usr/lib/liblldb.so"
+    "forwardPorts": [8080],
+    "customizations": {
+      "vscode": {
+        "extensions": [
+          "sswg.swift-lang"
+        ],
+        "settings": {
+          "lldb.library": "/usr/lib/liblldb.so"
+        }
+      }
     },
-    "forwardPorts": [8080]
+    "runArgs": [
+      "--network=spi_test"
+    ]
 }

.env.testing.template

@@ -1,5 +1,8 @@
-DATABASE_HOST=localhost
-DATABASE_PORT=5432
+# DATABASE_HOST - localhost
+# DATABASE_PORT - unused
+# CI uses a hard-coded range 6000-6007 of ports and corresponding hosts,
+# locally it discovers running containers
+# and launches dbs as needed.
 DATABASE_NAME=spi_test
 DATABASE_USERNAME=spi_test
 DATABASE_PASSWORD=xxx

.github/workflows/ci.yml

@@ -31,20 +31,6 @@ jobs:
     container:
       image: registry.gitlab.com/finestructure/spi-base:1.2.0
       options: --privileged
-    services:
-      postgres:
-        image: postgres:16-alpine
-        env:
-          POSTGRES_DB: spi_test
-          POSTGRES_USER: spi_test
-          POSTGRES_PASSWORD: xxx
-        ports:
-          - '5432:5432'
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
     steps:
       - name: GH Runner bug workaround
         run: sysctl -w vm.mmap_rnd_bits=28
@@ -57,8 +43,95 @@ jobs:
         run: cp .env.testing.template .env.testing && make test
         env:
           COLLECTION_SIGNING_PRIVATE_KEY: ${{ secrets.COLLECTION_SIGNING_PRIVATE_KEY }}
-          DATABASE_HOST: postgres
-          DATABASE_PORT: '5432'
+    services:
+      spi_test_0:
+        image: postgres:16-alpine
+        env:
+          POSTGRES_DB: spi_test
+          POSTGRES_USER: spi_test
+          POSTGRES_PASSWORD: xxx
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      spi_test_1:
+        image: postgres:16-alpine
+        env:
+          POSTGRES_DB: spi_test
+          POSTGRES_USER: spi_test
+          POSTGRES_PASSWORD: xxx
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      spi_test_2:
+        image: postgres:16-alpine
+        env:
+          POSTGRES_DB: spi_test
+          POSTGRES_USER: spi_test
+          POSTGRES_PASSWORD: xxx
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      spi_test_3:
+        image: postgres:16-alpine
+        env:
+          POSTGRES_DB: spi_test
+          POSTGRES_USER: spi_test
+          POSTGRES_PASSWORD: xxx
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      spi_test_4:
+        image: postgres:16-alpine
+        env:
+          POSTGRES_DB: spi_test
+          POSTGRES_USER: spi_test
+          POSTGRES_PASSWORD: xxx
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      spi_test_5:
+        image: postgres:16-alpine
+        env:
+          POSTGRES_DB: spi_test
+          POSTGRES_USER: spi_test
+          POSTGRES_PASSWORD: xxx
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      spi_test_6:
+        image: postgres:16-alpine
+        env:
+          POSTGRES_DB: spi_test
+          POSTGRES_USER: spi_test
+          POSTGRES_PASSWORD: xxx
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      spi_test_7:
+        image: postgres:16-alpine
+        env:
+          POSTGRES_DB: spi_test
+          POSTGRES_USER: spi_test
+          POSTGRES_PASSWORD: xxx
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
 
   release-build-linux:
     name: Release build
@@ -93,8 +166,6 @@ jobs:
   #         POSTGRES_DB: spi_dev
   #         POSTGRES_USER: spi_dev
   #         POSTGRES_PASSWORD: xxx
-  #       ports:
-  #         - 5432:5432
   #       options: >-
   #         --health-cmd pg_isready
   #         --health-interval 10s

LOCAL_DEVELOPMENT_SETUP.md

@@ -14,7 +14,7 @@ Once you have the project cloned locally, the `Makefile` defines a set of useful
 
 You'll need some environment variables configured before you can run the project. There are template files in the repository as `.env.testing.template` and `.env.development.template` and your first step should be to copy these files as `.env.testing` and `.env.development` and review their content in case your setup deviates from the default.
 
-Then, to create Postgres databases in Docker for your development and test environments, run:
+Then, to create Postgres databases in Docker for your development environment, run:
 
 ```
 make db-up
@@ -38,15 +38,17 @@ Close the scheme editor and run the application by selecting "Run" from the Xcod
 [ NOTICE ] Server starting on http://127.0.0.1:8080 [component: server]
 ```
 
-When Xcode opens the `Package.swift` file, it will auto-create a test plan based on all tests in the project. This works for most cases, but we need to tell Xcode to run our tests sequentially, not in parallel. The first thing to do is to persist the autocreated test plan. From the Product menu, select "Test Plan" then "Manage Test Plans...", then click the small arrow button:
+When working locally, it's helpful to have a database with pre-populated data from the live system. [Talk to us on Discord](https://discord.gg/vQRb6KkYRw), and we'll supply you with a recent database dump that you can load with `./scripts/load-db.sh`.
 
-![A screenshot of Xcode's scheme editor showing a small arrow next to 'SPI-Server-Package (Autocreated)'.](.readme-images/manage-test-plans.png)
+### Running the tests
 
-Once you open the autocreated test plan, you will be asked if you would like to persist the test plan. Click "Save" and accept the default location in the `.swiftpm` directory. Then, for each item in the test plan, click the "Options" and select "Disabled" for the "Paralellization" setting.
+The suite is capabale of running the tests in parallel against a database pool of configurable size. The default is 8 databases and it can be changed via the environment variable `DATABASEPOOL_SIZE`.
 
-![A screenshot of Xcode's test plan editor showing the parallelization options.](.readme-images/test-plan-options.png)
+The docker test databases will be launched and configured automatically when the tests are run. They will remain active after the tests have completed. If you prefer to have the database containers removed when the tests finish running, set the environment variable `DATABASEPOOL_TEARDOWN` to `true` or `1`.
 
-When working locally, it's helpful to have a database with pre-populated data from the live system. [Talk to us on Discord](https://discord.gg/vQRb6KkYRw), and we'll supply you with a recent database dump that you can load with `./scripts/load-db.sh`.
+If you have an existing server project, make sure parallel testing is enable for the test target via its `Options...`.
+
+![A screenshot of Xcode's test plan editor showing the parallelization options.](.readme-images/test-plan-options.png)
 
 ### Setup the Front End
 

Makefile

@@ -35,7 +35,7 @@ run:
 
 test: xcbeautify
 	set -o pipefail \
-	&& swift test --disable-automatic-resolution --sanitize=thread --no-parallel \
+	&& swift test --disable-automatic-resolution \
 	2>&1 | ./xcbeautify --renderer github-actions
 
 test-query-performance: xcbeautify
@@ -98,32 +98,21 @@ redis-up-dev:
 redis-down-dev:
 	docker rm -f spi_redis
 
-db-up: db-up-dev db-up-test redis-up-dev
+db-up: db-up-dev redis-up-dev
 
 db-up-dev:
 	docker run --name spi_dev -e POSTGRES_DB=spi_dev -e POSTGRES_USER=spi_dev -e POSTGRES_PASSWORD=xxx -p 6432:5432 -d postgres:16-alpine
 
-# Keep test db on postgres:13 for now, to make local testing faster. See
-# https://github.com/SwiftPackageIndex/SwiftPackageIndex-Server/issues/3360#issuecomment-2331103211
-# for details
-db-up-test:
-	docker run --name spi_test \
-		-e POSTGRES_DB=spi_test \
-		-e POSTGRES_USER=spi_test \
-		-e POSTGRES_PASSWORD=xxx \
-		-e PGDATA=/pgdata \
-		--tmpfs /pgdata:rw,noexec,nosuid,size=1024m \
-		-p 5432:5432 \
-		-d \
-		postgres:13-alpine
-
-db-down: db-down-dev db-down-test redis-down-dev
+db-up-ci:
+	./scripts/start-ci-dbs.sh
+
+db-down: db-down-dev redis-down-dev
 
 db-down-dev:
 	docker rm -f spi_dev
 
-db-down-test:
-	docker rm -f spi_test
+db-down-ci:
+	./scripts/stop-ci-dbs.sh
 
 db-reset: db-down db-up migrate
 

Sources/App/Core/Retry.swift

@@ -0,0 +1,56 @@
+// Copyright Dave Verwer, Sven A. Schmidt, and other contributors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+public enum Retry {
+    public enum Error: Swift.Error {
+        case maxAttemptsExceeded
+    }
+
+    public enum BackoffStrategy {
+        case constant(Duration)
+
+        func delay(attempt: Int) async throws {
+            switch self {
+                case .constant(let duration):
+                    try await Task.sleep(for: duration)
+            }
+        }
+    }
+}
+
+
+@discardableResult
+public func run<T>(
+    maxAttempts: Int = 3,
+    backoff: Retry.BackoffStrategy = .constant(.milliseconds(100)),
+    operation: (_ attempt: Int) async throws -> T,
+    errorLogger logError: ((Error) -> Void) = { print("\($0)") }
+) async throws -> T {
+    var attemptsLeft = maxAttempts
+    while attemptsLeft > 0 {
+        let attempt = maxAttempts - attemptsLeft + 1
+        do {
+            return try await operation(attempt)
+        } catch {
+            logError(error)
+            if attemptsLeft != maxAttempts {
+                try? await backoff.delay(attempt: attempt)
+            }
+            attemptsLeft -= 1
+        }
+    }
+    throw Retry.Error.maxAttemptsExceeded
+}
+

Sources/App/configure.swift

@@ -18,8 +18,7 @@ import FluentPostgresDriver
 import Vapor
 
 
-@discardableResult
-public func configure(_ app: Application, databasePort: Int? = nil) async throws -> String {
+public func configure(_ app: Application, databaseHost: String? = nil, databasePort: Int? = nil) async throws {
     #if DEBUG && os(macOS)
     // The bundle is only loaded if /Applications/InjectionIII.app exists on the local development machine.
     // Requires InjectionIII 4.7.3 or higher to be loaded for compatibility with Package.swift files.
@@ -52,7 +51,7 @@ public func configure(_ app: Application, databasePort: Int? = nil) async throws
 
     // Setup database connection
     guard
-        let host = Environment.get("DATABASE_HOST"),
+        let host = databaseHost ?? Environment.get("DATABASE_HOST"),
         let port = databasePort ?? Environment.get("DATABASE_PORT").flatMap(Int.init),
         let username = Environment.get("DATABASE_USERNAME"),
         let password = Environment.get("DATABASE_PASSWORD"),
@@ -367,6 +366,4 @@ public func configure(_ app: Application, databasePort: Int? = nil) async throws
     // bootstrap app metrics
     @Dependency(\.metricsSystem) var metricsSystem
     metricsSystem.bootstrap()
-
-    return host
 }

Tests/AppTests/AllTests.swift

@@ -20,12 +20,14 @@ import Testing
 
 
 @Suite(
+    .setupDatabasePool,
     .dependency(\.date.now, .t0),
     .dependency(\.metricsSystem, .mock),
     .snapshots(record: .failed)
 ) struct AllTests { }
 
 
+
 extension AllTests {
     @Suite struct AlertingTests { }
     @Suite struct AnalyzerTests { }