Skip to content

Commit 21101b7

Browse files
finestructurefinestructure
authored
Merge pull request #3749 from SwiftPackageIndex/parallelise-tests-6
Parallelise tests
2 parents 3a42f4a + 80b785c commit 21101b7

File tree

16 files changed

+693
-209
lines changed

16 files changed

+693
-209
lines changed

‎.devcontainer/devcontainer.json‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,18 @@
11
{
22
"name": "spi-base",
33
"build": { "dockerfile": "Dockerfile" },
4-
"extensions": [
5-
"sswg.swift-lang"
6-
],
7-
"settings": {
8-
"lldb.library": "/usr/lib/liblldb.so"
4+
"forwardPorts": [8080],
5+
"customizations": {
6+
"vscode": {
7+
"extensions": [
8+
"sswg.swift-lang"
9+
],
10+
"settings": {
11+
"lldb.library": "/usr/lib/liblldb.so"
12+
}
13+
}
914
},
10-
"forwardPorts": [8080]
15+
"runArgs": [
16+
"--network=spi_test"
17+
]
1118
}

‎.env.testing.template‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
1-
DATABASE_HOST=localhost
2-
DATABASE_PORT=5432
1+
# DATABASE_HOST - localhost
2+
# DATABASE_PORT - unused
3+
# CI uses a hard-coded range 6000-6007 of ports and corresponding hosts,
4+
# locally it discovers running containers
5+
# and launches dbs as needed.
36
DATABASE_NAME=spi_test
47
DATABASE_USERNAME=spi_test
58
DATABASE_PASSWORD=xxx

‎.github/workflows/ci.yml‎

Lines changed: 89 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -31,20 +31,6 @@ jobs:
3131
container:
3232
image: registry.gitlab.com/finestructure/spi-base:1.2.0
3333
options: --privileged
34-
services:
35-
postgres:
36-
image: postgres:16-alpine
37-
env:
38-
POSTGRES_DB: spi_test
39-
POSTGRES_USER: spi_test
40-
POSTGRES_PASSWORD: xxx
41-
ports:
42-
- '5432:5432'
43-
options: >-
44-
--health-cmd pg_isready
45-
--health-interval 10s
46-
--health-timeout 5s
47-
--health-retries 5
4834
steps:
4935
- name: GH Runner bug workaround
5036
run: sysctl -w vm.mmap_rnd_bits=28
@@ -57,8 +43,95 @@ jobs:
5743
run: cp .env.testing.template .env.testing && make test
5844
env:
5945
COLLECTION_SIGNING_PRIVATE_KEY: ${{ secrets.COLLECTION_SIGNING_PRIVATE_KEY }}
60-
DATABASE_HOST: postgres
61-
DATABASE_PORT: '5432'
46+
services:
47+
spi_test_0:
48+
image: postgres:16-alpine
49+
env:
50+
POSTGRES_DB: spi_test
51+
POSTGRES_USER: spi_test
52+
POSTGRES_PASSWORD: xxx
53+
options: >-
54+
--health-cmd pg_isready
55+
--health-interval 10s
56+
--health-timeout 5s
57+
--health-retries 5
58+
spi_test_1:
59+
image: postgres:16-alpine
60+
env:
61+
POSTGRES_DB: spi_test
62+
POSTGRES_USER: spi_test
63+
POSTGRES_PASSWORD: xxx
64+
options: >-
65+
--health-cmd pg_isready
66+
--health-interval 10s
67+
--health-timeout 5s
68+
--health-retries 5
69+
spi_test_2:
70+
image: postgres:16-alpine
71+
env:
72+
POSTGRES_DB: spi_test
73+
POSTGRES_USER: spi_test
74+
POSTGRES_PASSWORD: xxx
75+
options: >-
76+
--health-cmd pg_isready
77+
--health-interval 10s
78+
--health-timeout 5s
79+
--health-retries 5
80+
spi_test_3:
81+
image: postgres:16-alpine
82+
env:
83+
POSTGRES_DB: spi_test
84+
POSTGRES_USER: spi_test
85+
POSTGRES_PASSWORD: xxx
86+
options: >-
87+
--health-cmd pg_isready
88+
--health-interval 10s
89+
--health-timeout 5s
90+
--health-retries 5
91+
spi_test_4:
92+
image: postgres:16-alpine
93+
env:
94+
POSTGRES_DB: spi_test
95+
POSTGRES_USER: spi_test
96+
POSTGRES_PASSWORD: xxx
97+
options: >-
98+
--health-cmd pg_isready
99+
--health-interval 10s
100+
--health-timeout 5s
101+
--health-retries 5
102+
spi_test_5:
103+
image: postgres:16-alpine
104+
env:
105+
POSTGRES_DB: spi_test
106+
POSTGRES_USER: spi_test
107+
POSTGRES_PASSWORD: xxx
108+
options: >-
109+
--health-cmd pg_isready
110+
--health-interval 10s
111+
--health-timeout 5s
112+
--health-retries 5
113+
spi_test_6:
114+
image: postgres:16-alpine
115+
env:
116+
POSTGRES_DB: spi_test
117+
POSTGRES_USER: spi_test
118+
POSTGRES_PASSWORD: xxx
119+
options: >-
120+
--health-cmd pg_isready
121+
--health-interval 10s
122+
--health-timeout 5s
123+
--health-retries 5
124+
spi_test_7:
125+
image: postgres:16-alpine
126+
env:
127+
POSTGRES_DB: spi_test
128+
POSTGRES_USER: spi_test
129+
POSTGRES_PASSWORD: xxx
130+
options: >-
131+
--health-cmd pg_isready
132+
--health-interval 10s
133+
--health-timeout 5s
134+
--health-retries 5
62135
63136
release-build-linux:
64137
name: Release build
@@ -93,8 +166,6 @@ jobs:
93166
# POSTGRES_DB: spi_dev
94167
# POSTGRES_USER: spi_dev
95168
# POSTGRES_PASSWORD: xxx
96-
# ports:
97-
# - 5432:5432
98169
# options: >-
99170
# --health-cmd pg_isready
100171
# --health-interval 10s

‎.readme-images/test-plan-options.png‎

74.2 KB
Loading

‎LOCAL_DEVELOPMENT_SETUP.md‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ Once you have the project cloned locally, the `Makefile` defines a set of useful
1414

1515
You'll need some environment variables configured before you can run the project. There are template files in the repository as `.env.testing.template` and `.env.development.template` and your first step should be to copy these files as `.env.testing` and `.env.development` and review their content in case your setup deviates from the default.
1616

17-
Then, to create Postgres databases in Docker for your development and test environments, run:
17+
Then, to create Postgres databases in Docker for your development environment, run:
1818

1919
```
2020
make db-up
@@ -38,15 +38,17 @@ Close the scheme editor and run the application by selecting "Run" from the Xcod
3838
[ NOTICE ] Server starting on http://127.0.0.1:8080 [component: server]
3939
```
4040

41-
When Xcode opens the `Package.swift` file, it will auto-create a test plan based on all tests in the project. This works for most cases, but we need to tell Xcode to run our tests sequentially, not in parallel. The first thing to do is to persist the autocreated test plan. From the Product menu, select "Test Plan" then "Manage Test Plans...", then click the small arrow button:
41+
When working locally, it's helpful to have a database with pre-populated data from the live system. [Talk to us on Discord](https://discord.gg/vQRb6KkYRw), and we'll supply you with a recent database dump that you can load with `./scripts/load-db.sh`.
4242

43-
![A screenshot of Xcode's scheme editor showing a small arrow next to 'SPI-Server-Package (Autocreated)'.](.readme-images/manage-test-plans.png)
43+
### Running the tests
4444

45-
Once you open the autocreated test plan, you will be asked if you would like to persist the test plan. Click "Save" and accept the default location in the `.swiftpm` directory. Then, for each item in the test plan, click the "Options" and select "Disabled" for the "Paralellization" setting.
45+
The suite is capabale of running the tests in parallel against a database pool of configurable size. The default is 8 databases and it can be changed via the environment variable `DATABASEPOOL_SIZE`.
4646

47-
![A screenshot of Xcode's test plan editor showing the parallelization options.](.readme-images/test-plan-options.png)
47+
The docker test databases will be launched and configured automatically when the tests are run. They will remain active after the tests have completed. If you prefer to have the database containers removed when the tests finish running, set the environment variable `DATABASEPOOL_TEARDOWN` to `true` or `1`.
4848

49-
When working locally, it's helpful to have a database with pre-populated data from the live system. [Talk to us on Discord](https://discord.gg/vQRb6KkYRw), and we'll supply you with a recent database dump that you can load with `./scripts/load-db.sh`.
49+
If you have an existing server project, make sure parallel testing is enable for the test target via its `Options...`.
50+
51+
![A screenshot of Xcode's test plan editor showing the parallelization options.](.readme-images/test-plan-options.png)
5052

5153
### Setup the Front End
5254

‎Makefile‎

Lines changed: 8 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ run:
3535

3636
test: xcbeautify
3737
set -o pipefail \
38-
&& swift test --disable-automatic-resolution --sanitize=thread --no-parallel \
38+
&& swift test --disable-automatic-resolution \
3939
2>&1 | ./xcbeautify --renderer github-actions
4040

4141
test-query-performance: xcbeautify
@@ -98,32 +98,21 @@ redis-up-dev:
9898
redis-down-dev:
9999
docker rm -f spi_redis
100100

101-
db-up: db-up-dev db-up-test redis-up-dev
101+
db-up: db-up-dev redis-up-dev
102102

103103
db-up-dev:
104104
docker run --name spi_dev -e POSTGRES_DB=spi_dev -e POSTGRES_USER=spi_dev -e POSTGRES_PASSWORD=xxx -p 6432:5432 -d postgres:16-alpine
105105

106-
# Keep test db on postgres:13 for now, to make local testing faster. See
107-
# https://github.com/SwiftPackageIndex/SwiftPackageIndex-Server/issues/3360#issuecomment-2331103211
108-
# for details
109-
db-up-test:
110-
docker run --name spi_test \
111-
-e POSTGRES_DB=spi_test \
112-
-e POSTGRES_USER=spi_test \
113-
-e POSTGRES_PASSWORD=xxx \
114-
-e PGDATA=/pgdata \
115-
--tmpfs /pgdata:rw,noexec,nosuid,size=1024m \
116-
-p 5432:5432 \
117-
-d \
118-
postgres:13-alpine
119-
120-
db-down: db-down-dev db-down-test redis-down-dev
106+
db-up-ci:
107+
./scripts/start-ci-dbs.sh
108+
109+
db-down: db-down-dev redis-down-dev
121110

122111
db-down-dev:
123112
docker rm -f spi_dev
124113

125-
db-down-test:
126-
docker rm -f spi_test
114+
db-down-ci:
115+
./scripts/stop-ci-dbs.sh
127116

128117
db-reset: db-down db-up migrate
129118

‎Sources/App/Core/Retry.swift‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
// Copyright Dave Verwer, Sven A. Schmidt, and other contributors.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
16+
public enum Retry {
17+
public enum Error: Swift.Error {
18+
case maxAttemptsExceeded
19+
}
20+
21+
public enum BackoffStrategy {
22+
case constant(Duration)
23+
24+
func delay(attempt: Int) async throws {
25+
switch self {
26+
case .constant(let duration):
27+
try await Task.sleep(for: duration)
28+
}
29+
}
30+
}
31+
}
32+
33+
34+
@discardableResult
35+
public func run<T>(
36+
maxAttempts: Int = 3,
37+
backoff: Retry.BackoffStrategy = .constant(.milliseconds(100)),
38+
operation: (_ attempt: Int) async throws -> T,
39+
errorLogger logError: ((Error) -> Void) = { print("\($0)") }
40+
) async throws -> T {
41+
var attemptsLeft = maxAttempts
42+
while attemptsLeft > 0 {
43+
let attempt = maxAttempts - attemptsLeft + 1
44+
do {
45+
return try await operation(attempt)
46+
} catch {
47+
logError(error)
48+
if attemptsLeft != maxAttempts {
49+
try? await backoff.delay(attempt: attempt)
50+
}
51+
attemptsLeft -= 1
52+
}
53+
}
54+
throw Retry.Error.maxAttemptsExceeded
55+
}
56+

‎Sources/App/configure.swift‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@ import FluentPostgresDriver
1818
import Vapor
1919

2020

21-
@discardableResult
22-
public func configure(_ app: Application, databasePort: Int? = nil) async throws -> String {
21+
public func configure(_ app: Application, databaseHost: String? = nil, databasePort: Int? = nil) async throws {
2322
#if DEBUG && os(macOS)
2423
// The bundle is only loaded if /Applications/InjectionIII.app exists on the local development machine.
2524
// Requires InjectionIII 4.7.3 or higher to be loaded for compatibility with Package.swift files.
@@ -52,7 +51,7 @@ public func configure(_ app: Application, databasePort: Int? = nil) async throws
5251

5352
// Setup database connection
5453
guard
55-
let host = Environment.get("DATABASE_HOST"),
54+
let host = databaseHost ?? Environment.get("DATABASE_HOST"),
5655
let port = databasePort ?? Environment.get("DATABASE_PORT").flatMap(Int.init),
5756
let username = Environment.get("DATABASE_USERNAME"),
5857
let password = Environment.get("DATABASE_PASSWORD"),
@@ -367,6 +366,4 @@ public func configure(_ app: Application, databasePort: Int? = nil) async throws
367366
// bootstrap app metrics
368367
@Dependency(\.metricsSystem) var metricsSystem
369368
metricsSystem.bootstrap()
370-
371-
return host
372369
}

‎Tests/AppTests/AllTests.swift‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,14 @@ import Testing
2020

2121

2222
@Suite(
23+
.setupDatabasePool,
2324
.dependency(\.date.now, .t0),
2425
.dependency(\.metricsSystem, .mock),
2526
.snapshots(record: .failed)
2627
) struct AllTests { }
2728

2829

30+
2931
extension AllTests {
3032
@Suite struct AlertingTests { }
3133
@Suite struct AnalyzerTests { }

0 commit comments

Comments
 (0)