diff --git a/Sources/App/Commands/TriggerBuilds.swift b/Sources/App/Commands/TriggerBuilds.swift index 119b13ec5..4f32270cc 100644 --- a/Sources/App/Commands/TriggerBuilds.swift +++ b/Sources/App/Commands/TriggerBuilds.swift @@ -179,6 +179,7 @@ func triggerBuilds(on database: Database, packages: [Package.Id], force: Bool = false) async throws { @Dependency(\.environment) var environment + @Dependency(\.buildSystem) var buildSystem guard environment.allowBuildTriggers() else { Current.logger().info("Build trigger override switch OFF - no builds are being triggered") @@ -196,8 +197,9 @@ func triggerBuilds(on database: Database, } } - async let pendingJobsTask = Current.getStatusCount(client, .pending) - async let runningJobsTask = Current.getStatusCount(client, .running) + let getStatusCount = buildSystem.getStatusCount + async let pendingJobsTask = getStatusCount(client, .pending) + async let runningJobsTask = getStatusCount(client, .running) let pendingJobs = try await pendingJobsTask let runningJobs = try await runningJobsTask @@ -205,6 +207,7 @@ func triggerBuilds(on database: Database, AppMetrics.buildRunningJobsCount?.set(runningJobs) let newJobs = ActorIsolated(0) + let gitlabPipelineLimit = environment.gitlabPipelineLimit await withThrowingTaskGroup(of: Void.self) { group in for pkgId in packages { @@ -217,7 +220,7 @@ func triggerBuilds(on database: Database, group.addTask { // check if we have capacity to schedule more builds before querying for builds var newJobCount = await newJobs.value - guard pendingJobs + newJobCount < Current.gitlabPipelineLimit() else { + guard pendingJobs + newJobCount < gitlabPipelineLimit() else { Current.logger().info("too many pending pipelines (\(pendingJobs + newJobCount))") return } @@ -226,7 +229,7 @@ func triggerBuilds(on database: Database, let triggers = try await findMissingBuilds(database, packageId: pkgId) newJobCount = await newJobs.value - guard pendingJobs + newJobCount < Current.gitlabPipelineLimit() else { + guard pendingJobs + newJobCount < gitlabPipelineLimit() else { Current.logger().info("too many pending pipelines (\(pendingJobs + newJobCount))") return } diff --git a/Sources/App/Core/AppEnvironment.swift b/Sources/App/Core/AppEnvironment.swift index 4dd628ba6..bfa4450f5 100644 --- a/Sources/App/Core/AppEnvironment.swift +++ b/Sources/App/Core/AppEnvironment.swift @@ -24,22 +24,10 @@ import FoundationNetworking struct AppEnvironment: Sendable { var fileManager: FileManager - var getStatusCount: @Sendable (_ client: Client, _ status: Gitlab.Builder.Status) async throws -> Int var git: Git - var gitlabApiToken: @Sendable () -> String? - var gitlabPipelineToken: @Sendable () -> String? - var gitlabPipelineLimit: @Sendable () -> Int var logger: @Sendable () -> Logger var setLogger: @Sendable (Logger) -> Void var shell: Shell - var triggerBuild: @Sendable (_ client: Client, - _ buildId: Build.Id, - _ cloneURL: String, - _ isDocBuild: Bool, - _ platform: Build.Platform, - _ reference: Reference, - _ swiftVersion: SwiftVersion, - _ versionID: Version.Id) async throws -> Build.TriggerResponse } @@ -48,33 +36,10 @@ extension AppEnvironment { static let live = AppEnvironment( fileManager: .live, - getStatusCount: { client, status in - try await Gitlab.Builder.getStatusCount(client: client, - status: status, - page: 1, - pageSize: 100, - maxPageCount: 5) - }, git: .live, - gitlabApiToken: { Environment.get("GITLAB_API_TOKEN") }, - gitlabPipelineToken: { Environment.get("GITLAB_PIPELINE_TOKEN") }, - gitlabPipelineLimit: { - Environment.get("GITLAB_PIPELINE_LIMIT").flatMap(Int.init) - ?? Constants.defaultGitlabPipelineLimit - }, logger: { logger }, setLogger: { logger in Self.logger = logger }, - shell: .live, - triggerBuild: { client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in - try await Gitlab.Builder.triggerBuild(client: client, - buildId: buildId, - cloneURL: cloneURL, - isDocBuild: isDocBuild, - platform: platform, - reference: ref, - swiftVersion: swiftVersion, - versionID: versionID) - } + shell: .live ) } diff --git a/Sources/App/Core/Dependencies/BuildSystemClient.swift b/Sources/App/Core/Dependencies/BuildSystemClient.swift new file mode 100644 index 000000000..02abdb17c --- /dev/null +++ b/Sources/App/Core/Dependencies/BuildSystemClient.swift @@ -0,0 +1,73 @@ +// Copyright Dave Verwer, Sven A. Schmidt, and other contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Dependencies +import DependenciesMacros +import Vapor + + +@DependencyClient +struct BuildSystemClient { +#warning("remove client") + var getStatusCount: @Sendable (_ client: Client, _ status: Gitlab.Builder.Status) async throws -> Int +#warning("remove client") + var triggerBuild: @Sendable (_ client: Client, + _ buildId: Build.Id, + _ cloneURL: String, + _ isDocBuild: Bool, + _ platform: Build.Platform, + _ reference: Reference, + _ swiftVersion: SwiftVersion, + _ versionID: Version.Id) async throws -> Build.TriggerResponse +} + + +extension BuildSystemClient: DependencyKey { + static var liveValue: Self { + .init( + getStatusCount: { client, status in + try await Gitlab.Builder.getStatusCount(client: client, + status: status, + page: 1, + pageSize: 100, + maxPageCount: 5) + }, + triggerBuild: { client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in + try await Gitlab.Builder.triggerBuild(client: client, + buildId: buildId, + cloneURL: cloneURL, + isDocBuild: isDocBuild, + platform: platform, + reference: ref, + swiftVersion: swiftVersion, + versionID: versionID) + } + ) + } +} + + +extension BuildSystemClient: TestDependencyKey { + static var testValue: Self { Self() } +} + + +extension DependencyValues { + var buildSystem: BuildSystemClient { + get { self[BuildSystemClient.self] } + set { self[BuildSystemClient.self] = newValue } + } +} + + diff --git a/Sources/App/Core/Dependencies/EnvironmentClient.swift b/Sources/App/Core/Dependencies/EnvironmentClient.swift index 836473183..46d00bcfe 100644 --- a/Sources/App/Core/Dependencies/EnvironmentClient.swift +++ b/Sources/App/Core/Dependencies/EnvironmentClient.swift @@ -41,6 +41,9 @@ struct EnvironmentClient { var collectionSigningPrivateKey: @Sendable () -> Data? var current: @Sendable () -> Environment = { XCTFail("current"); return .development } var dbId: @Sendable () -> String? + var gitlabApiToken: @Sendable () -> String? + var gitlabPipelineLimit: @Sendable () -> Int = { XCTFail("gitlabPipelineLimit"); return 100 } + var gitlabPipelineToken: @Sendable () -> String? var hideStagingBanner: @Sendable () -> Bool = { XCTFail("hideStagingBanner"); return Constants.defaultHideStagingBanner } var loadSPIManifest: @Sendable (String) -> SPIManifest.Manifest? var maintenanceMessage: @Sendable () -> String? @@ -109,6 +112,12 @@ extension EnvironmentClient: DependencyKey { }, current: { (try? Environment.detect()) ?? .development }, dbId: { Environment.get("DATABASE_ID") }, + gitlabApiToken: { Environment.get("GITLAB_API_TOKEN") }, + gitlabPipelineLimit: { + Environment.get("GITLAB_PIPELINE_LIMIT").flatMap(Int.init) + ?? Constants.defaultGitlabPipelineLimit + }, + gitlabPipelineToken: { Environment.get("GITLAB_PIPELINE_TOKEN") }, hideStagingBanner: { Environment.get("HIDE_STAGING_BANNER").flatMap(\.asBool) ?? Constants.defaultHideStagingBanner diff --git a/Sources/App/Core/Gitlab.swift b/Sources/App/Core/Gitlab.swift index 2f83402d6..6225ea05c 100644 --- a/Sources/App/Core/Gitlab.swift +++ b/Sources/App/Core/Gitlab.swift @@ -79,9 +79,10 @@ extension Gitlab.Builder { versionID: Version.Id) async throws -> Build.TriggerResponse { @Dependency(\.environment) var environment - guard let pipelineToken = Current.gitlabPipelineToken(), + guard let pipelineToken = environment.gitlabPipelineToken(), let builderToken = environment.builderToken() else { throw Gitlab.Error.missingToken } + guard let awsDocsBucket = environment.awsDocsBucket() else { throw Gitlab.Error.missingConfiguration("AWS_DOCS_BUCKET") } @@ -157,10 +158,11 @@ extension Gitlab.Builder { status: Status, page: Int, pageSize: Int = 20) async throws -> [Pipeline] { - guard let apiToken = Current.gitlabApiToken() else { throw Gitlab.Error.missingToken } + @Dependency(\.environment) var environment + guard let apiToken = environment.gitlabApiToken() else { throw Gitlab.Error.missingToken } let uri: URI = .init(string: "\(projectURL)/pipelines?status=\(status)&page=\(page)&per_page=\(pageSize)") - let response = try await client.get(uri, headers: HTTPHeaders([("Authorization", "Bearer \(apiToken)")])) + let response = try await client.get(uri, headers: .bearer(apiToken)) guard response.status == .ok else { throw Gitlab.Error.requestFailed(response.status, uri) } @@ -197,3 +199,12 @@ private extension DateFormatter { return formatter } } + + +private extension HTTPHeaders { + static func bearer(_ token: String) -> Self { + .init([("Authorization", "Bearer \(token)")]) + } +} + + diff --git a/Sources/App/Models/Build.swift b/Sources/App/Models/Build.swift index 1b758c597..0cee9f3c4 100644 --- a/Sources/App/Models/Build.swift +++ b/Sources/App/Models/Build.swift @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +import Dependencies import Fluent import PostgresKit import Vapor @@ -186,14 +187,16 @@ extension Build { .with(\.$package) .first() .unwrap(or: Abort(.notFound)) - return try await Current.triggerBuild(client, - buildId, - version.package.url, - isDocBuild, - platform, - version.reference, - swiftVersion, - versionId) + + @Dependency(\.buildSystem) var buildSystem + return try await buildSystem.triggerBuild(client, + buildId, + version.package.url, + isDocBuild, + platform, + version.reference, + swiftVersion, + versionId) } } diff --git a/Tests/AppTests/BuildTests.swift b/Tests/AppTests/BuildTests.swift index 046340e6c..5d4b1b55b 100644 --- a/Tests/AppTests/BuildTests.swift +++ b/Tests/AppTests/BuildTests.swift @@ -133,19 +133,11 @@ class BuildTests: AppTestCase { $0.environment.awsDocsBucket = { "awsDocsBucket" } $0.environment.builderToken = { "builder token" } $0.environment.buildTimeout = { 10 } + $0.environment.gitlabPipelineToken = { "pipeline token" } $0.environment.siteURL = { "http://example.com" } - } operation: { - Current.gitlabPipelineToken = { "pipeline token" } - // setup - let p = try await savePackage(on: app.db, "1") - let v = try Version(package: p, reference: .branch("main")) - try await v.save(on: app.db) - let buildId = UUID() - let versionID = try XCTUnwrap(v.id) - // Use live dependency but replace actual client with a mock so we can // assert on the details being sent without actually making a request - Current.triggerBuild = { client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in + $0.buildSystem.triggerBuild = { @Sendable client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in try await Gitlab.Builder.triggerBuild(client: client, buildId: buildId, cloneURL: cloneURL, @@ -155,6 +147,14 @@ class BuildTests: AppTestCase { swiftVersion: swiftVersion, versionID: versionID) } + } operation: { + // setup + let p = try await savePackage(on: app.db, "1") + let v = try Version(package: p, reference: .branch("main")) + try await v.save(on: app.db) + let buildId = UUID() + let versionID = try XCTUnwrap(v.id) + var called = false let client = MockClient { req, res in called = true @@ -201,21 +201,11 @@ class BuildTests: AppTestCase { $0.environment.awsDocsBucket = { "awsDocsBucket" } $0.environment.builderToken = { "builder token" } $0.environment.buildTimeout = { 10 } + $0.environment.gitlabPipelineToken = { "pipeline token" } $0.environment.siteURL = { "http://example.com" } - } operation: { - // Same test as test_trigger above, except we trigger with isDocBuild: true - // and expect a 15m TIMEOUT instead of 10m - Current.gitlabPipelineToken = { "pipeline token" } - // setup - let p = try await savePackage(on: app.db, "1") - let v = try Version(package: p, reference: .branch("main")) - try await v.save(on: app.db) - let buildId = UUID() - let versionID = try XCTUnwrap(v.id) - // Use live dependency but replace actual client with a mock so we can // assert on the details being sent without actually making a request - Current.triggerBuild = { client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in + $0.buildSystem.triggerBuild = { @Sendable client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in try await Gitlab.Builder.triggerBuild(client: client, buildId: buildId, cloneURL: cloneURL, @@ -225,6 +215,16 @@ class BuildTests: AppTestCase { swiftVersion: swiftVersion, versionID: versionID) } + } operation: { + // Same test as test_trigger above, except we trigger with isDocBuild: true + // and expect a 15m TIMEOUT instead of 10m + // setup + let p = try await savePackage(on: app.db, "1") + let v = try Version(package: p, reference: .branch("main")) + try await v.save(on: app.db) + let buildId = UUID() + let versionID = try XCTUnwrap(v.id) + var called = false let client = MockClient { req, res in called = true diff --git a/Tests/AppTests/BuildTriggerTests.swift b/Tests/AppTests/BuildTriggerTests.swift index a30cde1e0..8e284474c 100644 --- a/Tests/AppTests/BuildTriggerTests.swift +++ b/Tests/AppTests/BuildTriggerTests.swift @@ -345,14 +345,11 @@ class BuildTriggerTests: AppTestCase { $0.environment.awsDocsBucket = { "awsDocsBucket" } $0.environment.builderToken = { "builder token" } $0.environment.buildTimeout = { 10 } + $0.environment.gitlabPipelineToken = { "pipeline token" } $0.environment.siteURL = { "http://example.com" } - } operation: { - // setup - Current.gitlabPipelineToken = { "pipeline token" } - // Use live dependency but replace actual client with a mock so we can // assert on the details being sent without actually making a request - Current.triggerBuild = { client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in + $0.buildSystem.triggerBuild = { @Sendable client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in try await Gitlab.Builder.triggerBuild(client: client, buildId: buildId, cloneURL: cloneURL, @@ -362,6 +359,8 @@ class BuildTriggerTests: AppTestCase { swiftVersion: swiftVersion, versionID: versionID) } + } operation: { + // setup let queries = QueueIsolated<[Gitlab.Builder.PostDTO]>([]) let client = MockClient { req, res in guard let query = try? req.query.decode(Gitlab.Builder.PostDTO.self) else { return } @@ -408,15 +407,11 @@ class BuildTriggerTests: AppTestCase { $0.environment.builderToken = { "builder token" } $0.environment.buildTimeout = { 10 } $0.environment.buildTriggerAllowList = { [] } + $0.environment.gitlabPipelineToken = { "pipeline token" } $0.environment.siteURL = { "http://example.com" } - } operation: { - // Explicitly test the full range of all currently triggered platforms and swift versions - // setup - Current.gitlabPipelineToken = { "pipeline token" } - // Use live dependency but replace actual client with a mock so we can // assert on the details being sent without actually making a request - Current.triggerBuild = { client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in + $0.buildSystem.triggerBuild = { @Sendable client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in try await Gitlab.Builder.triggerBuild(client: client, buildId: buildId, cloneURL: cloneURL, @@ -426,6 +421,9 @@ class BuildTriggerTests: AppTestCase { swiftVersion: swiftVersion, versionID: versionID) } + } operation: { + // Explicitly test the full range of all currently triggered platforms and swift versions + // setup let queries = QueueIsolated<[Gitlab.Builder.PostDTO]>([]) let client = MockClient { req, res in guard let query = try? req.query.decode(Gitlab.Builder.PostDTO.self) else { return } @@ -489,7 +487,20 @@ class BuildTriggerTests: AppTestCase { $0.environment.awsDocsBucket = { "awsDocsBucket" } $0.environment.builderToken = { "builder token" } $0.environment.buildTimeout = { 10 } + $0.environment.gitlabPipelineToken = { "pipeline token" } $0.environment.siteURL = { "http://example.com" } + // Use live dependency but replace actual client with a mock so we can + // assert on the details being sent without actually making a request + $0.buildSystem.triggerBuild = { @Sendable client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in + try await Gitlab.Builder.triggerBuild(client: client, + buildId: buildId, + cloneURL: cloneURL, + isDocBuild: isDocBuild, + platform: platform, + reference: ref, + swiftVersion: swiftVersion, + versionID: versionID) + } } operation: { // Tests error handling when a build record already exists and `create` raises a // uq:builds.version_id+builds.platform+builds.swift_version+v2 @@ -502,20 +513,6 @@ class BuildTriggerTests: AppTestCase { // being completely ignored because the command errors out. // See https://github.com/SwiftPackageIndex/SwiftPackageIndex-Server/issues/2237 // setup - Current.gitlabPipelineToken = { "pipeline token" } - - // Use live dependency but replace actual client with a mock so we can - // assert on the details being sent without actually making a request - Current.triggerBuild = { client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in - try await Gitlab.Builder.triggerBuild(client: client, - buildId: buildId, - cloneURL: cloneURL, - isDocBuild: isDocBuild, - platform: platform, - reference: ref, - swiftVersion: swiftVersion, - versionID: versionID) - } let queries = QueueIsolated<[Gitlab.Builder.PostDTO]>([]) let client = MockClient { req, res in guard let query = try? req.query.decode(Gitlab.Builder.PostDTO.self) else { return } @@ -572,16 +569,13 @@ class BuildTriggerTests: AppTestCase { $0.environment.buildTimeout = { 10 } $0.environment.buildTriggerAllowList = { [] } $0.environment.buildTriggerDownscaling = { 1 } + $0.environment.gitlabPipelineLimit = { 300 } + $0.environment.gitlabPipelineToken = { "pipeline token" } $0.environment.random = { @Sendable _ in 0 } $0.environment.siteURL = { "http://example.com" } - } operation: { - // Ensure we respect the pipeline limit when triggering builds - // setup - Current.gitlabPipelineToken = { "pipeline token" } - Current.gitlabPipelineLimit = { 300 } // Use live dependency but replace actual client with a mock so we can // assert on the details being sent without actually making a request - Current.triggerBuild = { client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in + $0.buildSystem.triggerBuild = { @Sendable client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in try await Gitlab.Builder.triggerBuild(client: client, buildId: buildId, cloneURL: cloneURL, @@ -591,6 +585,9 @@ class BuildTriggerTests: AppTestCase { swiftVersion: swiftVersion, versionID: versionID) } + } operation: { + // Ensure we respect the pipeline limit when triggering builds + // setup var triggerCount = 0 let client = MockClient { _, res in triggerCount += 1 @@ -600,88 +597,98 @@ class BuildTriggerTests: AppTestCase { } do { // fist run: we are at capacity and should not be triggering more builds - Current.getStatusCount = { _, _ in 300 } - - let pkgId = UUID() - let versionId = UUID() - let p = Package(id: pkgId, url: "1") - try await p.save(on: app.db) - try await Version(id: versionId, package: p, latest: .defaultBranch, reference: .branch("main")) - .save(on: app.db) - - // MUT - try await triggerBuilds(on: app.db, - client: client, - mode: .packageId(pkgId, force: false)) + try await withDependencies { + $0.buildSystem.getStatusCount = { @Sendable _, _ in 300 } + } operation: { + let pkgId = UUID() + let versionId = UUID() + let p = Package(id: pkgId, url: "1") + try await p.save(on: app.db) + try await Version(id: versionId, package: p, latest: .defaultBranch, reference: .branch("main")) + .save(on: app.db) - // validate - XCTAssertEqual(triggerCount, 0) - // ensure no build stubs have been created either - let v = try await Version.find(versionId, on: app.db) - try await v?.$builds.load(on: app.db) - XCTAssertEqual(v?.builds.count, 0) + // MUT + try await triggerBuilds(on: app.db, + client: client, + mode: .packageId(pkgId, force: false)) + + // validate + XCTAssertEqual(triggerCount, 0) + // ensure no build stubs have been created either + let v = try await Version.find(versionId, on: app.db) + try await v?.$builds.load(on: app.db) + XCTAssertEqual(v?.builds.count, 0) + } } triggerCount = 0 do { // second run: we are just below capacity and allow more builds to be triggered - Current.getStatusCount = { c, _ in 299 } - - let pkgId = UUID() - let versionId = UUID() - let p = Package(id: pkgId, url: "2") - try await p.save(on: app.db) - try await Version(id: versionId, package: p, latest: .defaultBranch, reference: .branch("main")) - .save(on: app.db) - - // MUT - try await triggerBuilds(on: app.db, - client: client, - mode: .packageId(pkgId, force: false)) + try await withDependencies { + $0.buildSystem.getStatusCount = { @Sendable _, _ in 299 } + } operation: { + let pkgId = UUID() + let versionId = UUID() + let p = Package(id: pkgId, url: "2") + try await p.save(on: app.db) + try await Version(id: versionId, package: p, latest: .defaultBranch, reference: .branch("main")) + .save(on: app.db) - // validate - XCTAssertEqual(triggerCount, 27) - // ensure builds are now in progress - let v = try await Version.find(versionId, on: app.db) - try await v?.$builds.load(on: app.db) - XCTAssertEqual(v?.builds.count, 27) + // MUT + try await triggerBuilds(on: app.db, + client: client, + mode: .packageId(pkgId, force: false)) + + // validate + XCTAssertEqual(triggerCount, 27) + // ensure builds are now in progress + let v = try await Version.find(versionId, on: app.db) + try await v?.$builds.load(on: app.db) + XCTAssertEqual(v?.builds.count, 27) + } } do { // third run: we are at capacity and using the `force` flag - Current.getStatusCount = { c, _ in 300 } - - var triggerCount = 0 - let client = MockClient { _, res in - triggerCount += 1 - try? res.content.encode( - Gitlab.Builder.Response.init(webUrl: "http://web_url") - ) - } - - let pkgId = UUID() - let versionId = UUID() - let p = Package(id: pkgId, url: "3") - try await p.save(on: app.db) - try await Version(id: versionId, package: p, latest: .defaultBranch, reference: .branch("main")) - .save(on: app.db) + try await withDependencies { + $0.buildSystem.getStatusCount = { @Sendable _, _ in 300 } + } operation: { + var triggerCount = 0 + let client = MockClient { _, res in + triggerCount += 1 + try? res.content.encode( + Gitlab.Builder.Response.init(webUrl: "http://web_url") + ) + } - // MUT - try await triggerBuilds(on: app.db, - client: client, - mode: .packageId(pkgId, force: true)) + let pkgId = UUID() + let versionId = UUID() + let p = Package(id: pkgId, url: "3") + try await p.save(on: app.db) + try await Version(id: versionId, package: p, latest: .defaultBranch, reference: .branch("main")) + .save(on: app.db) - // validate - XCTAssertEqual(triggerCount, 27) - // ensure builds are now in progress - let v = try await Version.find(versionId, on: app.db) - try await v?.$builds.load(on: app.db) - XCTAssertEqual(v?.builds.count, 27) + // MUT + try await triggerBuilds(on: app.db, + client: client, + mode: .packageId(pkgId, force: true)) + + // validate + XCTAssertEqual(triggerCount, 27) + // ensure builds are now in progress + let v = try await Version.find(versionId, on: app.db) + try await v?.$builds.load(on: app.db) + XCTAssertEqual(v?.builds.count, 27) + } } } } func test_triggerBuilds_multiplePackages() async throws { + let triggerCount = NIOLockedValueBox(0) try await withDependencies { + $0.buildSystem.getStatusCount = { @Sendable c, _ in + 299 + triggerCount.withLockedValue { $0 } + } $0.environment.allowBuildTriggers = { true } $0.environment.awsDocsBucket = { "awsDocsBucket" } $0.environment.builderToken = { "builder token" } @@ -689,16 +696,13 @@ class BuildTriggerTests: AppTestCase { $0.environment.buildTriggerAllowList = { [] } $0.environment.buildTriggerDownscaling = { 1 } $0.environment.buildTriggerLatestSwiftVersionDownscaling = { 1 } + $0.environment.gitlabPipelineLimit = { 300 } + $0.environment.gitlabPipelineToken = { "pipeline token" } $0.environment.random = { @Sendable _ in 0 } $0.environment.siteURL = { "http://example.com" } - } operation: { - // Ensure we respect the pipeline limit when triggering builds for multiple package ids - // setup - Current.gitlabPipelineToken = { "pipeline token" } - Current.gitlabPipelineLimit = { 300 } // Use live dependency but replace actual client with a mock so we can // assert on the details being sent without actually making a request - Current.triggerBuild = { client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in + $0.buildSystem.triggerBuild = { @Sendable client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in try await Gitlab.Builder.triggerBuild(client: client, buildId: buildId, cloneURL: cloneURL, @@ -708,14 +712,15 @@ class BuildTriggerTests: AppTestCase { swiftVersion: swiftVersion, versionID: versionID) } - let triggerCount = NIOLockedValueBox(0) + } operation: { + // Ensure we respect the pipeline limit when triggering builds for multiple package ids + // setup let client = MockClient { _, res in triggerCount.withLockedValue { $0 += 1 } try? res.content.encode( Gitlab.Builder.Response.init(webUrl: "http://web_url") ) } - Current.getStatusCount = { c, _ in 299 + triggerCount.withLockedValue { $0 } } let pkgIds = [UUID(), UUID()] for id in pkgIds { @@ -737,18 +742,19 @@ class BuildTriggerTests: AppTestCase { func test_triggerBuilds_trimming() async throws { try await withDependencies { + $0.buildSystem.getStatusCount = { @Sendable _, _ in 100 } $0.environment.allowBuildTriggers = { true } $0.environment.awsDocsBucket = { "awsDocsBucket" } $0.environment.builderToken = { "builder token" } $0.environment.buildTriggerAllowList = { [] } $0.environment.buildTriggerDownscaling = { 1 } + $0.environment.gitlabPipelineLimit = { 300 } + $0.environment.gitlabPipelineToken = { "pipeline token" } $0.environment.random = { @Sendable _ in 0 } $0.environment.siteURL = { "http://example.com" } } operation: { // Ensure we trim builds as part of triggering // setup - Current.gitlabPipelineToken = { "pipeline token" } - Current.gitlabPipelineLimit = { 300 } let client = MockClient { _, _ in } @@ -776,22 +782,20 @@ class BuildTriggerTests: AppTestCase { func test_triggerBuilds_error() async throws { try await withDependencies { + $0.buildSystem.getStatusCount = { @Sendable _, _ in 100 } $0.environment.allowBuildTriggers = { true } $0.environment.awsDocsBucket = { "awsDocsBucket" } $0.environment.builderToken = { "builder token" } $0.environment.buildTimeout = { 10 } $0.environment.buildTriggerAllowList = { [] } $0.environment.buildTriggerDownscaling = { 1 } + $0.environment.gitlabPipelineLimit = { 300 } + $0.environment.gitlabPipelineToken = { "pipeline token" } $0.environment.random = { @Sendable _ in 0 } $0.environment.siteURL = { "http://example.com" } - } operation: { - // Ensure we trim builds as part of triggering - // setup - Current.gitlabPipelineToken = { "pipeline token" } - Current.gitlabPipelineLimit = { 300 } // Use live dependency but replace actual client with a mock so we can // assert on the details being sent without actually making a request - Current.triggerBuild = { client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in + $0.buildSystem.triggerBuild = { @Sendable client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in try await Gitlab.Builder.triggerBuild(client: client, buildId: buildId, cloneURL: cloneURL, @@ -801,6 +805,9 @@ class BuildTriggerTests: AppTestCase { swiftVersion: swiftVersion, versionID: versionID) } + } operation: { + // Ensure we trim builds as part of triggering + // setup var triggerCount = 0 let client = MockClient { _, res in // let the 5th trigger succeed to ensure we don't early out on errors @@ -905,20 +912,19 @@ class BuildTriggerTests: AppTestCase { func test_override_switch() async throws { try await withDependencies { + $0.buildSystem.getStatusCount = { @Sendable _, _ in 100 } $0.environment.awsDocsBucket = { "awsDocsBucket" } $0.environment.builderToken = { "builder token" } $0.environment.buildTimeout = { 10 } $0.environment.buildTriggerAllowList = { [] } $0.environment.buildTriggerDownscaling = { 1 } + $0.environment.gitlabPipelineLimit = { Constants.defaultGitlabPipelineLimit } + $0.environment.gitlabPipelineToken = { "pipeline token" } $0.environment.random = { @Sendable _ in 0 } $0.environment.siteURL = { "http://example.com" } - } operation: { - // Ensure don't trigger if the override is off - // setup - Current.gitlabPipelineToken = { "pipeline token" } // Use live dependency but replace actual client with a mock so we can // assert on the details being sent without actually making a request - Current.triggerBuild = { client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in + $0.buildSystem.triggerBuild = { @Sendable client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in try await Gitlab.Builder.triggerBuild(client: client, buildId: buildId, cloneURL: cloneURL, @@ -928,6 +934,9 @@ class BuildTriggerTests: AppTestCase { swiftVersion: swiftVersion, versionID: versionID) } + } operation: { + // Ensure don't trigger if the override is off + // setup var triggerCount = 0 let client = MockClient { _, res in triggerCount += 1 @@ -982,20 +991,19 @@ class BuildTriggerTests: AppTestCase { func test_downscaling() async throws { try await withDependencies { + $0.buildSystem.getStatusCount = { @Sendable _, _ in 100 } $0.environment.allowBuildTriggers = { true } $0.environment.awsDocsBucket = { "awsDocsBucket" } $0.environment.builderToken = { "builder token" } $0.environment.buildTimeout = { 10 } $0.environment.buildTriggerAllowList = { [] } $0.environment.buildTriggerDownscaling = { 0.05 } // 5% downscaling rate + $0.environment.gitlabPipelineLimit = { Constants.defaultGitlabPipelineLimit } + $0.environment.gitlabPipelineToken = { "pipeline token" } $0.environment.siteURL = { "http://example.com" } - } operation: { - // Test build trigger downscaling behaviour - // setup - Current.gitlabPipelineToken = { "pipeline token" } // Use live dependency but replace actual client with a mock so we can // assert on the details being sent without actually making a request - Current.triggerBuild = { client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in + $0.buildSystem.triggerBuild = { @Sendable client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in try await Gitlab.Builder.triggerBuild(client: client, buildId: buildId, cloneURL: cloneURL, @@ -1005,6 +1013,9 @@ class BuildTriggerTests: AppTestCase { swiftVersion: swiftVersion, versionID: versionID) } + } operation: { + // Test build trigger downscaling behaviour + // setup var triggerCount = 0 let client = MockClient { _, res in triggerCount += 1 @@ -1059,20 +1070,19 @@ class BuildTriggerTests: AppTestCase { func test_downscaling_allow_list_override() async throws { try await withDependencies { + $0.buildSystem.getStatusCount = { @Sendable _, _ in 100 } $0.environment.allowBuildTriggers = { true } $0.environment.awsDocsBucket = { "awsDocsBucket" } $0.environment.builderToken = { "builder token" } $0.environment.buildTimeout = { 10 } $0.environment.buildTriggerAllowList = { [.id0] } $0.environment.buildTriggerDownscaling = { 0.05 } // 5% downscaling rate + $0.environment.gitlabPipelineLimit = { Constants.defaultGitlabPipelineLimit } + $0.environment.gitlabPipelineToken = { "pipeline token" } $0.environment.siteURL = { "http://example.com" } - } operation: { - // Test build trigger downscaling behaviour for allow-listed packages - // setup - Current.gitlabPipelineToken = { "pipeline token" } // Use live dependency but replace actual client with a mock so we can // assert on the details being sent without actually making a request - Current.triggerBuild = { client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in + $0.buildSystem.triggerBuild = { @Sendable client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in try await Gitlab.Builder.triggerBuild(client: client, buildId: buildId, cloneURL: cloneURL, @@ -1082,6 +1092,9 @@ class BuildTriggerTests: AppTestCase { swiftVersion: swiftVersion, versionID: versionID) } + } operation: { + // Test build trigger downscaling behaviour for allow-listed packages + // setup var triggerCount = 0 let client = MockClient { _, res in triggerCount += 1 diff --git a/Tests/AppTests/GitlabBuilderTests.swift b/Tests/AppTests/GitlabBuilderTests.swift index 2884dbcf8..ce311f662 100644 --- a/Tests/AppTests/GitlabBuilderTests.swift +++ b/Tests/AppTests/GitlabBuilderTests.swift @@ -58,9 +58,9 @@ class GitlabBuilderTests: AppTestCase { $0.environment.awsDocsBucket = { "docs-bucket" } $0.environment.builderToken = { "builder token" } $0.environment.buildTimeout = { 10 } + $0.environment.gitlabPipelineToken = { "pipeline token" } $0.environment.siteURL = { "http://example.com" } } operation: { - Current.gitlabPipelineToken = { "pipeline token" } let buildId = UUID() let versionID = UUID() @@ -107,10 +107,9 @@ class GitlabBuilderTests: AppTestCase { $0.environment.awsDocsBucket = { "docs-bucket" } $0.environment.builderToken = { "builder token" } $0.environment.buildTimeout = { 10 } + $0.environment.gitlabPipelineToken = { "pipeline token" } $0.environment.siteURL = { "http://example.com" } } operation: { - Current.gitlabPipelineToken = { "pipeline token" } - var called = false let client = MockClient { req, res in called = true @@ -137,32 +136,33 @@ class GitlabBuilderTests: AppTestCase { } func test_getStatusCount() async throws { - Current.gitlabApiToken = { "api token" } - Current.gitlabPipelineToken = { nil } - - var page = 1 - let client = MockClient { req, res in - XCTAssertEqual(req.url.string, "https://gitlab.com/api/v4/projects/19564054/pipelines?status=pending&page=\(page)&per_page=20") - res.status = .ok - let pending = #"{"id": 1, "status": "pending"}"# - switch page { - case 1: - let list = Array(repeating: pending, count: 20).joined(separator: ", ") - res.body = makeBody("[\(list)]") - case 2: - let list = Array(repeating: pending, count: 10).joined(separator: ", ") - res.body = makeBody("[\(list)]") - default: - XCTFail("unexpected page: \(page)") + try await withDependencies { + $0.environment.gitlabApiToken = { "api token" } + } operation: { + var page = 1 + let client = MockClient { req, res in + XCTAssertEqual(req.url.string, "https://gitlab.com/api/v4/projects/19564054/pipelines?status=pending&page=\(page)&per_page=20") + res.status = .ok + let pending = #"{"id": 1, "status": "pending"}"# + switch page { + case 1: + let list = Array(repeating: pending, count: 20).joined(separator: ", ") + res.body = makeBody("[\(list)]") + case 2: + let list = Array(repeating: pending, count: 10).joined(separator: ", ") + res.body = makeBody("[\(list)]") + default: + XCTFail("unexpected page: \(page)") + } + page += 1 } - page += 1 - } - let res = try await Gitlab.Builder.getStatusCount(client: client, - status: .pending, - pageSize: 20, - maxPageCount: 3) - XCTAssertEqual(res, 30) + let res = try await Gitlab.Builder.getStatusCount(client: client, + status: .pending, + pageSize: 20, + maxPageCount: 3) + XCTAssertEqual(res, 30) + } } } @@ -177,23 +177,21 @@ class LiveGitlabBuilderTests: AppTestCase { ) try await withDependencies { + // make sure environment variables are configured for live access + $0.environment.awsDocsBucket = { "spi-dev-docs" } $0.environment.builderToken = { // Set this to a valid value if you want to report build results back to the server ProcessInfo.processInfo.environment["LIVE_BUILDER_TOKEN"] } - // make sure environment variables are configured for live access - $0.environment.awsDocsBucket = { "spi-dev-docs" } + $0.environment.gitlabPipelineToken = { + // This Gitlab token is required in order to trigger the pipeline + ProcessInfo.processInfo.environment["LIVE_GITLAB_PIPELINE_TOKEN"] + } $0.environment.siteURL = { "https://staging.swiftpackageindex.com" } } operation: { // set build branch to trigger on Gitlab.Builder.branch = "main" - // make sure environment variables are configured for live access - Current.gitlabPipelineToken = { - // This Gitlab token is required in order to trigger the pipeline - ProcessInfo.processInfo.environment["LIVE_GITLAB_PIPELINE_TOKEN"] - } - let buildId = UUID() // use a valid uuid from a live db if reporting back should succeed diff --git a/Tests/AppTests/MetricsTests.swift b/Tests/AppTests/MetricsTests.swift index bef5f18b6..3280be75d 100644 --- a/Tests/AppTests/MetricsTests.swift +++ b/Tests/AppTests/MetricsTests.swift @@ -23,10 +23,13 @@ class MetricsTests: AppTestCase { func test_basic() async throws { try await withDependencies { + $0.buildSystem.triggerBuild = { @Sendable _, _, _, _, _, _, _, _ in + .init(status: .ok, webUrl: "") + } $0.environment.builderToken = { "builder token" } + $0.environment.gitlabPipelineToken = { "pipeline token" } } operation: { // setup - trigger build to increment counter - Current.gitlabPipelineToken = { "pipeline token" } let versionId = UUID() do { // save minimal package + version let p = Package(id: UUID(), url: "1") diff --git a/Tests/AppTests/Mocks/AppEnvironment+mock.swift b/Tests/AppTests/Mocks/AppEnvironment+mock.swift index e17d4fdc2..dd3aabb85 100644 --- a/Tests/AppTests/Mocks/AppEnvironment+mock.swift +++ b/Tests/AppTests/Mocks/AppEnvironment+mock.swift @@ -23,15 +23,10 @@ extension AppEnvironment { static func mock(eventLoop: EventLoop) -> Self { .init( fileManager: .mock, - getStatusCount: { _, _ in 100 }, git: .mock, - gitlabApiToken: { nil }, - gitlabPipelineToken: { nil }, - gitlabPipelineLimit: { Constants.defaultGitlabPipelineLimit }, logger: { logger }, setLogger: { logger in Self.logger = logger }, - shell: .mock, - triggerBuild: { _, _, _, _, _, _, _, _ in .init(status: .ok, webUrl: "http://web_url") } + shell: .mock ) } }