Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions Sources/App/Commands/TriggerBuilds.swift
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,7 @@ func triggerBuilds(on database: Database,
packages: [Package.Id],
force: Bool = false) async throws {
@Dependency(\.environment) var environment
@Dependency(\.buildSystem) var buildSystem

guard environment.allowBuildTriggers() else {
Current.logger().info("Build trigger override switch OFF - no builds are being triggered")
Expand All @@ -196,15 +197,17 @@ func triggerBuilds(on database: Database,
}
}

async let pendingJobsTask = Current.getStatusCount(client, .pending)
async let runningJobsTask = Current.getStatusCount(client, .running)
let getStatusCount = buildSystem.getStatusCount
async let pendingJobsTask = getStatusCount(client, .pending)
async let runningJobsTask = getStatusCount(client, .running)
let pendingJobs = try await pendingJobsTask
let runningJobs = try await runningJobsTask

AppMetrics.buildPendingJobsCount?.set(pendingJobs)
AppMetrics.buildRunningJobsCount?.set(runningJobs)

let newJobs = ActorIsolated(0)
let gitlabPipelineLimit = environment.gitlabPipelineLimit

await withThrowingTaskGroup(of: Void.self) { group in
for pkgId in packages {
Expand All @@ -217,7 +220,7 @@ func triggerBuilds(on database: Database,
group.addTask {
// check if we have capacity to schedule more builds before querying for builds
var newJobCount = await newJobs.value
guard pendingJobs + newJobCount < Current.gitlabPipelineLimit() else {
guard pendingJobs + newJobCount < gitlabPipelineLimit() else {
Current.logger().info("too many pending pipelines (\(pendingJobs + newJobCount))")
return
}
Expand All @@ -226,7 +229,7 @@ func triggerBuilds(on database: Database,
let triggers = try await findMissingBuilds(database, packageId: pkgId)

newJobCount = await newJobs.value
guard pendingJobs + newJobCount < Current.gitlabPipelineLimit() else {
guard pendingJobs + newJobCount < gitlabPipelineLimit() else {
Current.logger().info("too many pending pipelines (\(pendingJobs + newJobCount))")
return
}
Expand Down
37 changes: 1 addition & 36 deletions Sources/App/Core/AppEnvironment.swift
Original file line number Diff line number Diff line change
Expand Up @@ -24,22 +24,10 @@ import FoundationNetworking

struct AppEnvironment: Sendable {
var fileManager: FileManager
var getStatusCount: @Sendable (_ client: Client, _ status: Gitlab.Builder.Status) async throws -> Int
var git: Git
var gitlabApiToken: @Sendable () -> String?
var gitlabPipelineToken: @Sendable () -> String?
var gitlabPipelineLimit: @Sendable () -> Int
var logger: @Sendable () -> Logger
var setLogger: @Sendable (Logger) -> Void
var shell: Shell
var triggerBuild: @Sendable (_ client: Client,
_ buildId: Build.Id,
_ cloneURL: String,
_ isDocBuild: Bool,
_ platform: Build.Platform,
_ reference: Reference,
_ swiftVersion: SwiftVersion,
_ versionID: Version.Id) async throws -> Build.TriggerResponse
}


Expand All @@ -48,33 +36,10 @@ extension AppEnvironment {

static let live = AppEnvironment(
fileManager: .live,
getStatusCount: { client, status in
try await Gitlab.Builder.getStatusCount(client: client,
status: status,
page: 1,
pageSize: 100,
maxPageCount: 5)
},
git: .live,
gitlabApiToken: { Environment.get("GITLAB_API_TOKEN") },
gitlabPipelineToken: { Environment.get("GITLAB_PIPELINE_TOKEN") },
gitlabPipelineLimit: {
Environment.get("GITLAB_PIPELINE_LIMIT").flatMap(Int.init)
?? Constants.defaultGitlabPipelineLimit
},
logger: { logger },
setLogger: { logger in Self.logger = logger },
shell: .live,
triggerBuild: { client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in
try await Gitlab.Builder.triggerBuild(client: client,
buildId: buildId,
cloneURL: cloneURL,
isDocBuild: isDocBuild,
platform: platform,
reference: ref,
swiftVersion: swiftVersion,
versionID: versionID)
}
shell: .live
)
}

Expand Down
73 changes: 73 additions & 0 deletions Sources/App/Core/Dependencies/BuildSystemClient.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
// Copyright Dave Verwer, Sven A. Schmidt, and other contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import Dependencies
import DependenciesMacros
import Vapor


@DependencyClient
struct BuildSystemClient {
#warning("remove client")

Check warning on line 22 in Sources/App/Core/Dependencies/BuildSystemClient.swift

View workflow job for this annotation

GitHub Actions / Query Performance Test

remove client

Check warning on line 22 in Sources/App/Core/Dependencies/BuildSystemClient.swift

View workflow job for this annotation

GitHub Actions / Query Performance Test

remove client

Check warning on line 22 in Sources/App/Core/Dependencies/BuildSystemClient.swift

View workflow job for this annotation

GitHub Actions / Query Performance Test

remove client

Check warning on line 22 in Sources/App/Core/Dependencies/BuildSystemClient.swift

View workflow job for this annotation

GitHub Actions / Test

remove client

Check warning on line 22 in Sources/App/Core/Dependencies/BuildSystemClient.swift

View workflow job for this annotation

GitHub Actions / Test

remove client

Check warning on line 22 in Sources/App/Core/Dependencies/BuildSystemClient.swift

View workflow job for this annotation

GitHub Actions / Test

remove client
var getStatusCount: @Sendable (_ client: Client, _ status: Gitlab.Builder.Status) async throws -> Int
#warning("remove client")

Check warning on line 24 in Sources/App/Core/Dependencies/BuildSystemClient.swift

View workflow job for this annotation

GitHub Actions / Query Performance Test

remove client

Check warning on line 24 in Sources/App/Core/Dependencies/BuildSystemClient.swift

View workflow job for this annotation

GitHub Actions / Query Performance Test

remove client

Check warning on line 24 in Sources/App/Core/Dependencies/BuildSystemClient.swift

View workflow job for this annotation

GitHub Actions / Test

remove client

Check warning on line 24 in Sources/App/Core/Dependencies/BuildSystemClient.swift

View workflow job for this annotation

GitHub Actions / Test

remove client

Check warning on line 24 in Sources/App/Core/Dependencies/BuildSystemClient.swift

View workflow job for this annotation

GitHub Actions / Test

remove client
var triggerBuild: @Sendable (_ client: Client,
_ buildId: Build.Id,
_ cloneURL: String,
_ isDocBuild: Bool,
_ platform: Build.Platform,
_ reference: Reference,
_ swiftVersion: SwiftVersion,
_ versionID: Version.Id) async throws -> Build.TriggerResponse
}


extension BuildSystemClient: DependencyKey {
static var liveValue: Self {
.init(
getStatusCount: { client, status in
try await Gitlab.Builder.getStatusCount(client: client,
status: status,
page: 1,
pageSize: 100,
maxPageCount: 5)
},
triggerBuild: { client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in
try await Gitlab.Builder.triggerBuild(client: client,
buildId: buildId,
cloneURL: cloneURL,
isDocBuild: isDocBuild,
platform: platform,
reference: ref,
swiftVersion: swiftVersion,
versionID: versionID)
}
)
}
}


extension BuildSystemClient: TestDependencyKey {
static var testValue: Self { Self() }
}


extension DependencyValues {
var buildSystem: BuildSystemClient {
get { self[BuildSystemClient.self] }
set { self[BuildSystemClient.self] = newValue }
}
}


9 changes: 9 additions & 0 deletions Sources/App/Core/Dependencies/EnvironmentClient.swift
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ struct EnvironmentClient {
var collectionSigningPrivateKey: @Sendable () -> Data?
var current: @Sendable () -> Environment = { XCTFail("current"); return .development }
var dbId: @Sendable () -> String?
var gitlabApiToken: @Sendable () -> String?
var gitlabPipelineLimit: @Sendable () -> Int = { XCTFail("gitlabPipelineLimit"); return 100 }
var gitlabPipelineToken: @Sendable () -> String?
var hideStagingBanner: @Sendable () -> Bool = { XCTFail("hideStagingBanner"); return Constants.defaultHideStagingBanner }
var loadSPIManifest: @Sendable (String) -> SPIManifest.Manifest?
var maintenanceMessage: @Sendable () -> String?
Expand Down Expand Up @@ -109,6 +112,12 @@ extension EnvironmentClient: DependencyKey {
},
current: { (try? Environment.detect()) ?? .development },
dbId: { Environment.get("DATABASE_ID") },
gitlabApiToken: { Environment.get("GITLAB_API_TOKEN") },
gitlabPipelineLimit: {
Environment.get("GITLAB_PIPELINE_LIMIT").flatMap(Int.init)
?? Constants.defaultGitlabPipelineLimit
},
gitlabPipelineToken: { Environment.get("GITLAB_PIPELINE_TOKEN") },
hideStagingBanner: {
Environment.get("HIDE_STAGING_BANNER").flatMap(\.asBool)
?? Constants.defaultHideStagingBanner
Expand Down
17 changes: 14 additions & 3 deletions Sources/App/Core/Gitlab.swift
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,10 @@ extension Gitlab.Builder {
versionID: Version.Id) async throws -> Build.TriggerResponse {
@Dependency(\.environment) var environment

guard let pipelineToken = Current.gitlabPipelineToken(),
guard let pipelineToken = environment.gitlabPipelineToken(),
let builderToken = environment.builderToken()
else { throw Gitlab.Error.missingToken }

guard let awsDocsBucket = environment.awsDocsBucket() else {
throw Gitlab.Error.missingConfiguration("AWS_DOCS_BUCKET")
}
Expand Down Expand Up @@ -157,10 +158,11 @@ extension Gitlab.Builder {
status: Status,
page: Int,
pageSize: Int = 20) async throws -> [Pipeline] {
guard let apiToken = Current.gitlabApiToken() else { throw Gitlab.Error.missingToken }
@Dependency(\.environment) var environment
guard let apiToken = environment.gitlabApiToken() else { throw Gitlab.Error.missingToken }

let uri: URI = .init(string: "\(projectURL)/pipelines?status=\(status)&page=\(page)&per_page=\(pageSize)")
let response = try await client.get(uri, headers: HTTPHeaders([("Authorization", "Bearer \(apiToken)")]))
let response = try await client.get(uri, headers: .bearer(apiToken))

guard response.status == .ok else { throw Gitlab.Error.requestFailed(response.status, uri) }

Expand Down Expand Up @@ -197,3 +199,12 @@ private extension DateFormatter {
return formatter
}
}


private extension HTTPHeaders {
static func bearer(_ token: String) -> Self {
.init([("Authorization", "Bearer \(token)")])
}
}


19 changes: 11 additions & 8 deletions Sources/App/Models/Build.swift
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

import Dependencies
import Fluent
import PostgresKit
import Vapor
Expand Down Expand Up @@ -186,14 +187,16 @@ extension Build {
.with(\.$package)
.first()
.unwrap(or: Abort(.notFound))
return try await Current.triggerBuild(client,
buildId,
version.package.url,
isDocBuild,
platform,
version.reference,
swiftVersion,
versionId)

@Dependency(\.buildSystem) var buildSystem
return try await buildSystem.triggerBuild(client,
buildId,
version.package.url,
isDocBuild,
platform,
version.reference,
swiftVersion,
versionId)
}

}
Expand Down
44 changes: 22 additions & 22 deletions Tests/AppTests/BuildTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -133,19 +133,11 @@ class BuildTests: AppTestCase {
$0.environment.awsDocsBucket = { "awsDocsBucket" }
$0.environment.builderToken = { "builder token" }
$0.environment.buildTimeout = { 10 }
$0.environment.gitlabPipelineToken = { "pipeline token" }
$0.environment.siteURL = { "http://example.com" }
} operation: {
Current.gitlabPipelineToken = { "pipeline token" }
// setup
let p = try await savePackage(on: app.db, "1")
let v = try Version(package: p, reference: .branch("main"))
try await v.save(on: app.db)
let buildId = UUID()
let versionID = try XCTUnwrap(v.id)

// Use live dependency but replace actual client with a mock so we can
// assert on the details being sent without actually making a request
Current.triggerBuild = { client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in
$0.buildSystem.triggerBuild = { @Sendable client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in
try await Gitlab.Builder.triggerBuild(client: client,
buildId: buildId,
cloneURL: cloneURL,
Expand All @@ -155,6 +147,14 @@ class BuildTests: AppTestCase {
swiftVersion: swiftVersion,
versionID: versionID)
}
} operation: {
// setup
let p = try await savePackage(on: app.db, "1")
let v = try Version(package: p, reference: .branch("main"))
try await v.save(on: app.db)
let buildId = UUID()
let versionID = try XCTUnwrap(v.id)

var called = false
let client = MockClient { req, res in
called = true
Expand Down Expand Up @@ -201,21 +201,11 @@ class BuildTests: AppTestCase {
$0.environment.awsDocsBucket = { "awsDocsBucket" }
$0.environment.builderToken = { "builder token" }
$0.environment.buildTimeout = { 10 }
$0.environment.gitlabPipelineToken = { "pipeline token" }
$0.environment.siteURL = { "http://example.com" }
} operation: {
// Same test as test_trigger above, except we trigger with isDocBuild: true
// and expect a 15m TIMEOUT instead of 10m
Current.gitlabPipelineToken = { "pipeline token" }
// setup
let p = try await savePackage(on: app.db, "1")
let v = try Version(package: p, reference: .branch("main"))
try await v.save(on: app.db)
let buildId = UUID()
let versionID = try XCTUnwrap(v.id)

// Use live dependency but replace actual client with a mock so we can
// assert on the details being sent without actually making a request
Current.triggerBuild = { client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in
$0.buildSystem.triggerBuild = { @Sendable client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in
try await Gitlab.Builder.triggerBuild(client: client,
buildId: buildId,
cloneURL: cloneURL,
Expand All @@ -225,6 +215,16 @@ class BuildTests: AppTestCase {
swiftVersion: swiftVersion,
versionID: versionID)
}
} operation: {
// Same test as test_trigger above, except we trigger with isDocBuild: true
// and expect a 15m TIMEOUT instead of 10m
// setup
let p = try await savePackage(on: app.db, "1")
let v = try Version(package: p, reference: .branch("main"))
try await v.save(on: app.db)
let buildId = UUID()
let versionID = try XCTUnwrap(v.id)

var called = false
let client = MockClient { req, res in
called = true
Expand Down
Loading
Loading