Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
546a641
Drop client parameter from getStatusCount
finestructure Jan 22, 2025
f38df7d
Drop client parameter from triggerBuild
finestructure Jan 22, 2025
a4f6237
Fix test_trigger
finestructure Jan 23, 2025
3de3ed0
Fix test_trigger_isDocBuild
finestructure Jan 23, 2025
a7828b8
Fix test_triggerBuild
finestructure Jan 23, 2025
078c517
Fix test_issue_588
finestructure Jan 23, 2025
b440c32
Fix test_triggerBuildsUnchecked
finestructure Jan 23, 2025
40351e1
Cleanup
finestructure Jan 24, 2025
98811c3
Fix test_triggerBuildsUnchecked_supported
finestructure Jan 24, 2025
0403d4c
Cleanup
finestructure Jan 24, 2025
000b797
Fix test_triggerBuildsUnchecked_build_exists
finestructure Jan 24, 2025
1fedd6f
Fix test_triggerBuilds_checked
finestructure Jan 24, 2025
a1d5796
Fix test_triggerBuilds_multiplePackages
finestructure Jan 24, 2025
97cecdc
Fix test_triggerBuilds_error
finestructure Jan 24, 2025
74d7a53
Fix test_override_switch
finestructure Jan 24, 2025
610fbb1
Cleanup
finestructure Jan 24, 2025
38eeec9
Fix test_downscaling
finestructure Jan 24, 2025
87956bc
Fix test_downscaling_allow_list_override
finestructure Jan 24, 2025
39b1fa7
Fix test_triggerBuild
finestructure Jan 24, 2025
1211802
Fix test_issue_588
finestructure Jan 24, 2025
f60ed03
Cleanup
finestructure Jan 24, 2025
ab2db51
Drop client from Build.trigger
finestructure Jan 24, 2025
110a6a4
Drop client from triggerBuildsUnchecked
finestructure Jan 24, 2025
121c9ca
Drop client from triggerBuilds
finestructure Jan 24, 2025
b48ebf6
Cleanup
finestructure Jan 24, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 9 additions & 24 deletions Sources/App/Commands/TriggerBuilds.swift
Original file line number Diff line number Diff line change
Expand Up @@ -85,9 +85,7 @@ struct TriggerBuildsCommand: AsyncCommand {
}

do {
try await triggerBuilds(on: context.application.db,
client: context.application.client,
mode: mode)
try await triggerBuilds(on: context.application.db, mode: mode)
} catch {
Current.logger().critical("\(error)")
}
Expand Down Expand Up @@ -122,9 +120,7 @@ extension TriggerBuildsCommand {
/// - client: `Client` used for http request
/// - parameter: `BuildTriggerCommand.Parameter` holding either a list of package ids
/// or a fetch limit for candidate selection.
func triggerBuilds(on database: Database,
client: Client,
mode: TriggerBuildsCommand.Mode) async throws {
func triggerBuilds(on database: Database, mode: TriggerBuildsCommand.Mode) async throws {
@Dependency(\.environment) var environment
let start = DispatchTime.now().uptimeNanoseconds

Expand All @@ -138,15 +134,12 @@ func triggerBuilds(on database: Database,
AppMetrics.buildCandidatesCount?.set(candidates.count)

let limitedCandidates = Array(candidates.prefix(limit))
try await triggerBuilds(on: database,
client: client,
packages: limitedCandidates)
try await triggerBuilds(on: database, packages: limitedCandidates)
AppMetrics.buildTriggerDurationSeconds?.time(since: start)

case let .packageId(id, force):
Current.logger().info("Triggering builds (packageID: \(id)) ...")
try await triggerBuilds(on: database,
client: client,
packages: [id],
force: force)
AppMetrics.buildTriggerDurationSeconds?.time(since: start)
Expand All @@ -159,9 +152,7 @@ func triggerBuilds(on database: Database,
Current.logger().error("Failed to create trigger.")
return
}
try await triggerBuildsUnchecked(on: database,
client: client,
triggers: [trigger])
try await triggerBuildsUnchecked(on: database, triggers: [trigger])

}
}
Expand All @@ -175,7 +166,6 @@ func triggerBuilds(on database: Database,
/// - packages: list of `Package.Id`s to trigger
/// - force: do not check pipeline capacity and ignore downscaling
func triggerBuilds(on database: Database,
client: Client,
packages: [Package.Id],
force: Bool = false) async throws {
@Dependency(\.environment) var environment
Expand All @@ -191,15 +181,15 @@ func triggerBuilds(on database: Database,
for package in packages {
group.addTask {
let triggerInfo = try await findMissingBuilds(database, packageId: package)
try await triggerBuildsUnchecked(on: database, client: client, triggers: triggerInfo)
try await triggerBuildsUnchecked(on: database, triggers: triggerInfo)
}
}
}
}

let getStatusCount = buildSystem.getStatusCount
async let pendingJobsTask = getStatusCount(client, .pending)
async let runningJobsTask = getStatusCount(client, .running)
async let pendingJobsTask = getStatusCount(.pending)
async let runningJobsTask = getStatusCount(.running)
let pendingJobs = try await pendingJobsTask
let runningJobs = try await runningJobsTask

Expand Down Expand Up @@ -237,9 +227,7 @@ func triggerBuilds(on database: Database,
let triggeredJobCount = triggers.reduce(0) { $0 + $1.buildPairs.count }
await newJobs.withValue { $0 += triggeredJobCount }

try await triggerBuildsUnchecked(on: database,
client: client,
triggers: triggers)
try await triggerBuildsUnchecked(on: database, triggers: triggers)
}
}
}
Expand All @@ -255,9 +243,7 @@ func triggerBuilds(on database: Database,
/// - database: `Database` handle used for database access
/// - client: `Client` used for http request
/// - triggers: trigger information for builds to trigger
func triggerBuildsUnchecked(on database: Database,
client: Client,
triggers: [BuildTriggerInfo]) async throws {
func triggerBuildsUnchecked(on database: Database, triggers: [BuildTriggerInfo]) async throws {
await withThrowingTaskGroup(of: Void.self) { group in
for trigger in triggers {
if let packageName = trigger.packageName, let reference = trigger.reference {
Expand All @@ -272,7 +258,6 @@ func triggerBuildsUnchecked(on database: Database,
let buildId = Build.Id()

let response = try await Build.trigger(database: database,
client: client,
buildId: buildId,
isDocBuild: trigger.docPairs.contains(pair),
platform: pair.platform,
Expand Down
18 changes: 6 additions & 12 deletions Sources/App/Core/Dependencies/BuildSystemClient.swift
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,12 @@

import Dependencies
import DependenciesMacros
import Vapor


@DependencyClient
struct BuildSystemClient {
#warning("remove client")
var getStatusCount: @Sendable (_ client: Client, _ status: Gitlab.Builder.Status) async throws -> Int
#warning("remove client")
var triggerBuild: @Sendable (_ client: Client,
_ buildId: Build.Id,
var getStatusCount: @Sendable (_ status: Gitlab.Builder.Status) async throws -> Int
var triggerBuild: @Sendable (_ buildId: Build.Id,
_ cloneURL: String,
_ isDocBuild: Bool,
_ platform: Build.Platform,
Expand All @@ -36,16 +32,14 @@ struct BuildSystemClient {
extension BuildSystemClient: DependencyKey {
static var liveValue: Self {
.init(
getStatusCount: { client, status in
try await Gitlab.Builder.getStatusCount(client: client,
status: status,
getStatusCount: { status in
try await Gitlab.Builder.getStatusCount(status: status,
page: 1,
pageSize: 100,
maxPageCount: 5)
},
triggerBuild: { client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in
try await Gitlab.Builder.triggerBuild(client: client,
buildId: buildId,
triggerBuild: { buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in
try await Gitlab.Builder.triggerBuild(buildId: buildId,
cloneURL: cloneURL,
isDocBuild: isDocBuild,
platform: platform,
Expand Down
9 changes: 9 additions & 0 deletions Sources/App/Core/Dependencies/HTTPClient.swift
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ extension HTTPClient: DependencyKey {
}

func get(url: String) async throws -> Response { try await get(url: url, headers: .init()) }
func post(url: String, body: Data?) async throws -> Response {
try await post(url: url, headers: .init(), body: body)
}
}


Expand Down Expand Up @@ -112,6 +115,7 @@ extension HTTPClient.Response {
static var notFound: Self { .init(status: .notFound) }
static var tooManyRequests: Self { .init(status: .tooManyRequests) }
static var ok: Self { .init(status: .ok) }
static var created: Self { .init(status: .created) }

static func ok(body: String, headers: HTTPHeaders = .init()) -> Self {
.init(status: .ok, headers: headers, body: .init(string: body))
Expand All @@ -121,5 +125,10 @@ extension HTTPClient.Response {
let data = try JSONEncoder().encode(value)
return .init(status: .ok, headers: headers, body: .init(data: data))
}

static func created<T: Encodable>(jsonEncode value: T, headers: HTTPHeaders = .init()) throws -> Self {
let data = try JSONEncoder().encode(value)
return .init(status: .created, headers: headers, body: .init(data: data))
}
}
#endif
88 changes: 48 additions & 40 deletions Sources/App/Core/Gitlab.swift
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,16 @@ enum Gitlab {
enum Error: LocalizedError {
case missingConfiguration(String)
case missingToken
case requestFailed(HTTPStatus, URI)
case noBody
case requestFailed(status: HTTPStatus, url: String)
}

static let decoder: JSONDecoder = {
static var decoder: JSONDecoder {
let d = JSONDecoder()
d.keyDecodingStrategy = .convertFromSnakeCase
d.dateDecodingStrategy = .formatted(DateFormatter.iso8601Full)
return d
}()
}

}

Expand Down Expand Up @@ -69,15 +70,15 @@ extension Gitlab.Builder {
}
}

static func triggerBuild(client: Client,
buildId: Build.Id,
static func triggerBuild(buildId: Build.Id,
cloneURL: String,
isDocBuild: Bool,
platform: Build.Platform,
reference: Reference,
swiftVersion: SwiftVersion,
versionID: Version.Id) async throws -> Build.TriggerResponse {
@Dependency(\.environment) var environment
@Dependency(\.httpClient) var httpClient

guard let pipelineToken = environment.gitlabPipelineToken(),
let builderToken = environment.builderToken()
Expand All @@ -88,31 +89,33 @@ extension Gitlab.Builder {
}
let timeout = environment.buildTimeout() + (isDocBuild ? 5 : 0)

let uri: URI = .init(string: "\(projectURL)/trigger/pipeline")
let response = try await client
.post(uri) { req in
let data = PostDTO(
token: pipelineToken,
ref: branch,
variables: [
"API_BASEURL": SiteURL.apiBaseURL,
"AWS_DOCS_BUCKET": awsDocsBucket,
"BUILD_ID": buildId.uuidString,
"BUILD_PLATFORM": platform.rawValue,
"BUILDER_TOKEN": builderToken,
"CLONE_URL": cloneURL,
"REFERENCE": "\(reference)",
"SWIFT_VERSION": "\(swiftVersion.major).\(swiftVersion.minor)",
"TIMEOUT": "\(timeout)m",
"VERSION_ID": versionID.uuidString
])
try req.query.encode(data)
}
let dto = PostDTO(
token: pipelineToken,
ref: branch,
variables: [
"API_BASEURL": SiteURL.apiBaseURL,
"AWS_DOCS_BUCKET": awsDocsBucket,
"BUILD_ID": buildId.uuidString,
"BUILD_PLATFORM": platform.rawValue,
"BUILDER_TOKEN": builderToken,
"CLONE_URL": cloneURL,
"REFERENCE": "\(reference)",
"SWIFT_VERSION": "\(swiftVersion.major).\(swiftVersion.minor)",
"TIMEOUT": "\(timeout)m",
"VERSION_ID": versionID.uuidString
]
)
let body = try URLEncodedFormEncoder().encode(dto)
let response = try await httpClient.post(
url: "\(projectURL)/trigger/pipeline",
headers: .contentTypeFormURLEncoded,
body: Data(body.utf8)
)

do {
let res = Build.TriggerResponse(
status: response.status,
webUrl: try response.content.decode(Response.self).webUrl
)
guard let body = response.body else { throw Gitlab.Error.noBody }
let webUrl = try JSONDecoder().decode(Response.self, from: body).webUrl
let res = Build.TriggerResponse(status: response.status, webUrl: webUrl)
Current.logger().info("Triggered build: \(res.webUrl)")
return res
} catch {
Expand Down Expand Up @@ -154,30 +157,31 @@ extension Gitlab.Builder {
}

// https://docs.gitlab.com/ee/api/pipelines.html
static func fetchPipelines(client: Client,
status: Status,
static func fetchPipelines(status: Status,
page: Int,
pageSize: Int = 20) async throws -> [Pipeline] {
@Dependency(\.environment) var environment
@Dependency(\.httpClient) var httpClient
guard let apiToken = environment.gitlabApiToken() else { throw Gitlab.Error.missingToken }
let url = "\(projectURL)/pipelines?status=\(status)&page=\(page)&per_page=\(pageSize)"

let uri: URI = .init(string: "\(projectURL)/pipelines?status=\(status)&page=\(page)&per_page=\(pageSize)")
let response = try await client.get(uri, headers: .bearer(apiToken))
let response = try await httpClient.get(url: url, headers: .bearer(apiToken))

guard response.status == .ok else { throw Gitlab.Error.requestFailed(response.status, uri) }
guard response.status == .ok else {
throw Gitlab.Error.requestFailed(status: response.status, url: url)
}
guard let body = response.body else { throw Gitlab.Error.noBody }

return try response.content.decode([Pipeline].self, using: Gitlab.decoder)
return try Gitlab.decoder.decode([Pipeline].self, from: body)
}

static func getStatusCount(client: Client,
status: Status,
static func getStatusCount(status: Status,
page: Int = 1,
pageSize: Int = 20,
maxPageCount: Int = 5) async throws -> Int {
let count = try await fetchPipelines(client: client, status: status, page: page, pageSize: pageSize).count
let count = try await fetchPipelines(status: status, page: page, pageSize: pageSize).count
if count == pageSize && page < maxPageCount {
let statusCount = try await getStatusCount(client: client,
status: status,
let statusCount = try await getStatusCount(status: status,
page: page + 1,
pageSize: pageSize,
maxPageCount: maxPageCount)
Expand Down Expand Up @@ -205,6 +209,10 @@ private extension HTTPHeaders {
static func bearer(_ token: String) -> Self {
.init([("Authorization", "Bearer \(token)")])
}

static var contentTypeFormURLEncoded: Self {
.init([("Content-Type", "application/x-www-form-urlencoded")])
}
}


4 changes: 1 addition & 3 deletions Sources/App/Models/Build.swift
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,6 @@ extension Build {
}

static func trigger(database: Database,
client: Client,
buildId: Build.Id,
isDocBuild: Bool,
platform: Build.Platform,
Expand All @@ -189,8 +188,7 @@ extension Build {
.unwrap(or: Abort(.notFound))

@Dependency(\.buildSystem) var buildSystem
return try await buildSystem.triggerBuild(client,
buildId,
return try await buildSystem.triggerBuild(buildId,
version.package.url,
isDocBuild,
platform,
Expand Down
Loading
Loading