Skip to content

Convert gitlab to async await #3328

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 23, 2024
Merged

Convert gitlab to async await #3328

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Sources/App/Commands/Swift6Trigger.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ extension Swift6TriggerCommand {
if force {
Current.logger().info("Skipping pending pipeline check")
} else {
let pendingJobs = try await Current.getStatusCount(client, .pending).get()
let pendingJobs = try await Current.getStatusCount(client, .pending)
guard pendingJobs + triggers.count < Current.gitlabPipelineLimit() else {
Current.logger().info("too many pending pipelines (\(pendingJobs))")
return
Expand Down
6 changes: 3 additions & 3 deletions Sources/App/Commands/TriggerBuilds.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,8 +191,8 @@ func triggerBuilds(on database: Database,
}
}

async let pendingJobsTask = Current.getStatusCount(client, .pending).get()
async let runningJobsTask = Current.getStatusCount(client, .running).get()
async let pendingJobsTask = Current.getStatusCount(client, .pending)
async let runningJobsTask = Current.getStatusCount(client, .running)
let pendingJobs = try await pendingJobsTask
let runningJobs = try await runningJobsTask

Expand Down Expand Up @@ -270,7 +270,7 @@ func triggerBuildsUnchecked(on database: Database,
isDocBuild: trigger.docPairs.contains(pair),
platform: pair.platform,
swiftVersion: pair.swiftVersion,
versionId: trigger.versionId).get()
versionId: trigger.versionId)
guard [HTTPStatus.ok, .created].contains(response.status),
let jobUrl = response.webUrl
else { return }
Expand Down
32 changes: 15 additions & 17 deletions Sources/App/Core/AppEnvironment.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,7 @@ struct AppEnvironment: Sendable {
var fetchReadme: @Sendable (_ client: Client, _ owner: String, _ repository: String) async -> Github.Readme?
var fetchS3Readme: @Sendable (_ client: Client, _ owner: String, _ repository: String) async throws -> String
var fileManager: FileManager
var getStatusCount: @Sendable (_ client: Client,
_ status: Gitlab.Builder.Status) -> EventLoopFuture<Int>
var getStatusCount: @Sendable (_ client: Client, _ status: Gitlab.Builder.Status) async throws -> Int
var git: Git
var githubToken: @Sendable () -> String?
var gitlabApiToken: @Sendable () -> String?
Expand Down Expand Up @@ -87,7 +86,7 @@ struct AppEnvironment: Sendable {
_ platform: Build.Platform,
_ reference: Reference,
_ swiftVersion: SwiftVersion,
_ versionID: Version.Id) -> EventLoopFuture<Build.TriggerResponse>
_ versionID: Version.Id) async throws -> Build.TriggerResponse
}


Expand Down Expand Up @@ -176,12 +175,11 @@ extension AppEnvironment {
fetchS3Readme: { client, owner, repo in try await S3Store.fetchReadme(client:client, owner: owner, repository: repo) },
fileManager: .live,
getStatusCount: { client, status in
Gitlab.Builder.getStatusCount(
client: client,
status: status,
page: 1,
pageSize: 100,
maxPageCount: 5)
try await Gitlab.Builder.getStatusCount(client: client,
status: status,
page: 1,
pageSize: 100,
maxPageCount: 5)
},
git: .live,
githubToken: { Environment.get("GITHUB_TOKEN") },
Expand Down Expand Up @@ -224,14 +222,14 @@ extension AppEnvironment {
storeS3ReadmeImages: { client, images in try await S3Store.storeReadmeImages(client: client, imagesToCache: images) },
timeZone: { .current },
triggerBuild: { client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in
Gitlab.Builder.triggerBuild(client: client,
buildId: buildId,
cloneURL: cloneURL,
isDocBuild: isDocBuild,
platform: platform,
reference: ref,
swiftVersion: swiftVersion,
versionID: versionID)
try await Gitlab.Builder.triggerBuild(client: client,
buildId: buildId,
cloneURL: cloneURL,
isDocBuild: isDocBuild,
platform: platform,
reference: ref,
swiftVersion: swiftVersion,
versionID: versionID)
}
)
}
Expand Down
82 changes: 34 additions & 48 deletions Sources/App/Core/Gitlab.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,17 +75,17 @@ extension Gitlab.Builder {
platform: Build.Platform,
reference: Reference,
swiftVersion: SwiftVersion,
versionID: Version.Id) -> EventLoopFuture<Build.TriggerResponse> {
versionID: Version.Id) async throws -> Build.TriggerResponse {
guard let pipelineToken = Current.gitlabPipelineToken(),
let builderToken = Current.builderToken()
else { return client.eventLoop.future(error: Gitlab.Error.missingToken) }
else { throw Gitlab.Error.missingToken }
guard let awsDocsBucket = Current.awsDocsBucket() else {
return client.eventLoop.future(error: Gitlab.Error.missingConfiguration("AWS_DOCS_BUCKET"))
throw Gitlab.Error.missingConfiguration("AWS_DOCS_BUCKET")
}
let timeout = Current.buildTimeout() + (isDocBuild ? 5 : 0)

let uri: URI = .init(string: "\(projectURL)/trigger/pipeline")
let req = client
let response = try await client
.post(uri) { req in
let data = PostDTO(
token: pipelineToken,
Expand All @@ -104,19 +104,17 @@ extension Gitlab.Builder {
])
try req.query.encode(data)
}
return req.map { response in
do {
let res = Build.TriggerResponse(
status: response.status,
webUrl: try response.content.decode(Response.self).webUrl
)
Current.logger().info("Triggered build: \(res.webUrl)")
return res
} catch {
let body = response.body?.asString() ?? "nil"
Current.logger().error("Trigger failed: \(cloneURL) @ \(reference), \(platform) / \(swiftVersion), \(versionID), status: \(response.status), body: \(body)")
return .init(status: response.status, webUrl: nil)
}
do {
let res = Build.TriggerResponse(
status: response.status,
webUrl: try response.content.decode(Response.self).webUrl
)
Current.logger().info("Triggered build: \(res.webUrl)")
return res
} catch {
let body = response.body?.asString() ?? "nil"
Current.logger().error("Trigger failed: \(cloneURL) @ \(reference), \(platform) / \(swiftVersion), \(versionID), status: \(response.status), body: \(body)")
return .init(status: response.status, webUrl: nil)
}
}

Expand Down Expand Up @@ -155,45 +153,33 @@ extension Gitlab.Builder {
static func fetchPipelines(client: Client,
status: Status,
page: Int,
pageSize: Int = 20) -> EventLoopFuture<[Pipeline]> {
guard let apiToken = Current.gitlabApiToken()
else { return client.eventLoop.future(error: Gitlab.Error.missingToken) }
pageSize: Int = 20) async throws -> [Pipeline] {
guard let apiToken = Current.gitlabApiToken() else { throw Gitlab.Error.missingToken }

let uri: URI = .init(string: "\(projectURL)/pipelines?status=\(status)&page=\(page)&per_page=\(pageSize)")
return client
.get(uri, headers: HTTPHeaders([("Authorization", "Bearer \(apiToken)")]))
.flatMap { response -> EventLoopFuture<[Pipeline]> in
guard response.status == .ok else {
return client.eventLoop.future(error: Gitlab.Error.requestFailed(response.status, uri))
}
do {
let res = try response.content.decode([Pipeline].self, using: Gitlab.decoder)
return client.eventLoop.future(res)
} catch {
return client.eventLoop.future(error: error)
}
}
let response = try await client.get(uri, headers: HTTPHeaders([("Authorization", "Bearer \(apiToken)")]))

guard response.status == .ok else { throw Gitlab.Error.requestFailed(response.status, uri) }

return try response.content.decode([Pipeline].self, using: Gitlab.decoder)
}

static func getStatusCount(client: Client,
status: Status,
page: Int = 1,
pageSize: Int = 20,
maxPageCount: Int = 5) -> EventLoopFuture<Int> {
fetchPipelines(client: client, status: status, page: page, pageSize: pageSize)
.map(\.count)
.flatMap { count -> EventLoopFuture<Int> in
if count == pageSize && page < maxPageCount {
return getStatusCount(client: client,
status: status,
page: page + 1,
pageSize: pageSize,
maxPageCount: maxPageCount)
.map { count + $0 }
} else {
return client.eventLoop.future(count)
}
}
maxPageCount: Int = 5) async throws -> Int {
let count = try await fetchPipelines(client: client, status: status, page: page, pageSize: pageSize).count
if count == pageSize && page < maxPageCount {
let statusCount = try await getStatusCount(client: client,
status: status,
page: page + 1,
pageSize: pageSize,
maxPageCount: maxPageCount)
return count + statusCount
} else {
return count
}
}

}
Expand Down
22 changes: 10 additions & 12 deletions Sources/App/Models/Build.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,23 +179,21 @@ extension Build {
isDocBuild: Bool,
platform: Build.Platform,
swiftVersion: SwiftVersion,
versionId: Version.Id) -> EventLoopFuture<TriggerResponse> {
let version: EventLoopFuture<Version> = Version
versionId: Version.Id) async throws -> TriggerResponse {
let version = try await Version
.query(on: database)
.filter(\.$id == versionId)
.with(\.$package)
.first()
.unwrap(or: Abort(.notFound))
return version.flatMap {
return Current.triggerBuild(client,
buildId,
$0.package.url,
isDocBuild,
platform,
$0.reference,
swiftVersion,
versionId)
}
return try await Current.triggerBuild(client,
buildId,
version.package.url,
isDocBuild,
platform,
version.reference,
swiftVersion,
versionId)
}

}
Expand Down
Loading
Loading