Skip to content

Commit 225bcbf

Browse files
dschaefer2dschaefer2
authored
Set candidate URL for prebuilts download. (#8246)
I was using my fork of swift-syntax to host my test artifacts. Switch that to a more realistic yet not yet approved one. Also cuts down how often we reach out to fetch a manifest. We instead create an empty one in the scratch directory if it's not found on the download site. And we always now check the scratch directory first.
1 parent 979610a commit 225bcbf

File tree

4 files changed

+103
-64
lines changed

4 files changed

+103
-64
lines changed

‎Sources/Basics/Concurrency/SendableBox.swift

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,24 +16,24 @@ import struct Foundation.Date
1616
/// an `async` closure. This type serves as a replacement for `ThreadSafeBox`
1717
/// implemented with Swift Concurrency primitives.
1818
public actor SendableBox<Value: Sendable> {
19-
init(_ value: Value? = nil) {
19+
public init(_ value: Value) {
2020
self.value = value
2121
}
2222

23-
var value: Value?
23+
public var value: Value
24+
25+
public func set(_ value: Value) {
26+
self.value = value
27+
}
2428
}
2529

2630
extension SendableBox where Value == Int {
2731
func increment() {
28-
if let value {
29-
self.value = value + 1
30-
}
32+
self.value = value + 1
3133
}
3234

3335
func decrement() {
34-
if let value {
35-
self.value = value - 1
36-
}
36+
self.value = value - 1
3737
}
3838
}
3939

‎Sources/Workspace/Workspace+Prebuilts.swift

Lines changed: 59 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,7 @@ extension Workspace {
169169
private let cachePath: AbsolutePath?
170170
private let delegate: Delegate?
171171
private let hashAlgorithm: HashAlgorithm = SHA256()
172+
private let prebuiltsDownloadURL: URL
172173

173174
init(
174175
fileSystem: FileSystem,
@@ -187,6 +188,11 @@ extension Workspace {
187188
self.scratchPath = scratchPath
188189
self.cachePath = cachePath
189190
self.delegate = delegate
191+
if let prebuiltsDownloadURL, let url = URL(string: prebuiltsDownloadURL) {
192+
self.prebuiltsDownloadURL = url
193+
} else {
194+
self.prebuiltsDownloadURL = URL(string: "https://download.swift.org/prebuilts")!
195+
}
190196

191197
self.prebuiltPackages = [
192198
// TODO: we should have this in a manifest somewhere, not hardcoded like this
@@ -202,17 +208,13 @@ extension Workspace {
202208
kind: .remoteSourceControl("[email protected]:swiftlang/swift-syntax.git")
203209
),
204210
],
205-
prebuiltsURL: URL(
206-
string: prebuiltsDownloadURL ?? "https://github.com/dschaefer2/swift-syntax/releases/download"
207-
)!
208211
),
209212
]
210213
}
211214

212215
struct PrebuiltPackage {
213216
let identity: PackageIdentity
214217
let packageRefs: [PackageReference]
215-
let prebuiltsURL: URL
216218
}
217219

218220
private let prebuiltPackages: [PrebuiltPackage]
@@ -260,13 +262,11 @@ extension Workspace {
260262
observabilityScope: ObservabilityScope
261263
) async throws -> PrebuiltsManifest? {
262264
let manifestFile = swiftVersion + "-manifest.json"
263-
let prebuiltsDir = cachePath ?? scratchPath
264-
let destination = prebuiltsDir.appending(components:
265-
package.identity.description,
266-
version.description,
267-
manifestFile
268-
)
269-
if fileSystem.exists(destination) {
265+
let manifestPath = try RelativePath(validating: "\(package.identity)/\(version)/\(manifestFile)")
266+
let destination = scratchPath.appending(manifestPath)
267+
let cacheDest = cachePath?.appending(manifestPath)
268+
269+
func loadManifest() throws -> PrebuiltsManifest? {
270270
do {
271271
return try JSONDecoder().decode(
272272
path: destination,
@@ -280,21 +280,41 @@ extension Workspace {
280280
underlyingError: error
281281
)
282282
try fileSystem.removeFileTree(destination)
283+
return nil
283284
}
284285
}
285-
try fileSystem.createDirectory(
286-
destination.parentDirectory,
287-
recursive: true
288-
)
289286

290-
let manifestURL = package.prebuiltsURL.appending(
291-
components: version.description,
292-
manifestFile
287+
if fileSystem.exists(destination), let manifest = try? loadManifest() {
288+
return manifest
289+
} else if let cacheDest, fileSystem.exists(cacheDest) {
290+
// Pull it out of the cache
291+
try fileSystem.createDirectory(destination.parentDirectory, recursive: true)
292+
try fileSystem.copy(from: cacheDest, to: destination)
293+
294+
if let manifest = try? loadManifest() {
295+
return manifest
296+
}
297+
}
298+
299+
try fileSystem.createDirectory(destination.parentDirectory, recursive: true)
300+
301+
let manifestURL = self.prebuiltsDownloadURL.appending(
302+
components: package.identity.description, version.description, manifestFile
293303
)
294304

295305
if manifestURL.scheme == "file" {
296-
// simply copy it over
297-
try fileSystem.copy(from: AbsolutePath(validating: manifestURL.path), to: destination)
306+
let sourcePath = try AbsolutePath(validating: manifestURL.path)
307+
if fileSystem.exists(sourcePath) {
308+
// simply copy it over
309+
try fileSystem.copy(from: sourcePath, to: destination)
310+
// and cache it
311+
if let cacheDest {
312+
try fileSystem.createDirectory(cacheDest.parentDirectory, recursive: true)
313+
try fileSystem.copy(from: destination, to: cacheDest)
314+
}
315+
} else {
316+
return nil
317+
}
298318
} else {
299319
var headers = HTTPClientHeaders()
300320
headers.add(name: "Accept", value: "application/json")
@@ -321,21 +341,21 @@ extension Workspace {
321341
info: "Prebuilt \(manifestFile)",
322342
underlyingError: error
323343
)
344+
// Create an empty manifest so we don't keep trying to download it
345+
let manifest = PrebuiltsManifest(libraries: [])
346+
try? fileSystem.writeFileContents(destination, data: JSONEncoder().encode(manifest))
324347
return nil
325348
}
326349
}
327350

328-
do {
329-
return try JSONDecoder().decode(
330-
path: destination,
331-
fileSystem: fileSystem,
332-
as: PrebuiltsManifest.self
333-
)
334-
} catch {
335-
observabilityScope.emit(
336-
info: "Failed to decode prebuilt manifest",
337-
underlyingError: error
338-
)
351+
if let manifest = try loadManifest() {
352+
// Cache the manifest
353+
if let cacheDest {
354+
try fileSystem.createDirectory(cacheDest.parentDirectory, recursive: true)
355+
try fileSystem.copy(from: destination, to: cacheDest)
356+
}
357+
return manifest
358+
} else {
339359
return nil
340360
}
341361
}
@@ -384,14 +404,18 @@ extension Workspace {
384404
)
385405

386406
// Download
387-
let artifactURL = package.prebuiltsURL.appending(
388-
components: version.description,
389-
artifactFile
407+
let artifactURL = self.prebuiltsDownloadURL.appending(
408+
components: package.identity.description, version.description, artifactFile
390409
)
391410

392411
let fetchStart = DispatchTime.now()
393412
if artifactURL.scheme == "file" {
394-
try fileSystem.copy(from: AbsolutePath(validating: artifactURL.path), to: destination)
413+
let artifactPath = try AbsolutePath(validating: artifactURL.path)
414+
if fileSystem.exists(artifactPath) {
415+
try fileSystem.copy(from: artifactPath, to: destination)
416+
} else {
417+
return nil
418+
}
395419
} else {
396420
var headers = HTTPClientHeaders()
397421
headers.add(name: "Accept", value: "application/octet-stream")

‎Tests/BasicsTests/HTTPClientTests.swift

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -227,15 +227,15 @@ final class HTTPClientTests: XCTestCase {
227227

228228
func testExponentialBackoff() async throws {
229229
let counter = SendableBox(0)
230-
let lastCall = SendableBox<Date>()
230+
let lastCall = SendableBox<Date>(Date())
231231
let maxAttempts = 5
232232
let errorCode = Int.random(in: 500 ..< 600)
233233
let delay = SendableTimeInterval.milliseconds(100)
234234

235235
let httpClient = HTTPClient { _, _ in
236-
let count = await counter.value!
236+
let count = await counter.value
237237
let expectedDelta = pow(2.0, Double(count - 1)) * delay.timeInterval()!
238-
let delta = await lastCall.value.flatMap { Date().timeIntervalSince($0) } ?? 0
238+
let delta = await Date().timeIntervalSince(lastCall.value)
239239
XCTAssertEqual(delta, expectedDelta, accuracy: 0.1)
240240

241241
await counter.increment()
@@ -404,7 +404,7 @@ final class HTTPClientTests: XCTestCase {
404404
let httpClient = HTTPClient(configuration: configuration) { request, _ in
405405
await concurrentRequests.increment()
406406

407-
if await concurrentRequests.value! > maxConcurrentRequests {
407+
if await concurrentRequests.value > maxConcurrentRequests {
408408
XCTFail("too many concurrent requests \(concurrentRequests), expected \(maxConcurrentRequests)")
409409
}
410410

‎Tests/WorkspaceTests/PrebuiltsTests.swift

Lines changed: 32 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -134,10 +134,10 @@ final class PrebuiltsTests: XCTestCase {
134134
throw StringError("invalid request \(request.kind)")
135135
}
136136

137-
if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-manifest.json" {
137+
if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-manifest.json" {
138138
try fileSystem.writeFileContents(destination, data: manifestData)
139139
return .okay()
140-
} else if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" {
140+
} else if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" {
141141
try fileSystem.writeFileContents(destination, data: artifact)
142142
return .okay()
143143
} else {
@@ -191,17 +191,17 @@ final class PrebuiltsTests: XCTestCase {
191191
throw StringError("invalid request \(request.kind)")
192192
}
193193

194-
if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-manifest.json" {
194+
if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-manifest.json" {
195195
try fileSystem.writeFileContents(destination, data: manifestData)
196196
return .okay()
197-
} else if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" {
197+
} else if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" {
198198
try fileSystem.writeFileContents(destination, data: artifact)
199199
return .okay()
200200
} else {
201201
// make sure it's the updated one
202202
XCTAssertEqual(
203203
request.url,
204-
"https://github.com/dschaefer2/swift-syntax/releases/download/601.0.0/\(self.swiftVersion)-manifest.json"
204+
"https://download.swift.org/prebuilts/swift-syntax/601.0.0/\(self.swiftVersion)-manifest.json"
205205
)
206206
return .notFound()
207207
}
@@ -295,10 +295,10 @@ final class PrebuiltsTests: XCTestCase {
295295
throw StringError("invalid request \(request.kind)")
296296
}
297297

298-
if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-manifest.json" {
298+
if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-manifest.json" {
299299
try fileSystem.writeFileContents(destination, data: manifestData)
300300
return .okay()
301-
} else if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" {
301+
} else if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" {
302302
try fileSystem.writeFileContents(destination, data: artifact)
303303
return .okay()
304304
} else {
@@ -355,10 +355,10 @@ final class PrebuiltsTests: XCTestCase {
355355
throw StringError("invalid request \(request.kind)")
356356
}
357357

358-
if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-manifest.json" {
358+
if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-manifest.json" {
359359
try fileSystem.writeFileContents(destination, data: manifestData)
360360
return .okay()
361-
} else if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" {
361+
} else if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" {
362362
XCTFail("Unexpect download of archive")
363363
try fileSystem.writeFileContents(destination, data: artifact)
364364
return .okay()
@@ -407,8 +407,12 @@ final class PrebuiltsTests: XCTestCase {
407407
let artifact = Data()
408408
let (_, rootPackage, swiftSyntax) = try initData(artifact: artifact, swiftSyntaxVersion: "600.0.2")
409409

410+
let secondFetch = SendableBox(false)
411+
410412
let httpClient = HTTPClient { request, progressHandler in
411-
if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.2/\(self.swiftVersion)-manifest.json" {
413+
if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.2/\(self.swiftVersion)-manifest.json" {
414+
let secondFetch = await secondFetch.value
415+
XCTAssertFalse(secondFetch, "unexpected second fetch")
412416
return .notFound()
413417
} else {
414418
XCTFail("Unexpected URL \(request.url)")
@@ -444,6 +448,17 @@ final class PrebuiltsTests: XCTestCase {
444448
try checkSettings(rootPackage, "Foo", usePrebuilt: false)
445449
try checkSettings(rootPackage, "FooClient", usePrebuilt: false)
446450
}
451+
452+
await secondFetch.set(true)
453+
454+
try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in
455+
XCTAssertTrue(diagnostics.filter({ $0.severity == .error }).isEmpty)
456+
let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first)
457+
try checkSettings(rootPackage, "FooMacros", usePrebuilt: false)
458+
try checkSettings(rootPackage, "FooTests", usePrebuilt: false)
459+
try checkSettings(rootPackage, "Foo", usePrebuilt: false)
460+
try checkSettings(rootPackage, "FooClient", usePrebuilt: false)
461+
}
447462
}
448463

449464
func testUnsupportedArch() async throws {
@@ -459,7 +474,7 @@ final class PrebuiltsTests: XCTestCase {
459474
throw StringError("invalid request \(request.kind)")
460475
}
461476

462-
if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-manifest.json" {
477+
if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-manifest.json" {
463478
try fileSystem.writeFileContents(destination, data: manifestData)
464479
return .okay()
465480
} else {
@@ -507,7 +522,7 @@ final class PrebuiltsTests: XCTestCase {
507522
let (_, rootPackage, swiftSyntax) = try initData(artifact: artifact, swiftSyntaxVersion: "600.0.1")
508523

509524
let httpClient = HTTPClient { request, progressHandler in
510-
if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-manifest.json" {
525+
if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-manifest.json" {
511526
// Pretend it's a different swift version
512527
return .notFound()
513528
} else {
@@ -561,10 +576,10 @@ final class PrebuiltsTests: XCTestCase {
561576
throw StringError("invalid request \(request.kind)")
562577
}
563578

564-
if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-manifest.json" {
579+
if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-manifest.json" {
565580
try fileSystem.writeFileContents(destination, data: manifestData)
566581
return .okay()
567-
} else if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" {
582+
} else if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" {
568583
try fileSystem.writeFileContents(destination, data: fakeArtifact)
569584
return .okay()
570585
} else {
@@ -621,10 +636,10 @@ final class PrebuiltsTests: XCTestCase {
621636
throw StringError("invalid request \(request.kind)")
622637
}
623638

624-
if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-manifest.json" {
639+
if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-manifest.json" {
625640
try fileSystem.writeFileContents(destination, data: manifestData)
626641
return .okay()
627-
} else if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" {
642+
} else if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" {
628643
try fileSystem.writeFileContents(destination, data: artifact)
629644
return .okay()
630645
} else {
@@ -678,7 +693,7 @@ final class PrebuiltsTests: XCTestCase {
678693
throw StringError("invalid request \(request.kind)")
679694
}
680695

681-
if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-manifest.json" {
696+
if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-manifest.json" {
682697
let badManifestData = manifestData + Data("bad".utf8)
683698
try fileSystem.writeFileContents(destination, data: badManifestData)
684699
return .okay()

0 commit comments

Comments
 (0)