Skip to content

Commit 60f3a10

Browse files
cachemeifyoucancachemeifyoucan
committed
[CAS] Don't create the same CAS multiple times from the same Oracle
If the oracle has been used to create a CAS for the path, reuse the CAS if it is asked to create the same CAS again. This not only save the work of creating the CAS, but also avoid a potential problem that the CAS closing synchronization doesn't work across threads in the same process due to its uses of file lock.
1 parent 8e47ffe commit 60f3a10

File tree

6 files changed

+89
-22
lines changed

6 files changed

+89
-22
lines changed

‎Sources/SwiftDriver/Driver/Driver.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,7 @@ public struct Driver {
275275
let useClangIncludeTree: Bool
276276

277277
/// CAS instance used for compilation.
278-
var cas: SwiftScanCAS? = nil
278+
@_spi(Testing) public var cas: SwiftScanCAS? = nil
279279

280280
/// Is swift caching enabled.
281281
lazy var isCachingEnabled: Bool = {

‎Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -165,11 +165,26 @@ public class InterModuleDependencyOracle {
165165
return diags.isEmpty ? nil : diags
166166
}
167167

168-
public func createCAS(pluginPath: AbsolutePath?, onDiskPath: AbsolutePath?, pluginOptions: [(String, String)]) throws -> SwiftScanCAS {
168+
public func getOrCreateCAS(pluginPath: AbsolutePath?, onDiskPath: AbsolutePath?, pluginOptions: [(String, String)]) throws -> SwiftScanCAS {
169169
guard let swiftScan = swiftScanLibInstance else {
170170
fatalError("Attempting to reset scanner cache with no scanner instance.")
171171
}
172-
return try swiftScan.createCAS(pluginPath: pluginPath?.pathString, onDiskPath: onDiskPath?.pathString, pluginOptions: pluginOptions)
172+
// Use synchronized queue to avoid creating multiple OnDisk CAS at the same location as that will leave to synchronization issues.
173+
return try queue.sync {
174+
let casOpt = CASConfig(onDiskPath: onDiskPath, pluginPath: pluginPath, pluginOptions: pluginOptions)
175+
if let cas = createdCASMap[casOpt] {
176+
return cas
177+
}
178+
if let path = onDiskPath {
179+
guard !seenCASPath.contains(path) else {
180+
throw DependencyScanningError.casError("Cannot create two different CAS at the same OnDiskPath")
181+
}
182+
seenCASPath.insert(path)
183+
}
184+
let cas = try swiftScan.createCAS(pluginPath: pluginPath?.pathString, onDiskPath: onDiskPath?.pathString, pluginOptions: pluginOptions)
185+
createdCASMap[casOpt] = cas
186+
return cas
187+
}
173188
}
174189

175190
private var hasScannerInstance: Bool { self.swiftScanLibInstance != nil }
@@ -181,5 +196,31 @@ public class InterModuleDependencyOracle {
181196
private var swiftScanLibInstance: SwiftScan? = nil
182197

183198
internal let scannerRequiresPlaceholderModules: Bool
199+
200+
internal struct CASConfig: Hashable, Equatable {
201+
static func == (lhs: InterModuleDependencyOracle.CASConfig, rhs: InterModuleDependencyOracle.CASConfig) -> Bool {
202+
return lhs.onDiskPath == rhs.onDiskPath &&
203+
lhs.pluginPath == rhs.pluginPath &&
204+
lhs.pluginOptions.elementsEqual(rhs.pluginOptions, by: ==)
205+
}
206+
207+
func hash(into hasher: inout Hasher) {
208+
hasher.combine(onDiskPath)
209+
hasher.combine(pluginPath)
210+
for opt in pluginOptions {
211+
hasher.combine(opt.0)
212+
hasher.combine(opt.1)
213+
}
214+
}
215+
216+
let onDiskPath: AbsolutePath?
217+
let pluginPath: AbsolutePath?
218+
let pluginOptions: [(String, String)]
219+
}
220+
221+
/// Storing the CAS created via CASConfig.
222+
internal var createdCASMap: [CASConfig: SwiftScanCAS] = [:]
223+
/// The on disk path seen by CAS creation function.
224+
internal var seenCASPath: Set<AbsolutePath> = []
184225
}
185226

‎Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -173,9 +173,9 @@ public extension Driver {
173173
}
174174
}
175175
if !fallbackToFrontend && isCachingEnabled {
176-
self.cas = try interModuleDependencyOracle.createCAS(pluginPath: try getCASPluginPath(),
177-
onDiskPath: try getOnDiskCASPath(),
178-
pluginOptions: try getCASPluginOptions())
176+
self.cas = try interModuleDependencyOracle.getOrCreateCAS(pluginPath: try getCASPluginPath(),
177+
onDiskPath: try getOnDiskCASPath(),
178+
pluginOptions: try getCASPluginOptions())
179179
}
180180
return fallbackToFrontend
181181
}

‎Sources/SwiftDriver/SwiftScan/SwiftScanCAS.swift

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,12 @@ public final class SwiftScanCAS {
244244
}
245245
}
246246

247+
extension SwiftScanCAS: Equatable {
248+
static public func == (lhs: SwiftScanCAS, rhs: SwiftScanCAS) -> Bool {
249+
return lhs.cas == rhs.cas
250+
}
251+
}
252+
247253
extension swiftscan_cached_compilation_t {
248254
func convert(_ lib: SwiftScan) -> CachedCompilation {
249255
return CachedCompilation(self, lib: lib)

‎Tests/SwiftDriverTests/CachingBuildTests.swift

Lines changed: 32 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,8 @@ private func checkCachingBuildJobDependencies(job: Job,
205205

206206

207207
final class CachingBuildTests: XCTestCase {
208+
let dependencyOracle = InterModuleDependencyOracle()
209+
208210
override func setUpWithError() throws {
209211
try super.setUpWithError()
210212

@@ -499,12 +501,12 @@ final class CachingBuildTests: XCTestCase {
499501
"-cache-compile-job", "-cas-path", casPath.nativePathString(escaped: true),
500502
"-working-directory", path.nativePathString(escaped: true),
501503
main.nativePathString(escaped: true)] + sdkArgumentsForTesting,
502-
env: ProcessEnv.vars)
504+
env: ProcessEnv.vars,
505+
interModuleDependencyOracle: dependencyOracle)
503506
let jobs = try driver.planBuild()
504507
try driver.run(jobs: jobs)
505508
XCTAssertFalse(driver.diagnosticEngine.hasErrors)
506509

507-
let dependencyOracle = InterModuleDependencyOracle()
508510
let scanLibPath = try XCTUnwrap(driver.toolchain.lookupSwiftScanLib())
509511
guard try dependencyOracle
510512
.verifyOrCreateScannerInstance(fileSystem: localFileSystem,
@@ -513,7 +515,12 @@ final class CachingBuildTests: XCTestCase {
513515
return
514516
}
515517

516-
let cas = try dependencyOracle.createCAS(pluginPath: nil, onDiskPath: casPath, pluginOptions: [])
518+
let cas = try dependencyOracle.getOrCreateCAS(pluginPath: nil, onDiskPath: casPath, pluginOptions: [])
519+
if let driverCAS = driver.cas {
520+
XCTAssertEqual(cas, driverCAS, "CAS should only be created once")
521+
} else {
522+
XCTFail("Cached compilation doesn't have a CAS")
523+
}
517524
try checkCASForResults(jobs: jobs, cas: cas, fs: driver.fileSystem)
518525
}
519526
}
@@ -556,9 +563,9 @@ final class CachingBuildTests: XCTestCase {
556563
"-pch-output-dir", PCHPath.nativePathString(escaped: true),
557564
FooInstallPath.appending(component: "Foo.swiftmodule").nativePathString(escaped: true)]
558565
+ sdkArgumentsForTesting,
559-
env: ProcessEnv.vars)
566+
env: ProcessEnv.vars,
567+
interModuleDependencyOracle: dependencyOracle)
560568

561-
let dependencyOracle = InterModuleDependencyOracle()
562569
let scanLibPath = try XCTUnwrap(fooBuildDriver.toolchain.lookupSwiftScanLib())
563570
guard try dependencyOracle
564571
.verifyOrCreateScannerInstance(fileSystem: localFileSystem,
@@ -574,7 +581,12 @@ final class CachingBuildTests: XCTestCase {
574581
try fooBuildDriver.run(jobs: fooJobs)
575582
XCTAssertFalse(fooBuildDriver.diagnosticEngine.hasErrors)
576583

577-
let cas = try dependencyOracle.createCAS(pluginPath: nil, onDiskPath: casPath, pluginOptions: [])
584+
let cas = try dependencyOracle.getOrCreateCAS(pluginPath: nil, onDiskPath: casPath, pluginOptions: [])
585+
if let driverCAS = fooBuildDriver.cas {
586+
XCTAssertEqual(cas, driverCAS, "CAS should only be created once")
587+
} else {
588+
XCTFail("Cached compilation doesn't have a CAS")
589+
}
578590
try checkCASForResults(jobs: fooJobs, cas: cas, fs: fooBuildDriver.fileSystem)
579591

580592
var driver = try Driver(args: ["swiftc",
@@ -585,7 +597,8 @@ final class CachingBuildTests: XCTestCase {
585597
"-cache-compile-job", "-cas-path", casPath.nativePathString(escaped: true),
586598
"-working-directory", path.nativePathString(escaped: true),
587599
main.nativePathString(escaped: true)] + sdkArgumentsForTesting,
588-
env: ProcessEnv.vars)
600+
env: ProcessEnv.vars,
601+
interModuleDependencyOracle: dependencyOracle)
589602
// This is currently not supported.
590603
XCTAssertThrowsError(try driver.planBuild()) {
591604
XCTAssertEqual($0 as? Driver.Error, .unsupportedConfigurationForCaching("module Foo has prebuilt header dependency"))
@@ -623,8 +636,8 @@ final class CachingBuildTests: XCTestCase {
623636
"-Xcc", "-ivfsoverlay", "-Xcc", vfsoverlay.nativePathString(escaped: true),
624637
"-disable-clang-target",
625638
main.nativePathString(escaped: true)] + sdkArgumentsForTesting,
626-
env: ProcessEnv.vars)
627-
let dependencyOracle = InterModuleDependencyOracle()
639+
env: ProcessEnv.vars,
640+
interModuleDependencyOracle: dependencyOracle)
628641
let scanLibPath = try XCTUnwrap(driver.toolchain.lookupSwiftScanLib())
629642
guard try dependencyOracle
630643
.verifyOrCreateScannerInstance(fileSystem: localFileSystem,
@@ -760,11 +773,11 @@ final class CachingBuildTests: XCTestCase {
760773
"-scanner-prefix-map", testInputsPath.description + "=/^src",
761774
"-scanner-prefix-map", path.description + "=/^tmp",
762775
main.nativePathString(escaped: true)] + sdkArgumentsForTesting,
763-
env: ProcessEnv.vars)
776+
env: ProcessEnv.vars,
777+
interModuleDependencyOracle: dependencyOracle)
764778
guard driver.isFrontendArgSupported(.scannerPrefixMap) else {
765779
throw XCTSkip("frontend doesn't support prefix map")
766780
}
767-
let dependencyOracle = InterModuleDependencyOracle()
768781
let scanLibPath = try XCTUnwrap(driver.toolchain.lookupSwiftScanLib())
769782
guard try dependencyOracle
770783
.verifyOrCreateScannerInstance(fileSystem: localFileSystem,
@@ -830,12 +843,12 @@ final class CachingBuildTests: XCTestCase {
830843
"-output-file-map", ofm.nativePathString(escaped: true),
831844
"-working-directory", path.nativePathString(escaped: true),
832845
main.nativePathString(escaped: true)] + sdkArgumentsForTesting,
833-
env: ProcessEnv.vars)
846+
env: ProcessEnv.vars,
847+
interModuleDependencyOracle: dependencyOracle)
834848
let jobs = try driver.planBuild()
835849
try driver.run(jobs: jobs)
836850
XCTAssertFalse(driver.diagnosticEngine.hasErrors)
837851

838-
let dependencyOracle = InterModuleDependencyOracle()
839852
let scanLibPath = try XCTUnwrap(driver.toolchain.lookupSwiftScanLib())
840853
guard try dependencyOracle
841854
.verifyOrCreateScannerInstance(fileSystem: localFileSystem,
@@ -844,7 +857,12 @@ final class CachingBuildTests: XCTestCase {
844857
return
845858
}
846859

847-
let cas = try dependencyOracle.createCAS(pluginPath: nil, onDiskPath: casPath, pluginOptions: [])
860+
let cas = try dependencyOracle.getOrCreateCAS(pluginPath: nil, onDiskPath: casPath, pluginOptions: [])
861+
if let driverCAS = driver.cas {
862+
XCTAssertEqual(cas, driverCAS, "CAS should only be created once")
863+
} else {
864+
XCTFail("Cached compilation doesn't have a CAS")
865+
}
848866
try checkCASForResults(jobs: jobs, cas: cas, fs: driver.fileSystem)
849867

850868
// try replan the job and make sure some key command-line options are generated.

‎Tests/TestUtilities/DriverExtensions.swift

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ extension Driver {
2323
env: [String: String] = ProcessEnv.vars,
2424
diagnosticsEngine: DiagnosticsEngine = DiagnosticsEngine(handlers: [Driver.stderrDiagnosticsHandler]),
2525
fileSystem: FileSystem = localFileSystem,
26-
integratedDriver: Bool = true
26+
integratedDriver: Bool = true,
27+
interModuleDependencyOracle: InterModuleDependencyOracle? = nil
2728
) throws {
2829
let executor = try SwiftDriverExecutor(diagnosticsEngine: diagnosticsEngine,
2930
processSet: ProcessSet(),
@@ -34,7 +35,8 @@ extension Driver {
3435
diagnosticsOutput: .engine(diagnosticsEngine),
3536
fileSystem: fileSystem,
3637
executor: executor,
37-
integratedDriver: integratedDriver)
38+
integratedDriver: integratedDriver,
39+
interModuleDependencyOracle: interModuleDependencyOracle)
3840
}
3941

4042
/// For tests that need to set the sdk path.

0 commit comments

Comments
 (0)