diff --git a/Sources/CSwiftScan/include/swiftscan_header.h b/Sources/CSwiftScan/include/swiftscan_header.h index 1409fba5f..79ea98bd4 100644 --- a/Sources/CSwiftScan/include/swiftscan_header.h +++ b/Sources/CSwiftScan/include/swiftscan_header.h @@ -18,7 +18,7 @@ #include #define SWIFTSCAN_VERSION_MAJOR 0 -#define SWIFTSCAN_VERSION_MINOR 4 +#define SWIFTSCAN_VERSION_MINOR 5 //=== Public Scanner Data Types -------------------------------------------===// @@ -79,6 +79,7 @@ typedef void *swiftscan_scanner_t; //=== CAS/Caching Specification -------------------------------------------===// typedef struct swiftscan_cas_s *swiftscan_cas_t; +typedef struct swiftscan_cas_options_s *swiftscan_cas_options_t; typedef enum { SWIFTSCAN_OUTPUT_TYPE_OBJECT = 0, @@ -274,15 +275,24 @@ typedef struct { void (*swiftscan_scanner_cache_reset)(swiftscan_scanner_t scanner); //=== Scanner CAS Operations ----------------------------------------------===// - swiftscan_cas_t (*swiftscan_cas_create)(const char *path); + swiftscan_cas_options_t (*swiftscan_cas_options_create)(void); + void (*swiftscan_cas_options_dispose)(swiftscan_cas_options_t options); + void (*swiftscan_cas_options_set_ondisk_path)(swiftscan_cas_options_t options, + const char *path); + void (*swiftscan_cas_options_set_plugin_path)(swiftscan_cas_options_t options, + const char *path); + bool (*swiftscan_cas_options_set_option)(swiftscan_cas_options_t options, + const char *name, const char *value, + swiftscan_string_ref_t *error); + swiftscan_cas_t (*swiftscan_cas_create_from_options)( + swiftscan_cas_options_t options, swiftscan_string_ref_t *error); void (*swiftscan_cas_dispose)(swiftscan_cas_t cas); swiftscan_string_ref_t (*swiftscan_cas_store)(swiftscan_cas_t cas, - uint8_t *data, unsigned size); - swiftscan_string_ref_t (*swiftscan_compute_cache_key)(swiftscan_cas_t cas, - int argc, - const char *argv, - const char *input, - swiftscan_output_kind_t); + uint8_t *data, unsigned size, + swiftscan_string_ref_t *error); + swiftscan_string_ref_t (*swiftscan_compute_cache_key)( + swiftscan_cas_t cas, int argc, const char *argv, const char *input, + swiftscan_output_kind_t, swiftscan_string_ref_t *error); } swiftscan_functions_t; diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift index 21af10e04..31e430b57 100644 --- a/Sources/SwiftDriver/Driver/Driver.swift +++ b/Sources/SwiftDriver/Driver/Driver.swift @@ -270,7 +270,6 @@ public struct Driver { /// CAS/Caching related options. let enableCaching: Bool let useClangIncludeTree: Bool - let casPath: String /// Code & data for incremental compilation. Nil if not running in incremental mode. /// Set during planning because needs the jobs to look at outputs. @@ -595,13 +594,6 @@ public struct Driver { let cachingEnableOverride = parsedOptions.hasArgument(.driverExplicitModuleBuild) && env.keys.contains("SWIFT_ENABLE_CACHING") self.enableCaching = parsedOptions.hasArgument(.cacheCompileJob) || cachingEnableOverride self.useClangIncludeTree = enableCaching && env.keys.contains("SWIFT_CACHING_USE_INCLUDE_TREE") - if let casPathOpt = parsedOptions.getLastArgument(.casPath)?.asSingle { - self.casPath = casPathOpt.description - } else if let cacheEnv = env["CCHROOT"] { - self.casPath = cacheEnv - } else { - self.casPath = "" - } // Compute the working directory. workingDirectory = try parsedOptions.getLastArgument(.workingDirectory).map { workingDirectoryArg in @@ -3470,3 +3462,32 @@ extension Driver { return try VirtualPath.intern(path: moduleName.appendingFileTypeExtension(type)) } } + +// CAS and Caching. +extension Driver { + mutating func getCASPluginPath() throws -> AbsolutePath? { + if let pluginOpt = parsedOptions.getLastArgument(.casPluginOption)?.asSingle { + return try AbsolutePath(validating: pluginOpt.description) + } + return try toolchain.lookupToolchainCASPluginLib() + } + + mutating func getOnDiskCASPath() throws -> AbsolutePath? { + if let casPathOpt = parsedOptions.getLastArgument(.casPath)?.asSingle { + return try AbsolutePath(validating: casPathOpt.description) + } + return nil; + } + + mutating func getCASPluginOptions() throws -> [(String, String)] { + var options : [(String, String)] = [] + for opt in parsedOptions.arguments(for: .casPluginOption) { + let pluginArg = opt.argument.asSingle.split(separator: "=", maxSplits: 1) + if pluginArg.count != 2 { + throw Error.invalidArgumentValue(Option.casPluginOption.spelling, opt.argument.asSingle) + } + options.append((String(pluginArg[0]), String(pluginArg[1]))) + } + return options + } +} diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift index 29ffd97ea..6065ab71f 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift @@ -171,11 +171,11 @@ public class InterModuleDependencyOracle { return diags.isEmpty ? nil : diags } - public func createCAS(path: String) throws { + public func createCAS(pluginPath: AbsolutePath?, onDiskPath: AbsolutePath?, pluginOptions: [(String, String)]) throws { guard let swiftScan = swiftScanLibInstance else { fatalError("Attempting to reset scanner cache with no scanner instance.") } - try swiftScan.createCAS(casPath: path) + try swiftScan.createCAS(pluginPath: pluginPath?.pathString, onDiskPath: onDiskPath?.pathString, pluginOptions: pluginOptions) } public func store(data: Data) throws -> String { diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift index b0971c81d..a12c1e14f 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift @@ -167,7 +167,9 @@ public extension Driver { } } if !fallbackToFrontend && enableCaching { - try interModuleDependencyOracle.createCAS(path: casPath) + try interModuleDependencyOracle.createCAS(pluginPath: try getCASPluginPath(), + onDiskPath: try getOnDiskCASPath(), + pluginOptions: try getCASPluginOptions()) } return fallbackToFrontend } diff --git a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift index 4d2a8adef..519029afa 100644 --- a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift +++ b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift @@ -336,13 +336,16 @@ extension Driver { // CAS related options. if enableCaching { commandLine.appendFlag(.cacheCompileJob) - if !casPath.isEmpty { + if let casPath = try getOnDiskCASPath() { commandLine.appendFlag(.casPath) - commandLine.appendFlag(casPath) + commandLine.appendFlag(casPath.pathString) + } + if let pluginPath = try getCASPluginPath() { + commandLine.appendFlag(.casPluginPath) + commandLine.appendFlag(pluginPath.pathString) } - try commandLine.appendLast(.cacheRemarks, from: &parsedOptions) - try commandLine.appendLast(.casPluginPath, from: &parsedOptions) try commandLine.appendAll(.casPluginOption, from: &parsedOptions) + try commandLine.appendLast(.cacheRemarks, from: &parsedOptions) } if useClangIncludeTree { commandLine.appendFlag(.clangIncludeTree) diff --git a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift index 939f07e0a..a77cb1227 100644 --- a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift +++ b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift @@ -25,7 +25,7 @@ public enum DependencyScanningError: Error, DiagnosticData { case missingRequiredSymbol(String) case dependencyScanFailed case failedToInstantiateScanner - case failedToInstantiateCAS + case casError(String) case missingField(String) case moduleNameDecodeFailure(String) case unsupportedDependencyDetailsKind(Int) @@ -42,8 +42,8 @@ public enum DependencyScanningError: Error, DiagnosticData { return "libSwiftScan dependency scan query failed" case .failedToInstantiateScanner: return "libSwiftScan failed to create scanner instance" - case .failedToInstantiateCAS: - return "libSwiftScan failed to create CAS" + case .casError(let reason): + return "libSwiftScan CAS error: \(reason)" case .missingField(let fieldName): return "libSwiftScan scan result missing required field: `\(fieldName)`" case .moduleNameDecodeFailure(let encodedName): @@ -285,7 +285,12 @@ internal extension swiftscan_diagnostic_severity_t { // Caching is currently not supported on Windows hosts. return false #else - return api.swiftscan_cas_create != nil && + return api.swiftscan_cas_options_create != nil && + api.swiftscan_cas_options_dispose != nil && + api.swiftscan_cas_options_set_ondisk_path != nil && + api.swiftscan_cas_options_set_plugin_path != nil && + api.swiftscan_cas_options_set_option != nil && + api.swiftscan_cas_create_from_options != nil && api.swiftscan_cas_dispose != nil && api.swiftscan_compute_cache_key != nil && api.swiftscan_cas_store != nil && @@ -387,20 +392,48 @@ internal extension swiftscan_diagnostic_severity_t { } } - func createCAS(casPath: String) throws { - self.cas = api.swiftscan_cas_create(casPath.cString(using: String.Encoding.utf8)) - guard self.cas != nil else { - throw DependencyScanningError.failedToInstantiateCAS + private func handleCASError(_ closure: (inout swiftscan_string_ref_t) -> Bool) throws { + var err_msg : swiftscan_string_ref_t = swiftscan_string_ref_t() + guard !closure(&err_msg) else { + let err_str = try toSwiftString(err_msg) + api.swiftscan_string_dispose(err_msg) + throw DependencyScanningError.casError(err_str) + } + } + + func createCAS(pluginPath: String?, onDiskPath: String?, pluginOptions: [(String, String)]) throws { + let casOpts = api.swiftscan_cas_options_create() + defer { + api.swiftscan_cas_options_dispose(casOpts) + } + if let path = pluginPath { + api.swiftscan_cas_options_set_plugin_path(casOpts, path) + } + if let path = onDiskPath { + api.swiftscan_cas_options_set_ondisk_path(casOpts, path) + } + for (name, value) in pluginOptions { + try handleCASError { err_msg in + return api.swiftscan_cas_options_set_option(casOpts, name, value, &err_msg) + } + } + try handleCASError { err_msg in + self.cas = api.swiftscan_cas_create_from_options(casOpts, &err_msg) + return self.cas == nil } } func store(data: Data) throws -> String { guard let scan_cas = self.cas else { - throw DependencyScanningError.failedToInstantiateCAS + throw DependencyScanningError.casError("cannot store into CAS because CAS is not yet created") } let bytes = UnsafeMutablePointer.allocate(capacity: data.count) data.copyBytes(to: bytes, count: data.count) - let casid = api.swiftscan_cas_store(scan_cas, bytes, UInt32(data.count)) + var casid: swiftscan_string_ref_t = swiftscan_string_ref_t() + try handleCASError { err_msg in + casid = api.swiftscan_cas_store(scan_cas, bytes, UInt32(data.count), &err_msg) + return casid.data == nil + } return try toSwiftString(casid) } @@ -425,15 +458,19 @@ internal extension swiftscan_diagnostic_severity_t { func computeCacheKeyForOutput(kind: FileType, commandLine: [String], input: String) throws -> String { guard let scan_cas = self.cas else { - throw DependencyScanningError.failedToInstantiateCAS + throw DependencyScanningError.casError("cannot compute CacheKey for compilation because CAS is not yet created") } - var casid : swiftscan_string_ref_t = swiftscan_string_ref_t() - withArrayOfCStrings(commandLine) { commandArray in - casid = api.swiftscan_compute_cache_key(scan_cas, - Int32(commandLine.count), - commandArray, - input.cString(using: String.Encoding.utf8), - getSwiftScanOutputKind(kind: kind)) + var casid: swiftscan_string_ref_t = swiftscan_string_ref_t() + try handleCASError { err_msg in + withArrayOfCStrings(commandLine) { commandArray in + casid = api.swiftscan_compute_cache_key(scan_cas, + Int32(commandLine.count), + commandArray, + input.cString(using: String.Encoding.utf8), + getSwiftScanOutputKind(kind: kind), + &err_msg) + } + return casid.data == nil } return try toSwiftString(casid) } @@ -518,7 +555,12 @@ private extension swiftscan_functions_t { self.swiftscan_clang_detail_get_module_cache_key = try loadOptional("swiftscan_clang_detail_get_module_cache_key") - self.swiftscan_cas_create = try loadOptional("swiftscan_cas_create") + self.swiftscan_cas_options_create = try loadOptional("swiftscan_cas_options_create") + self.swiftscan_cas_options_set_plugin_path = try loadOptional("swiftscan_cas_options_set_plugin_path") + self.swiftscan_cas_options_set_ondisk_path = try loadOptional("swiftscan_cas_options_set_ondisk_path") + self.swiftscan_cas_options_set_option = try loadOptional("swiftscan_cas_options_set_option") + self.swiftscan_cas_options_dispose = try loadOptional("swiftscan_cas_options_dispose") + self.swiftscan_cas_create_from_options = try loadOptional("swiftscan_cas_create_from_options") self.swiftscan_cas_dispose = try loadOptional("swiftscan_cas_dispose") self.swiftscan_compute_cache_key = try loadOptional("swiftscan_compute_cache_key") diff --git a/Sources/SwiftDriver/Toolchains/Toolchain.swift b/Sources/SwiftDriver/Toolchains/Toolchain.swift index a629063f5..c5fdad5c8 100644 --- a/Sources/SwiftDriver/Toolchains/Toolchain.swift +++ b/Sources/SwiftDriver/Toolchains/Toolchain.swift @@ -284,6 +284,35 @@ extension Toolchain { #endif } + /// Looks for the executable in the `SWIFT_DRIVER_TOOLCHAIN_CASPLUGIN_LIB` environment variable, if found nothing, + /// looks in the `lib` relative to the compiler executable. + @_spi(Testing) public func lookupToolchainCASPluginLib() throws -> AbsolutePath? { + if let overrideString = env["SWIFT_DRIVER_TOOLCHAIN_CASPLUGIN_LIB"], + let path = try? AbsolutePath(validating: overrideString) { + return path + } +#if os(Windows) + return nil +#else + // Try to look for libToolchainCASPlugin in the developer dir, if found, + // prefer using that. Otherwise, just return nil, and auto fallback to + // builtin CAS. + let libraryName = sharedLibraryName("libToolchainCASPlugin") + let compilerPath = try getToolPath(.swiftCompiler) + let developerPath = compilerPath.parentDirectory // bin + .parentDirectory // toolchain root + .parentDirectory // toolchains + .parentDirectory // developer + let libraryPath = developerPath.appending(component: "usr") + .appending(component: "lib") + .appending(component: libraryName) + if fileSystem.isFile(libraryPath) { + return libraryPath + } + return nil +#endif + } + private func xcrunFind(executable: String) throws -> AbsolutePath { let xcrun = "xcrun" guard lookupExecutablePath(filename: xcrun, searchPaths: searchPaths) != nil else {