diff --git a/Sources/SWBBuildSystem/BuildOperation.swift b/Sources/SWBBuildSystem/BuildOperation.swift index 5542c89e..69b8be28 100644 --- a/Sources/SWBBuildSystem/BuildOperation.swift +++ b/Sources/SWBBuildSystem/BuildOperation.swift @@ -419,7 +419,7 @@ package final class BuildOperation: BuildSystemOperation { } // Perform any needed steps before we kick off the build. - if let (warnings, errors) = prepareForBuilding() { + if let (warnings, errors) = await prepareForBuilding() { // Emit any warnings and errors. If there were any errors, then bail out. for message in warnings { buildOutputDelegate.warning(message) } for message in errors { buildOutputDelegate.error(message) } @@ -809,7 +809,7 @@ package final class BuildOperation: BuildSystemOperation { return delegate.buildComplete(self, status: effectiveStatus, delegate: buildOutputDelegate, metrics: .init(counters: aggregatedCounters)) } - func prepareForBuilding() -> ([String], [String])? { + func prepareForBuilding() async -> ([String], [String])? { let warnings = [String]() // Not presently used var errors = [String]() @@ -829,9 +829,61 @@ package final class BuildOperation: BuildSystemOperation { } } + if UserDefaults.enableCASValidation { + for info in buildDescription.casValidationInfos { + do { + try await validateCAS(info) + } catch { + errors.append("cas validation failed for \(info.options.casPath.str)") + } + } + } + return (warnings.count > 0 || errors.count > 0) ? (warnings, errors) : nil } + func validateCAS(_ info: BuildDescription.CASValidationInfo) async throws { + assert(UserDefaults.enableCASValidation) + + let casPath = info.options.casPath + let ruleInfo = "ValidateCAS \(casPath.str) \(info.llvmCasExec.str)" + + let signatureCtx = InsecureHashContext() + signatureCtx.add(string: "ValidateCAS") + signatureCtx.add(string: casPath.str) + signatureCtx.add(string: info.llvmCasExec.str) + let signature = signatureCtx.signature + + let activityId = delegate.beginActivity(self, ruleInfo: ruleInfo, executionDescription: "Validate CAS contents at \(casPath.str)", signature: signature, target: nil, parentActivity: nil) + var status: BuildOperationTaskEnded.Status = .failed + defer { + delegate.endActivity(self, id: activityId, signature: signature, status: status) + } + + var commandLine = [ + info.llvmCasExec.str, + "-cas", casPath.str, + "-validate-if-needed", + "-check-hash", + "-allow-recovery", + ] + if let pluginPath = info.options.pluginPath { + commandLine.append(contentsOf: [ + "-fcas-plugin-path", pluginPath.str + ]) + } + let result: Processes.ExecutionResult = try await clientDelegate.executeExternalTool(commandLine: commandLine) + // In a task we might use a discovered tool info to detect if the tool supports validation, but without that scaffolding, just check the specific error. + if result.exitStatus == .exit(1) && result.stderr.contains(ByteString("Unknown command line argument '-validate-if-needed'")) { + delegate.emit(data: ByteString("validation not supported").bytes, for: activityId, signature: signature) + status = .succeeded + } else { + delegate.emit(data: ByteString(result.stderr).bytes, for: activityId, signature: signature) + delegate.emit(data: ByteString(result.stdout).bytes, for: activityId, signature: signature) + status = result.exitStatus.isSuccess ? .succeeded : result.exitStatus.wasCanceled ? .cancelled : .failed + } + } + /// Cancel the executing build operation. package func cancel() { queue.blocking_sync() { diff --git a/Sources/SWBCore/Settings/BuiltinMacros.swift b/Sources/SWBCore/Settings/BuiltinMacros.swift index 1ca2ea8e..1461bd51 100644 --- a/Sources/SWBCore/Settings/BuiltinMacros.swift +++ b/Sources/SWBCore/Settings/BuiltinMacros.swift @@ -1126,6 +1126,7 @@ public final class BuiltinMacros { public static let TAPI_HEADER_SEARCH_PATHS = BuiltinMacros.declarePathListMacro("TAPI_HEADER_SEARCH_PATHS") public static let USE_HEADER_SYMLINKS = BuiltinMacros.declareBooleanMacro("USE_HEADER_SYMLINKS") public static let USE_HIERARCHICAL_LAYOUT_FOR_COPIED_ASIDE_PRODUCTS = BuiltinMacros.declareBooleanMacro("USE_HIERARCHICAL_LAYOUT_FOR_COPIED_ASIDE_PRODUCTS") + public static let VALIDATE_CAS_EXEC = BuiltinMacros.declareStringMacro("VALIDATE_CAS_EXEC") public static let VALIDATE_PLIST_FILES_WHILE_COPYING = BuiltinMacros.declareBooleanMacro("VALIDATE_PLIST_FILES_WHILE_COPYING") public static let VALIDATE_PRODUCT = BuiltinMacros.declareBooleanMacro("VALIDATE_PRODUCT") public static let VALIDATE_DEPENDENCIES = BuiltinMacros.declareEnumMacro("VALIDATE_DEPENDENCIES") as EnumMacroDeclaration @@ -2327,6 +2328,7 @@ public final class BuiltinMacros { USE_HEADER_SYMLINKS, USE_HIERARCHICAL_LAYOUT_FOR_COPIED_ASIDE_PRODUCTS, VALIDATE_PLIST_FILES_WHILE_COPYING, + VALIDATE_CAS_EXEC, VALIDATE_PRODUCT, VALIDATE_DEPENDENCIES, VALIDATE_DEVELOPMENT_ASSET_PATHS, diff --git a/Sources/SWBCore/TaskGeneration.swift b/Sources/SWBCore/TaskGeneration.swift index 1539cd82..216ed173 100644 --- a/Sources/SWBCore/TaskGeneration.swift +++ b/Sources/SWBCore/TaskGeneration.swift @@ -768,7 +768,7 @@ extension CoreClientTargetDiagnosticProducingDelegate { private let externalToolExecutionQueue = AsyncOperationQueue(concurrentTasks: ProcessInfo.processInfo.activeProcessorCount) extension CoreClientDelegate { - func executeExternalTool(commandLine: [String], workingDirectory: String? = nil, environment: [String: String] = [:]) async throws -> Processes.ExecutionResult { + package func executeExternalTool(commandLine: [String], workingDirectory: String? = nil, environment: [String: String] = [:]) async throws -> Processes.ExecutionResult { switch try await executeExternalTool(commandLine: commandLine, workingDirectory: workingDirectory, environment: environment) { case .deferred: guard let url = commandLine.first.map(URL.init(fileURLWithPath:)) else { diff --git a/Sources/SWBTaskExecution/BuildDescription.swift b/Sources/SWBTaskExecution/BuildDescription.swift index a7de2774..419e0187 100644 --- a/Sources/SWBTaskExecution/BuildDescription.swift +++ b/Sources/SWBTaskExecution/BuildDescription.swift @@ -141,6 +141,16 @@ package final class BuildDescription: Serializable, Sendable, Encodable, Cacheab private let rootPathsPerTarget: [ConfiguredTarget: [Path]] private let moduleCachePathsPerTarget: [ConfiguredTarget: [Path]] + /// A description of a CAS for validation, including how it is configured + /// and which llvm-cas should be used to validate it. + package struct CASValidationInfo { + package var options: CASOptions + package var llvmCasExec: Path + } + + /// The list of all CAS directories for validation. + package let casValidationInfos: [CASValidationInfo] + private let dependencyValidationPerTarget: [ConfiguredTarget: BooleanWarningLevel] /// The map of used in-process classes. @@ -193,13 +203,14 @@ package final class BuildDescription: Serializable, Sendable, Encodable, Cacheab package let emitFrontendCommandLines: Bool /// Load a build description from the given path. - fileprivate init(inDir dir: Path, signature: BuildDescriptionSignature, taskStore: FrozenTaskStore, allOutputPaths: Set, rootPathsPerTarget: [ConfiguredTarget: [Path]], moduleCachePathsPerTarget: [ConfiguredTarget: [Path]], settingsPerTarget: [ConfiguredTarget: Settings], enableStaleFileRemoval: Bool = true, taskActionMap: [String: TaskAction.Type], targetTaskCounts: [ConfiguredTarget: Int], moduleSessionFilePath: Path?, diagnostics: [ConfiguredTarget?: [Diagnostic]], fs: any FSProxy, invalidationPaths: [Path], recursiveSearchPathResults: [RecursiveSearchPathResolver.CachedResult], copiedPathMap: [String: String], targetDependencies: [TargetDependencyRelationship], definingTargetsByModuleName: [String: OrderedSet], capturedBuildInfo: CapturedBuildInfo?, bypassActualTasks: Bool, targetsBuildInParallel: Bool, emitFrontendCommandLines: Bool) throws { + fileprivate init(inDir dir: Path, signature: BuildDescriptionSignature, taskStore: FrozenTaskStore, allOutputPaths: Set, rootPathsPerTarget: [ConfiguredTarget: [Path]], moduleCachePathsPerTarget: [ConfiguredTarget: [Path]], casValidationInfos: [CASValidationInfo], settingsPerTarget: [ConfiguredTarget: Settings], enableStaleFileRemoval: Bool = true, taskActionMap: [String: TaskAction.Type], targetTaskCounts: [ConfiguredTarget: Int], moduleSessionFilePath: Path?, diagnostics: [ConfiguredTarget?: [Diagnostic]], fs: any FSProxy, invalidationPaths: [Path], recursiveSearchPathResults: [RecursiveSearchPathResolver.CachedResult], copiedPathMap: [String: String], targetDependencies: [TargetDependencyRelationship], definingTargetsByModuleName: [String: OrderedSet], capturedBuildInfo: CapturedBuildInfo?, bypassActualTasks: Bool, targetsBuildInParallel: Bool, emitFrontendCommandLines: Bool) throws { self.dir = dir self.signature = signature self.taskStore = taskStore self.allOutputPaths = allOutputPaths self.rootPathsPerTarget = rootPathsPerTarget self.moduleCachePathsPerTarget = moduleCachePathsPerTarget + self.casValidationInfos = casValidationInfos self.dependencyValidationPerTarget = settingsPerTarget.mapValues { $0.globalScope.evaluate(BuiltinMacros.VALIDATE_DEPENDENCIES) } self.taskActionMap = taskActionMap self.targetTaskCounts = targetTaskCounts @@ -320,7 +331,7 @@ package final class BuildDescription: Serializable, Sendable, Encodable, Cacheab package func serialize(to serializer: T) { guard serializer.delegate is BuildDescriptionSerializerDelegate else { fatalError("delegate must be a BuildDescriptionSerializerDelegate") } - serializer.beginAggregate(19) + serializer.beginAggregate(20) serializer.serialize(dir) serializer.serialize(signature) // Serialize the tasks first so we can index into this array during deserialization. @@ -352,6 +363,7 @@ package final class BuildDescription: Serializable, Sendable, Encodable, Cacheab serializer.serialize(bypassActualTasks) serializer.serialize(targetsBuildInParallel) serializer.serialize(emitFrontendCommandLines) + serializer.serialize(casValidationInfos) serializer.endAggregate() } @@ -359,7 +371,7 @@ package final class BuildDescription: Serializable, Sendable, Encodable, Cacheab // Check that we have the appropriate delegate. guard let delegate = deserializer.delegate as? BuildDescriptionDeserializerDelegate else { throw DeserializerError.invalidDelegate("delegate must be a BuildDescriptionDeserializerDelegate") } - try deserializer.beginAggregate(19) + try deserializer.beginAggregate(20) self.dir = try deserializer.deserialize() self.signature = try deserializer.deserialize() self.allOutputPaths = try deserializer.deserialize() @@ -395,6 +407,7 @@ package final class BuildDescription: Serializable, Sendable, Encodable, Cacheab throw DeserializerError.deserializationFailed("Expected delegate to provide a TaskStore") } self.taskStore = taskStore + self.casValidationInfos = try deserializer.deserialize() } package var cost: Int { @@ -533,6 +546,9 @@ package final class BuildDescriptionBuilder { // The map of module cache path per configured target. private let moduleCachePathsPerTarget: [ConfiguredTarget: [Path]] + /// The set of all CAS directories and their corresponding CASOptions. + private let casValidationInfos: [BuildDescription.CASValidationInfo] + // The map of stale file removal identifier per configured target. private let staleFileRemovalIdentifierPerTarget: [ConfiguredTarget?: String] @@ -550,7 +566,7 @@ package final class BuildDescriptionBuilder { /// - Parameters: /// - path: The path of a directory to store the build description to. /// - bypassActualTasks: If enabled, replace tasks with fake ones (`/usr/bin/true`). - init(path: Path, signature: BuildDescriptionSignature, buildCommand: BuildCommand, taskAdditionalInputs: [Ref: NodeList], mutatedNodes: Set>, mutatingTasks: [Ref: MutatingTaskInfo], bypassActualTasks: Bool, targetsBuildInParallel: Bool, emitFrontendCommandLines: Bool, moduleSessionFilePath: Path?, invalidationPaths: [Path], recursiveSearchPathResults: [RecursiveSearchPathResolver.CachedResult], copiedPathMap: [String: String], outputPathsPerTarget: [ConfiguredTarget?: [Path]], allOutputPaths: Set, rootPathsPerTarget: [ConfiguredTarget: [Path]], moduleCachePathsPerTarget: [ConfiguredTarget: [Path]], staleFileRemovalIdentifierPerTarget: [ConfiguredTarget?: String], settingsPerTarget: [ConfiguredTarget: Settings], targetDependencies: [TargetDependencyRelationship], definingTargetsByModuleName: [String: OrderedSet], workspace: Workspace, capturedBuildInfo: CapturedBuildInfo?) { + init(path: Path, signature: BuildDescriptionSignature, buildCommand: BuildCommand, taskAdditionalInputs: [Ref: NodeList], mutatedNodes: Set>, mutatingTasks: [Ref: MutatingTaskInfo], bypassActualTasks: Bool, targetsBuildInParallel: Bool, emitFrontendCommandLines: Bool, moduleSessionFilePath: Path?, invalidationPaths: [Path], recursiveSearchPathResults: [RecursiveSearchPathResolver.CachedResult], copiedPathMap: [String: String], outputPathsPerTarget: [ConfiguredTarget?: [Path]], allOutputPaths: Set, rootPathsPerTarget: [ConfiguredTarget: [Path]], moduleCachePathsPerTarget: [ConfiguredTarget: [Path]], casValidationInfos: [BuildDescription.CASValidationInfo], staleFileRemovalIdentifierPerTarget: [ConfiguredTarget?: String], settingsPerTarget: [ConfiguredTarget: Settings], targetDependencies: [TargetDependencyRelationship], definingTargetsByModuleName: [String: OrderedSet], workspace: Workspace, capturedBuildInfo: CapturedBuildInfo?) { self.path = path self.signature = signature self.taskAdditionalInputs = taskAdditionalInputs @@ -567,6 +583,7 @@ package final class BuildDescriptionBuilder { self.allOutputPaths = allOutputPaths self.rootPathsPerTarget = rootPathsPerTarget self.moduleCachePathsPerTarget = moduleCachePathsPerTarget + self.casValidationInfos = casValidationInfos self.staleFileRemovalIdentifierPerTarget = staleFileRemovalIdentifierPerTarget self.settingsPerTarget = settingsPerTarget self.targetDependencies = targetDependencies @@ -679,7 +696,7 @@ package final class BuildDescriptionBuilder { // Create the build description. let buildDescription: BuildDescription do { - buildDescription = try BuildDescription(inDir: path, signature: signature, taskStore: frozenTaskStore, allOutputPaths: allOutputPaths, rootPathsPerTarget: rootPathsPerTarget, moduleCachePathsPerTarget: moduleCachePathsPerTarget, settingsPerTarget: settingsPerTarget, taskActionMap: taskActionMap, targetTaskCounts: targetTaskCounts, moduleSessionFilePath: moduleSessionFilePath, diagnostics: diagnosticsEngines.mapValues { engine in engine.diagnostics }, fs: fs, invalidationPaths: invalidationPaths, recursiveSearchPathResults: recursiveSearchPathResults, copiedPathMap: copiedPathMap, targetDependencies: targetDependencies, definingTargetsByModuleName: definingTargetsByModuleName, capturedBuildInfo: capturedBuildInfo, bypassActualTasks: bypassActualTasks, targetsBuildInParallel: targetsBuildInParallel, emitFrontendCommandLines: emitFrontendCommandLines) + buildDescription = try BuildDescription(inDir: path, signature: signature, taskStore: frozenTaskStore, allOutputPaths: allOutputPaths, rootPathsPerTarget: rootPathsPerTarget, moduleCachePathsPerTarget: moduleCachePathsPerTarget, casValidationInfos: casValidationInfos, settingsPerTarget: settingsPerTarget, taskActionMap: taskActionMap, targetTaskCounts: targetTaskCounts, moduleSessionFilePath: moduleSessionFilePath, diagnostics: diagnosticsEngines.mapValues { engine in engine.diagnostics }, fs: fs, invalidationPaths: invalidationPaths, recursiveSearchPathResults: recursiveSearchPathResults, copiedPathMap: copiedPathMap, targetDependencies: targetDependencies, definingTargetsByModuleName: definingTargetsByModuleName, capturedBuildInfo: capturedBuildInfo, bypassActualTasks: bypassActualTasks, targetsBuildInParallel: targetsBuildInParallel, emitFrontendCommandLines: emitFrontendCommandLines) } catch { throw StubError.error("unable to create build description: \(error)") @@ -1009,7 +1026,7 @@ extension BuildDescription { // FIXME: Bypass actual tasks should go away, eventually. // // FIXME: This layering isn't working well, we are plumbing a bunch of stuff through here just because we don't want to talk to TaskConstruction. - static package func construct(workspace: Workspace, tasks: [any PlannedTask], path: Path, signature: BuildDescriptionSignature, buildCommand: BuildCommand, diagnostics: [ConfiguredTarget?: [Diagnostic]] = [:], indexingInfo: [(forTarget: ConfiguredTarget?, path: Path, indexingInfo: any SourceFileIndexingInfo)] = [], fs: any FSProxy = localFS, bypassActualTasks: Bool = false, targetsBuildInParallel: Bool = true, emitFrontendCommandLines: Bool = false, moduleSessionFilePath: Path? = nil, invalidationPaths: [Path] = [], recursiveSearchPathResults: [RecursiveSearchPathResolver.CachedResult] = [], copiedPathMap: [String: String] = [:], rootPathsPerTarget: [ConfiguredTarget:[Path]] = [:], moduleCachePathsPerTarget: [ConfiguredTarget: [Path]] = [:], staleFileRemovalIdentifierPerTarget: [ConfiguredTarget?: String] = [:], settingsPerTarget: [ConfiguredTarget: Settings] = [:], delegate: any BuildDescriptionConstructionDelegate, targetDependencies: [TargetDependencyRelationship] = [], definingTargetsByModuleName: [String: OrderedSet], capturedBuildInfo: CapturedBuildInfo?, userPreferences: UserPreferences) async throws -> BuildDescription? { + static package func construct(workspace: Workspace, tasks: [any PlannedTask], path: Path, signature: BuildDescriptionSignature, buildCommand: BuildCommand, diagnostics: [ConfiguredTarget?: [Diagnostic]] = [:], indexingInfo: [(forTarget: ConfiguredTarget?, path: Path, indexingInfo: any SourceFileIndexingInfo)] = [], fs: any FSProxy = localFS, bypassActualTasks: Bool = false, targetsBuildInParallel: Bool = true, emitFrontendCommandLines: Bool = false, moduleSessionFilePath: Path? = nil, invalidationPaths: [Path] = [], recursiveSearchPathResults: [RecursiveSearchPathResolver.CachedResult] = [], copiedPathMap: [String: String] = [:], rootPathsPerTarget: [ConfiguredTarget:[Path]] = [:], moduleCachePathsPerTarget: [ConfiguredTarget: [Path]] = [:], casValidationInfos: [BuildDescription.CASValidationInfo] = [], staleFileRemovalIdentifierPerTarget: [ConfiguredTarget?: String] = [:], settingsPerTarget: [ConfiguredTarget: Settings] = [:], delegate: any BuildDescriptionConstructionDelegate, targetDependencies: [TargetDependencyRelationship] = [], definingTargetsByModuleName: [String: OrderedSet], capturedBuildInfo: CapturedBuildInfo?, userPreferences: UserPreferences) async throws -> BuildDescription? { var diagnostics = diagnostics // We operate on the sorted tasks here to ensure that the list of task additional inputs is deterministic. @@ -1272,7 +1289,7 @@ extension BuildDescription { } // Create the builder. - let builder = BuildDescriptionBuilder(path: path, signature: signature, buildCommand: buildCommand, taskAdditionalInputs: taskAdditionalInputs, mutatedNodes: Set(mutableNodes.keys), mutatingTasks: mutatingTasks, bypassActualTasks: bypassActualTasks, targetsBuildInParallel: targetsBuildInParallel, emitFrontendCommandLines: emitFrontendCommandLines, moduleSessionFilePath: moduleSessionFilePath, invalidationPaths: invalidationPaths, recursiveSearchPathResults: recursiveSearchPathResults, copiedPathMap: copiedPathMap, outputPathsPerTarget: outputPathsPerTarget, allOutputPaths: Set(producers.keys.map { $0.instance.path }), rootPathsPerTarget: rootPathsPerTarget, moduleCachePathsPerTarget: moduleCachePathsPerTarget, staleFileRemovalIdentifierPerTarget: staleFileRemovalIdentifierPerTarget, settingsPerTarget: settingsPerTarget, targetDependencies: targetDependencies, definingTargetsByModuleName: definingTargetsByModuleName, workspace: workspace, capturedBuildInfo: capturedBuildInfo) + let builder = BuildDescriptionBuilder(path: path, signature: signature, buildCommand: buildCommand, taskAdditionalInputs: taskAdditionalInputs, mutatedNodes: Set(mutableNodes.keys), mutatingTasks: mutatingTasks, bypassActualTasks: bypassActualTasks, targetsBuildInParallel: targetsBuildInParallel, emitFrontendCommandLines: emitFrontendCommandLines, moduleSessionFilePath: moduleSessionFilePath, invalidationPaths: invalidationPaths, recursiveSearchPathResults: recursiveSearchPathResults, copiedPathMap: copiedPathMap, outputPathsPerTarget: outputPathsPerTarget, allOutputPaths: Set(producers.keys.map { $0.instance.path }), rootPathsPerTarget: rootPathsPerTarget, moduleCachePathsPerTarget: moduleCachePathsPerTarget, casValidationInfos: casValidationInfos, staleFileRemovalIdentifierPerTarget: staleFileRemovalIdentifierPerTarget, settingsPerTarget: settingsPerTarget, targetDependencies: targetDependencies, definingTargetsByModuleName: definingTargetsByModuleName, workspace: workspace, capturedBuildInfo: capturedBuildInfo) for (target, diagnostics) in diagnostics { let engine = builder.diagnosticsEngines.getOrInsert(target, { DiagnosticsEngine() }) for diag in diagnostics { @@ -1478,3 +1495,31 @@ package extension PlannedNode { } } } + +extension BuildDescription.CASValidationInfo: Serializable { + package func serialize(to serializer: T) where T : Serializer { + serializer.serializeAggregate(2) { + serializer.serialize(options) + serializer.serialize(llvmCasExec) + } + } + + package init(from deserializer: any Deserializer) throws { + try deserializer.beginAggregate(2) + self.options = try deserializer.deserialize() + self.llvmCasExec = try deserializer.deserialize() + } +} + +// Note: for the purposes of validation we intentionally ignore irrelevant +// differences in CASOptions. However, we need to keep the llvm-cas executable +// in case there are multiple cas format versions sharing the path. +extension BuildDescription.CASValidationInfo: Hashable { + package func hash(into hasher: inout Hasher) { + hasher.combine(options.casPath) + hasher.combine(llvmCasExec) + } + static package func ==(lhs: Self, rhs: Self) -> Bool { + return lhs.options.casPath == rhs.options.casPath && lhs.llvmCasExec == rhs.llvmCasExec + } +} diff --git a/Sources/SWBTaskExecution/BuildDescriptionManager.swift b/Sources/SWBTaskExecution/BuildDescriptionManager.swift index ae355c56..bc1da89a 100644 --- a/Sources/SWBTaskExecution/BuildDescriptionManager.swift +++ b/Sources/SWBTaskExecution/BuildDescriptionManager.swift @@ -180,7 +180,10 @@ package final class BuildDescriptionManager: Sendable { var settingsPerTarget = [ConfiguredTarget:Settings]() var rootPathsPerTarget = [ConfiguredTarget:[Path]]() var moduleCachePathsPerTarget = [ConfiguredTarget: [Path]]() + + var casValidationInfos: OrderedSet = [] let buildGraph = planRequest.buildGraph + let shouldValidateCAS = Settings.supportsCompilationCaching(plan.workspaceContext.core) && UserDefaults.enableCASValidation // Add the SFR identifier for target-independent tasks. staleFileRemovalIdentifierPerTarget[nil] = plan.staleFileRemovalTaskIdentifier(for: nil) @@ -199,6 +202,18 @@ package final class BuildDescriptionManager: Sendable { Path(settings.globalScope.evaluate(BuiltinMacros.CLANG_EXPLICIT_MODULES_OUTPUT_PATH)), ] + if shouldValidateCAS, settings.globalScope.evaluate(BuiltinMacros.CLANG_ENABLE_COMPILE_CACHE) || settings.globalScope.evaluate(BuiltinMacros.SWIFT_ENABLE_COMPILE_CACHE) { + // FIXME: currently we only handle the compiler cache here, because the plugin configuration for the generic CAS is not configured by build settings. + for purpose in [CASOptions.Purpose.compiler(.c)] { + if let casOpts = try? CASOptions.create(settings.globalScope, purpose) { + let execName = settings.globalScope.evaluate(BuiltinMacros.VALIDATE_CAS_EXEC).nilIfEmpty ?? "llvm-cas" + if let execPath = settings.executableSearchPaths.lookup(Path(execName)) { + casValidationInfos.append(.init(options: casOpts, llvmCasExec: execPath)) + } + } + } + } + staleFileRemovalIdentifierPerTarget[target] = plan.staleFileRemovalTaskIdentifier(for: target) settingsPerTarget[target] = settings } @@ -231,7 +246,7 @@ package final class BuildDescriptionManager: Sendable { } // Create the build description. - return try await BuildDescription.construct(workspace: buildGraph.workspaceContext.workspace, tasks: plan.tasks, path: path, signature: signature, buildCommand: planRequest.buildRequest.buildCommand, diagnostics: planningDiagnostics, indexingInfo: [], fs: fs, bypassActualTasks: bypassActualTasks, targetsBuildInParallel: buildGraph.targetsBuildInParallel, emitFrontendCommandLines: plan.emitFrontendCommandLines, moduleSessionFilePath: planRequest.workspaceContext.getModuleSessionFilePath(planRequest.buildRequest.parameters), invalidationPaths: plan.invalidationPaths, recursiveSearchPathResults: plan.recursiveSearchPathResults, copiedPathMap: plan.copiedPathMap, rootPathsPerTarget: rootPathsPerTarget, moduleCachePathsPerTarget: moduleCachePathsPerTarget, staleFileRemovalIdentifierPerTarget: staleFileRemovalIdentifierPerTarget, settingsPerTarget: settingsPerTarget, delegate: delegate, targetDependencies: buildGraph.targetDependenciesByGuid, definingTargetsByModuleName: definingTargetsByModuleName, capturedBuildInfo: capturedBuildInfo, userPreferences: buildGraph.workspaceContext.userPreferences) + return try await BuildDescription.construct(workspace: buildGraph.workspaceContext.workspace, tasks: plan.tasks, path: path, signature: signature, buildCommand: planRequest.buildRequest.buildCommand, diagnostics: planningDiagnostics, indexingInfo: [], fs: fs, bypassActualTasks: bypassActualTasks, targetsBuildInParallel: buildGraph.targetsBuildInParallel, emitFrontendCommandLines: plan.emitFrontendCommandLines, moduleSessionFilePath: planRequest.workspaceContext.getModuleSessionFilePath(planRequest.buildRequest.parameters), invalidationPaths: plan.invalidationPaths, recursiveSearchPathResults: plan.recursiveSearchPathResults, copiedPathMap: plan.copiedPathMap, rootPathsPerTarget: rootPathsPerTarget, moduleCachePathsPerTarget: moduleCachePathsPerTarget, casValidationInfos: casValidationInfos.elements, staleFileRemovalIdentifierPerTarget: staleFileRemovalIdentifierPerTarget, settingsPerTarget: settingsPerTarget, delegate: delegate, targetDependencies: buildGraph.targetDependenciesByGuid, definingTargetsByModuleName: definingTargetsByModuleName, capturedBuildInfo: capturedBuildInfo, userPreferences: buildGraph.workspaceContext.userPreferences) } /// Encapsulates the two ways `getNewOrCachedBuildDescription` can be called, whether we want to retrieve or create a build description based on a plan or whether we have an explicit build description ID that we want to retrieve and we don't need to create a new one. diff --git a/Sources/SWBTestSupport/CoreBasedTests.swift b/Sources/SWBTestSupport/CoreBasedTests.swift index 2ab4fe1d..2cfbeb6f 100644 --- a/Sources/SWBTestSupport/CoreBasedTests.swift +++ b/Sources/SWBTestSupport/CoreBasedTests.swift @@ -159,6 +159,14 @@ extension CoreBasedTests { } } + /// The path to llvm-cas in the default toolchain. + package var llvmCasToolPath: Path { + get async throws { + let (core, defaultToolchain) = try await coreAndToolchain() + return try #require(defaultToolchain.executableSearchPaths.findExecutable(operatingSystem: core.hostOperatingSystem, basename: "llvm-cas"), "couldn't find llvm-cas in default toolchain") + } + } + /// The path to the TAPI tool in the default toolchain. package var tapiToolPath: Path { get async throws { diff --git a/Sources/SWBTestSupport/SkippedTestSupport.swift b/Sources/SWBTestSupport/SkippedTestSupport.swift index 7ba31ee9..08ef7190 100644 --- a/Sources/SWBTestSupport/SkippedTestSupport.swift +++ b/Sources/SWBTestSupport/SkippedTestSupport.swift @@ -12,6 +12,7 @@ import class Foundation.FileManager import class Foundation.ProcessInfo +import struct Foundation.URL package import SWBUtil package import SWBCore @@ -394,6 +395,19 @@ extension Trait where Self == Testing.ConditionTrait { } } + package static var requireCASValidation: Self { + enabled { + guard try await ConditionTraitContext.shared.supportsCompilationCaching, UserDefaults.enableCASValidation else { + return false + } + guard let path = try? await ConditionTraitContext.shared.llvmCasToolPath else { + return false + } + let result = try await Process.getOutput(url: URL(fileURLWithPath: path.str), arguments: ["--help"]) + return result.stdout.contains(ByteString("validate-if-needed")) + } + } + package static var requireCASPlugin: Self { enabled("libclang does not support CAS plugins") { try await casOptions().canUseCASPlugin } } diff --git a/Sources/SWBTestSupport/TaskExecutionTestSupport.swift b/Sources/SWBTestSupport/TaskExecutionTestSupport.swift index 0fc3b973..9a5f167b 100644 --- a/Sources/SWBTestSupport/TaskExecutionTestSupport.swift +++ b/Sources/SWBTestSupport/TaskExecutionTestSupport.swift @@ -94,8 +94,8 @@ package struct TestManifest: Sendable { extension BuildDescription { /// Convenience testing method which omits the `capturedBuildInfo:` parameter. - static package func construct(workspace: Workspace, tasks: [any PlannedTask], path: Path, signature: BuildDescriptionSignature, buildCommand: BuildCommand, diagnostics: [ConfiguredTarget?: [Diagnostic]] = [:], indexingInfo: [(forTarget: ConfiguredTarget?, path: Path, indexingInfo: any SourceFileIndexingInfo)] = [], fs: any FSProxy = localFS, bypassActualTasks: Bool = false, moduleSessionFilePath: Path? = nil, invalidationPaths: [Path] = [], recursiveSearchPathResults: [RecursiveSearchPathResolver.CachedResult] = [], copiedPathMap: [String: String] = [:], rootPathsPerTarget: [ConfiguredTarget:[Path]] = [:], moduleCachePathsPerTarget: [ConfiguredTarget: [Path]] = [:], staleFileRemovalIdentifierPerTarget: [ConfiguredTarget: String] = [:], settingsPerTarget: [ConfiguredTarget: Settings] = [:], delegate: any BuildDescriptionConstructionDelegate, targetDependencies: [TargetDependencyRelationship] = [], definingTargetsByModuleName: [String: OrderedSet] = [:]) async throws -> BuildDescription? { - return try await construct(workspace: workspace, tasks: tasks, path: path, signature: signature, buildCommand: buildCommand, diagnostics: diagnostics, indexingInfo: indexingInfo, fs: fs, bypassActualTasks: bypassActualTasks, moduleSessionFilePath: moduleSessionFilePath, invalidationPaths: invalidationPaths, recursiveSearchPathResults: recursiveSearchPathResults, copiedPathMap: copiedPathMap, rootPathsPerTarget: rootPathsPerTarget, moduleCachePathsPerTarget: moduleCachePathsPerTarget, staleFileRemovalIdentifierPerTarget: staleFileRemovalIdentifierPerTarget, settingsPerTarget: settingsPerTarget, delegate: delegate, targetDependencies: targetDependencies, definingTargetsByModuleName: definingTargetsByModuleName, capturedBuildInfo: nil, userPreferences: .defaultForTesting) + static package func construct(workspace: Workspace, tasks: [any PlannedTask], path: Path, signature: BuildDescriptionSignature, buildCommand: BuildCommand, diagnostics: [ConfiguredTarget?: [Diagnostic]] = [:], indexingInfo: [(forTarget: ConfiguredTarget?, path: Path, indexingInfo: any SourceFileIndexingInfo)] = [], fs: any FSProxy = localFS, bypassActualTasks: Bool = false, moduleSessionFilePath: Path? = nil, invalidationPaths: [Path] = [], recursiveSearchPathResults: [RecursiveSearchPathResolver.CachedResult] = [], copiedPathMap: [String: String] = [:], rootPathsPerTarget: [ConfiguredTarget:[Path]] = [:], moduleCachePathsPerTarget: [ConfiguredTarget: [Path]] = [:], casValidationInfos: [BuildDescription.CASValidationInfo] = [], staleFileRemovalIdentifierPerTarget: [ConfiguredTarget: String] = [:], settingsPerTarget: [ConfiguredTarget: Settings] = [:], delegate: any BuildDescriptionConstructionDelegate, targetDependencies: [TargetDependencyRelationship] = [], definingTargetsByModuleName: [String: OrderedSet] = [:]) async throws -> BuildDescription? { + return try await construct(workspace: workspace, tasks: tasks, path: path, signature: signature, buildCommand: buildCommand, diagnostics: diagnostics, indexingInfo: indexingInfo, fs: fs, bypassActualTasks: bypassActualTasks, moduleSessionFilePath: moduleSessionFilePath, invalidationPaths: invalidationPaths, recursiveSearchPathResults: recursiveSearchPathResults, copiedPathMap: copiedPathMap, rootPathsPerTarget: rootPathsPerTarget, moduleCachePathsPerTarget: moduleCachePathsPerTarget, casValidationInfos: casValidationInfos, staleFileRemovalIdentifierPerTarget: staleFileRemovalIdentifierPerTarget, settingsPerTarget: settingsPerTarget, delegate: delegate, targetDependencies: targetDependencies, definingTargetsByModuleName: definingTargetsByModuleName, capturedBuildInfo: nil, userPreferences: .defaultForTesting) } } diff --git a/Sources/SWBUtil/UserDefaults.swift b/Sources/SWBUtil/UserDefaults.swift index 452ca333..e1a58763 100644 --- a/Sources/SWBUtil/UserDefaults.swift +++ b/Sources/SWBUtil/UserDefaults.swift @@ -203,6 +203,10 @@ public enum UserDefaults: Sendable { return hasValue(forKey: "EnableSDKStatCaching") ? bool(forKey: "EnableSDKStatCaching") : true } + public static var enableCASValidation: Bool { + return hasValue(forKey: "EnableCASValidation") ? bool(forKey: "EnableCASValidation") : true + } + public static var useTargetDependenciesForImpartedBuildSettings: Bool { return bool(forKey: "UseTargetDependenciesForImpartedBuildSettings") } diff --git a/Tests/SWBBuildSystemTests/ClangCompilationCachingTests.swift b/Tests/SWBBuildSystemTests/ClangCompilationCachingTests.swift index e8b436e0..f1dd82bc 100644 --- a/Tests/SWBBuildSystemTests/ClangCompilationCachingTests.swift +++ b/Tests/SWBBuildSystemTests/ClangCompilationCachingTests.swift @@ -1833,6 +1833,217 @@ fileprivate struct ClangCompilationCachingTests: CoreBasedTests { } } } + + @Test(.requireCASValidation, .requireSDKs(.macOS), arguments: [(true, true), (false, true), (false, false)]) + func validateCAS(usePlugin: Bool, enableCaching: Bool) async throws { + try await withTemporaryDirectory { tmpDirPath in + let casPath = tmpDirPath.join("CompilationCache") + var buildSettings: [String: String] = [ + "PRODUCT_NAME": "$(TARGET_NAME)", + "CLANG_ENABLE_COMPILE_CACHE": enableCaching ? "YES" : "NO", + "CLANG_ENABLE_MODULES": "NO", + "COMPILATION_CACHE_CAS_PATH": casPath.str, + ] + if usePlugin { + buildSettings["COMPILATION_CACHE_ENABLE_PLUGIN"] = "YES" + } + let testWorkspace = TestWorkspace( + "Test", + sourceRoot: tmpDirPath.join("Test"), + projects: [ + TestProject( + "aProject", + groupTree: TestGroup( + "Sources", + children: [ + TestFile("file.c"), + ]), + buildConfigurations: [TestBuildConfiguration( + "Debug", + buildSettings: buildSettings)], + targets: [ + TestStandardTarget( + "Library", + type: .staticLibrary, + buildPhases: [ + TestSourcesBuildPhase(["file.c"]), + ]), + ])]) + + let tester = try await BuildOperationTester(getCore(), testWorkspace, simulated: false) + try await tester.fs.writeFileContents(testWorkspace.sourceRoot.join("aProject/file.c")) { stream in + stream <<< + """ + #include + int something = 1; + """ + } + + let specificCAS = casPath.join(usePlugin ? "plugin" : "builtin") + let ruleInfo = "ValidateCAS \(specificCAS.str) \(try await ConditionTraitContext.shared.llvmCasToolPath.str)" + + let checkBuild = { (expectedOutput: ByteString?) in + try await tester.checkBuild(runDestination: .macOS, persistent: true) { results in + + if enableCaching { + results.check(contains: .activityStarted(ruleInfo: ruleInfo)) + if let expectedOutput { + results.check(contains: .activityEmittedData(ruleInfo: ruleInfo, expectedOutput.bytes)) + } + results.check(contains: .activityEnded(ruleInfo: ruleInfo, status: .succeeded)) + } else { + results.check(notContains: .activityStarted(ruleInfo: ruleInfo)) + } + results.checkNoDiagnostics() + } + } + + // Ignore output for plugin CAS since it may not yet support validation. + try await checkBuild(usePlugin ? nil : "validated successfully\n") + // The second build should not require validation. + try await checkBuild("validation skipped\n") + // Including clean builds. + try await tester.checkBuild(runDestination: .macOS, buildCommand: .cleanBuildFolder(style: .regular), body: { _ in }) + try await checkBuild("validation skipped\n") + } + } + + @Test(.requireCASValidation, .requireSDKs(.macOS)) + func validateCASRecovery() async throws { + try await withTemporaryDirectory { tmpDirPath in + let casPath = tmpDirPath.join("CompilationCache") + let buildSettings: [String: String] = [ + "PRODUCT_NAME": "$(TARGET_NAME)", + "CLANG_ENABLE_COMPILE_CACHE": "YES", + "CLANG_ENABLE_MODULES": "NO", + "COMPILATION_CACHE_CAS_PATH": casPath.str, + ] + let testWorkspace = TestWorkspace( + "Test", + sourceRoot: tmpDirPath.join("Test"), + projects: [ + TestProject( + "aProject", + groupTree: TestGroup( + "Sources", + children: [ + TestFile("file.c"), + ]), + buildConfigurations: [TestBuildConfiguration( + "Debug", + buildSettings: buildSettings)], + targets: [ + TestStandardTarget( + "Library", + type: .staticLibrary, + buildPhases: [ + TestSourcesBuildPhase(["file.c"]), + ]), + ])]) + + let tester = try await BuildOperationTester(getCore(), testWorkspace, simulated: false) + try await tester.fs.writeFileContents(testWorkspace.sourceRoot.join("aProject/file.c")) { stream in + stream <<< + """ + #include + int something = 1; + """ + } + + let specificCAS = casPath.join("builtin") + let ruleInfo = "ValidateCAS \(specificCAS.str) \(try await ConditionTraitContext.shared.llvmCasToolPath.str)" + + let checkBuild = { (expectedOutput: ByteString?) in + try await tester.checkBuild(runDestination: .macOS, persistent: true) { results in + results.check(contains: .activityStarted(ruleInfo: ruleInfo)) + if let expectedOutput { + results.check(contains: .activityEmittedData(ruleInfo: ruleInfo, expectedOutput.bytes)) + } + results.check(contains: .activityEnded(ruleInfo: ruleInfo, status: .succeeded)) + results.checkNoDiagnostics() + } + } + + // Ignore output for plugin CAS since it may not yet support validation. + try await checkBuild("validated successfully\n") + // Create an error and trigger revalidation by messing with the validation data. + try tester.fs.move(casPath.join("builtin/v1.1/v8.data"), to: casPath.join("builtin/v1.1/v8.data.moved")) + try await tester.fs.writeFileContents(casPath.join("builtin/v1.validation")) { stream in + stream <<< "0" + } + try await checkBuild("recovered from invalid data\n") + } + } + + @Test(.requireCASValidation, .requireSDKs(.macOS)) + func validateCASMultipleExec() async throws { + try await withTemporaryDirectory { (tmpDirPath: Path) in + let casPath = tmpDirPath.join("CompilationCache") + let buildSettings: [String: String] = [ + "PRODUCT_NAME": "$(TARGET_NAME)", + "CLANG_ENABLE_COMPILE_CACHE": "YES", + "CLANG_ENABLE_MODULES": "NO", + "COMPILATION_CACHE_CAS_PATH": casPath.str, + ] + let llvmCasExec = try await ConditionTraitContext.shared.llvmCasToolPath + // Create a trivially different path. If we ever canonicalize the path it will be harder to test this. + let llvmCasExec2 = Path("\(llvmCasExec.dirname.str)\(Path.pathSeparator)\(Path.pathSeparator)\(llvmCasExec.basename)") + let testWorkspace = TestWorkspace( + "Test", + sourceRoot: tmpDirPath.join("Test"), + projects: [ + TestProject( + "aProject", + groupTree: TestGroup( + "Sources", + children: [ + TestFile("file.c"), + ]), + buildConfigurations: [TestBuildConfiguration( + "Debug", + buildSettings: buildSettings)], + targets: [ + TestStandardTarget( + "Library1", + type: .staticLibrary, + buildPhases: [ + TestSourcesBuildPhase(["file.c"]), + ]), + TestStandardTarget( + "Library2", + type: .staticLibrary, + buildConfigurations: [TestBuildConfiguration( + "Debug", + buildSettings: [ + "VALIDATE_CAS_EXEC": llvmCasExec2.str, + ])], + buildPhases: [ + TestSourcesBuildPhase(["file.c"]), + ]), + ])]) + + let tester = try await BuildOperationTester(getCore(), testWorkspace, simulated: false) + try await tester.fs.writeFileContents(testWorkspace.sourceRoot.join("aProject/file.c")) { stream in + stream <<< + """ + #include + int something = 1; + """ + } + + let specificCAS = casPath.join("builtin") + let parameters = BuildParameters(configuration: "Debug", activeRunDestination: .macOS) + let targets = tester.workspace.allTargets.map({ BuildRequest.BuildTargetInfo(parameters: parameters, target: $0) }) + + try await tester.checkBuild(runDestination: .macOS, buildRequest: BuildRequest(parameters: parameters, buildTargets: targets, continueBuildingAfterErrors: false, useParallelTargets: true, useImplicitDependencies: true, useDryRun: false), persistent: true) { results in + for ruleInfo in ["ValidateCAS \(specificCAS.str) \(llvmCasExec.str)", "ValidateCAS \(specificCAS.str) \(llvmCasExec2.str)"] { + results.check(contains: .activityStarted(ruleInfo: ruleInfo)) + results.check(contains: .activityEnded(ruleInfo: ruleInfo, status: .succeeded)) + } + results.checkNoDiagnostics() + } + } + } } extension BuildOperationTester.BuildResults { diff --git a/Tests/SWBBuildSystemTests/SwiftCompilationCachingTests.swift b/Tests/SWBBuildSystemTests/SwiftCompilationCachingTests.swift index c01277a9..da5864bc 100644 --- a/Tests/SWBBuildSystemTests/SwiftCompilationCachingTests.swift +++ b/Tests/SWBBuildSystemTests/SwiftCompilationCachingTests.swift @@ -17,6 +17,7 @@ import SWBTestSupport import SWBUtil import SWBTaskExecution +import SWBProtocol @Suite(.requireSwiftFeatures(.compilationCaching), .requireCompilationCaching, .flaky("A handful of Swift Build CAS tests fail when running the entire test suite"), .bug("rdar://146781403")) @@ -202,6 +203,73 @@ fileprivate struct SwiftCompilationCachingTests: CoreBasedTests { } } } + + @Test(.requireCASValidation, .requireSDKs(.macOS)) + func validateCAS() async throws { + try await withTemporaryDirectory { tmpDirPath in + let casPath = tmpDirPath.join("CompilationCache") + let buildSettings: [String: String] = [ + "PRODUCT_NAME": "$(TARGET_NAME)", + "SWIFT_VERSION": try await swiftVersion, + "SWIFT_ENABLE_COMPILE_CACHE": "YES", + "SWIFT_ENABLE_EXPLICIT_MODULES": "YES", + "COMPILATION_CACHE_CAS_PATH": casPath.str, + "DSTROOT": tmpDirPath.join("dstroot").str, + ] + let testWorkspace = TestWorkspace( + "Test", + sourceRoot: tmpDirPath.join("Test"), + projects: [ + TestProject( + "aProject", + groupTree: TestGroup( + "Sources", + children: [ + TestFile("file.swift"), + ]), + buildConfigurations: [TestBuildConfiguration( + "Debug", + buildSettings: buildSettings)], + targets: [ + TestStandardTarget( + "Library", + type: .staticLibrary, + buildPhases: [ + TestSourcesBuildPhase(["file.swift"]), + ]), + ])]) + + let tester = try await BuildOperationTester(getCore(), testWorkspace, simulated: false) + try await tester.fs.writeFileContents(testWorkspace.sourceRoot.join("aProject/file.swift")) { stream in + stream <<< + """ + public func libFunc() {} + """ + } + + let specificCAS = casPath.join("builtin") + let ruleInfo = "ValidateCAS \(specificCAS.str) \(try await ConditionTraitContext.shared.llvmCasToolPath.str)" + + let checkBuild = { (expectedOutput: ByteString?) in + try await tester.checkBuild(runDestination: .macOS, persistent: true) { results in + results.check(contains: .activityStarted(ruleInfo: ruleInfo)) + if let expectedOutput { + results.check(contains: .activityEmittedData(ruleInfo: ruleInfo, expectedOutput.bytes)) + } + results.check(contains: .activityEnded(ruleInfo: ruleInfo, status: .succeeded)) + results.checkNoDiagnostics() + } + } + + // Ignore output for plugin CAS since it may not yet support validation. + try await checkBuild("validated successfully\n") + // The second build should not require validation. + try await checkBuild("validation skipped\n") + // Including clean builds. + try await tester.checkBuild(runDestination: .macOS, buildCommand: .cleanBuildFolder(style: .regular), body: { _ in }) + try await checkBuild("validation skipped\n") + } + } } extension BuildOperationTester.BuildResults { diff --git a/Tests/SwiftBuildTests/DeferredExecutionTests.swift b/Tests/SwiftBuildTests/DeferredExecutionTests.swift index 82227e70..04a6daf8 100644 --- a/Tests/SwiftBuildTests/DeferredExecutionTests.swift +++ b/Tests/SwiftBuildTests/DeferredExecutionTests.swift @@ -148,6 +148,8 @@ fileprivate struct DeferredExecutionTests: CoreBasedTests { case "derq": defer { derqExpectation.confirm() } return .result(status: .exit(0), stdout: Data(), stderr: Data()) + case "llvm-cas": + break default: Issue.record("Unexpected deferred execution request for command line: \(commandLine), workingDirectory: \(String(describing: workingDirectory)), environment: \(environment)") }