Merge pull request #699 from owenv/owenv/create-modules-dirs

Ensure module output directories are explicitly created by the build system

Author: owenv

Sources/SWBCore/Settings/BuiltinMacros.swift

@@ -506,8 +506,8 @@ public final class BuiltinMacros {
     public static let CLANG_ENABLE_EXPLICIT_MODULES_WITH_COMPILER_LAUNCHER = BuiltinMacros.declareBooleanMacro("CLANG_ENABLE_EXPLICIT_MODULES_WITH_COMPILER_LAUNCHER")
     public static let CLANG_EXPLICIT_MODULES_LIBCLANG_PATH = BuiltinMacros.declareStringMacro("CLANG_EXPLICIT_MODULES_LIBCLANG_PATH")
     public static let CLANG_EXPLICIT_MODULES_IGNORE_LIBCLANG_VERSION_MISMATCH = BuiltinMacros.declareBooleanMacro("CLANG_EXPLICIT_MODULES_IGNORE_LIBCLANG_VERSION_MISMATCH")
-    public static let CLANG_EXPLICIT_MODULES_OUTPUT_PATH = BuiltinMacros.declareStringMacro("CLANG_EXPLICIT_MODULES_OUTPUT_PATH")
-    public static let SWIFT_EXPLICIT_MODULES_OUTPUT_PATH = BuiltinMacros.declareStringMacro("SWIFT_EXPLICIT_MODULES_OUTPUT_PATH")
+    public static let CLANG_EXPLICIT_MODULES_OUTPUT_PATH = BuiltinMacros.declarePathMacro("CLANG_EXPLICIT_MODULES_OUTPUT_PATH")
+    public static let SWIFT_EXPLICIT_MODULES_OUTPUT_PATH = BuiltinMacros.declarePathMacro("SWIFT_EXPLICIT_MODULES_OUTPUT_PATH")
     public static let CLANG_ENABLE_COMPILE_CACHE = BuiltinMacros.declareBooleanMacro("CLANG_ENABLE_COMPILE_CACHE")
     public static let CLANG_CACHE_FINE_GRAINED_OUTPUTS = BuiltinMacros.declareEnumMacro("CLANG_CACHE_FINE_GRAINED_OUTPUTS") as EnumMacroDeclaration<FineGrainedCachingSetting>
     public static let CLANG_CACHE_FINE_GRAINED_OUTPUTS_VERIFICATION = BuiltinMacros.declareEnumMacro("CLANG_CACHE_FINE_GRAINED_OUTPUTS_VERIFICATION") as EnumMacroDeclaration<FineGrainedCachingVerificationSetting>

Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift

@@ -977,7 +977,7 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
                     usesCompilerLauncher: usesCompilerLauncher,
                     // This path is scoped to the project, so ideally different targets that use the same modules would
                     // share precompiled modules.
-                    outputPath: Path(cbc.scope.evaluate(BuiltinMacros.CLANG_EXPLICIT_MODULES_OUTPUT_PATH)),
+                    outputPath: cbc.scope.evaluate(BuiltinMacros.CLANG_EXPLICIT_MODULES_OUTPUT_PATH),
                     scanningOutputPath: scanningOutputPath,
                     casOptions: casOptions,
                     cacheFallbackIfNotAvailable: cbc.scope.evaluate(BuiltinMacros.CLANG_CACHE_FALLBACK_IF_UNAVAILABLE),

Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift

@@ -1809,7 +1809,7 @@ public final class SwiftCompilerSpec : CompilerSpec, SpecIdentifierType, SwiftDi
                 // Instructs the driver to perform build planning with explicit module builds
                 if explicitModuleBuildEnabled {
                     args.append("-explicit-module-build")
-                    let explicitDependencyOutputPath = Path(cbc.scope.evaluate(BuiltinMacros.SWIFT_EXPLICIT_MODULES_OUTPUT_PATH))
+                    let explicitDependencyOutputPath = cbc.scope.evaluate(BuiltinMacros.SWIFT_EXPLICIT_MODULES_OUTPUT_PATH)
                     args.append(contentsOf: ["-module-cache-path", explicitDependencyOutputPath.str])
                     let moduleCacheDir = cbc.scope.evaluate(BuiltinMacros.MODULE_CACHE_DIR)
                     if LibSwiftDriver.supportsDriverFlag(spelled: "-clang-scanner-module-cache-path"),
@@ -2296,7 +2296,7 @@ public final class SwiftCompilerSpec : CompilerSpec, SpecIdentifierType, SwiftDi
                 previewPayload: previewPayload,
                 localizationPayload: localizationPayload,
                 numExpectedCompileSubtasks: isUsingWholeModuleOptimization ? 1 : cbc.inputs.count,
-                driverPayload: await driverPayload(uniqueID: String(args.hashValue), scope: cbc.scope, delegate: delegate, compilationMode: compilationMode, isUsingWholeModuleOptimization: isUsingWholeModuleOptimization, args: args, tempDirPath: objectFileDir, explicitModulesTempDirPath: Path(cbc.scope.evaluate(BuiltinMacros.SWIFT_EXPLICIT_MODULES_OUTPUT_PATH)), variant: variant, arch: arch + compilationMode.moduleBaseNameSuffix, commandLine: ["builtin-SwiftDriver", "--"] + args, ruleInfo: ruleInfo(compilationMode.ruleNameIntegratedDriver, targetName), casOptions: casOptions, linkerResponseFilePath: moduleLinkerArgsPath), previewStyle: cbc.scope.previewStyle
+                driverPayload: await driverPayload(uniqueID: String(args.hashValue), scope: cbc.scope, delegate: delegate, compilationMode: compilationMode, isUsingWholeModuleOptimization: isUsingWholeModuleOptimization, args: args, tempDirPath: objectFileDir, explicitModulesTempDirPath: cbc.scope.evaluate(BuiltinMacros.SWIFT_EXPLICIT_MODULES_OUTPUT_PATH), variant: variant, arch: arch + compilationMode.moduleBaseNameSuffix, commandLine: ["builtin-SwiftDriver", "--"] + args, ruleInfo: ruleInfo(compilationMode.ruleNameIntegratedDriver, targetName), casOptions: casOptions, linkerResponseFilePath: moduleLinkerArgsPath), previewStyle: cbc.scope.previewStyle
             )
 
             // Finally, assemble the input and output paths and create the Swift compiler command.

Sources/SWBCore/WorkspaceContext.swift

@@ -352,7 +352,7 @@ public final class WorkspaceContext: Sendable {
     private let headerIndexCache = AsyncSingleValueCache<WorkspaceHeaderIndex, Never>()
 
     public var buildDirectoryMacros: [PathMacroDeclaration] {
-        return [BuiltinMacros.DSTROOT, BuiltinMacros.OBJROOT, BuiltinMacros.SYMROOT, BuiltinMacros.BUILT_PRODUCTS_DIR, BuiltinMacros.EAGER_LINKING_INTERMEDIATE_TBD_DIR]
+        return [BuiltinMacros.DSTROOT, BuiltinMacros.OBJROOT, BuiltinMacros.SYMROOT, BuiltinMacros.BUILT_PRODUCTS_DIR, BuiltinMacros.EAGER_LINKING_INTERMEDIATE_TBD_DIR, BuiltinMacros.SWIFT_EXPLICIT_MODULES_OUTPUT_PATH, BuiltinMacros.CLANG_EXPLICIT_MODULES_OUTPUT_PATH]
     }
 
     /// The path to the module session file for the workspace given a set of build parameters.

Sources/SWBTaskExecution/BuildDescriptionManager.swift

@@ -198,8 +198,8 @@ package final class BuildDescriptionManager: Sendable {
 
             moduleCachePathsPerTarget[target] = [
                 settings.globalScope.evaluate(BuiltinMacros.MODULE_CACHE_DIR),
-                Path(settings.globalScope.evaluate(BuiltinMacros.SWIFT_EXPLICIT_MODULES_OUTPUT_PATH)),
-                Path(settings.globalScope.evaluate(BuiltinMacros.CLANG_EXPLICIT_MODULES_OUTPUT_PATH)),
+                settings.globalScope.evaluate(BuiltinMacros.SWIFT_EXPLICIT_MODULES_OUTPUT_PATH),
+                settings.globalScope.evaluate(BuiltinMacros.CLANG_EXPLICIT_MODULES_OUTPUT_PATH),
             ]
 
             if shouldValidateCAS, settings.globalScope.evaluate(BuiltinMacros.CLANG_ENABLE_COMPILE_CACHE) || settings.globalScope.evaluate(BuiltinMacros.SWIFT_ENABLE_COMPILE_CACHE) {

Tests/SWBBuildSystemTests/CleanOperationTests.swift

@@ -106,7 +106,7 @@ fileprivate struct CleanOperationTests: CoreBasedTests {
     @Test(.requireSDKs(.macOS))
     func cleanFramework() async throws {
         try await withTestHarness { tester, tmpDirPath, _ in
-            let buildFolderPaths = [ tmpDirPath.join("Test/aProject/build"), tmpDirPath.join("Test/aProject/build/Debug"), tmpDirPath.join("Test/aProject/build/EagerLinkingTBDs/Debug")]
+            let buildFolderPaths = [ tmpDirPath.join("Test/aProject/build"), tmpDirPath.join("Test/aProject/build/Debug"), tmpDirPath.join("Test/aProject/build/EagerLinkingTBDs/Debug"), tmpDirPath.join("Test/aProject/build/ExplicitPrecompiledModules"), tmpDirPath.join("Test/aProject/build/SwiftExplicitPrecompiledModules")]
 
             try await tester.checkBuild(runDestination: .macOS, persistent: true) { results in
                 // Check if build folder tasks have run as expected.
@@ -136,7 +136,7 @@ fileprivate struct CleanOperationTests: CoreBasedTests {
     @Test(.requireSDKs(.macOS))
     func cleanFrameworkInstall() async throws {
         try await withTestHarness(install: true) { tester, tmpDirPath, dstRoot in
-            let buildFolderPaths = [ dstRoot, tmpDirPath.join("Test/aProject/build"), tmpDirPath.join("Test/aProject/build/Debug"), tmpDirPath.join("Test/aProject/build/EagerLinkingTBDs/Debug")]
+            let buildFolderPaths = [ dstRoot, tmpDirPath.join("Test/aProject/build"), tmpDirPath.join("Test/aProject/build/Debug"), tmpDirPath.join("Test/aProject/build/EagerLinkingTBDs/Debug"), tmpDirPath.join("Test/aProject/build/ExplicitPrecompiledModules"), tmpDirPath.join("Test/aProject/build/SwiftExplicitPrecompiledModules")]
 
             try await tester.checkBuild(runDestination: .macOS, persistent: true) { results in
                 // Check if build folder tasks have run as expected.
@@ -212,7 +212,7 @@ fileprivate struct CleanOperationTests: CoreBasedTests {
             let buildRequest = BuildRequest(parameters: parameters, buildTargets: buildTargets, continueBuildingAfterErrors: true, useParallelTargets: true, useImplicitDependencies: false, useDryRun: false)
             try await tester.checkBuild(runDestination: .macOS, buildRequest: buildRequest, persistent: true) { results in
                 results.checkTasks(.matchRuleType("CreateBuildDirectory")) { tasks in
-                    #expect(tasks.count == 6)
+                    #expect(tasks.count == 10)
                 }
 
                 results.checkNoTask(.matchRuleType("CreateBuildDirectory"))
@@ -234,7 +234,7 @@ fileprivate struct CleanOperationTests: CoreBasedTests {
     @Test(.requireSDKs(.macOS))
     func cleanDoesNotDeleteManuallyCreatedFolders() async throws {
         try await withTestHarness { tester, tmpDirPath, _ in
-            let buildFolderPaths = [ tmpDirPath.join("Test/aProject/build"), tmpDirPath.join("Test/aProject/build/Debug"), tmpDirPath.join("Test/aProject/build/EagerLinkingTBDs/Debug")]
+            let buildFolderPaths = [ tmpDirPath.join("Test/aProject/build"), tmpDirPath.join("Test/aProject/build/Debug"), tmpDirPath.join("Test/aProject/build/EagerLinkingTBDs/Debug"), tmpDirPath.join("Test/aProject/build/ExplicitPrecompiledModules"), tmpDirPath.join("Test/aProject/build/SwiftExplicitPrecompiledModules")]
 
             for folder in buildFolderPaths {
                 try tester.fs.createDirectory(folder, recursive: true)

Tests/SWBBuildSystemTests/PackageBuildOperationTests.swift

@@ -592,11 +592,15 @@ fileprivate struct PackageBuildOperationTests: CoreBasedTests {
                 packageBuildDirectory.join(configurationToBuild).str,
                 packageBuildDirectory.join(configurationToBuild).join("PackageFrameworks").str,
                 packageBuildDirectory.join("EagerLinkingTBDs").join(configurationToBuild).str,
+                packageBuildDirectory.join("ExplicitPrecompiledModules").str,
+                packageBuildDirectory.join("SwiftExplicitPrecompiledModules").str,
 
                 projectBuildDirectory.str,
                 projectBuildDirectory.join(configurationToBuild).str,
                 projectBuildDirectory.join(configurationToBuild).join("PackageFrameworks").str,
                 projectBuildDirectory.join("EagerLinkingTBDs").join(configurationToBuild).str,
+                projectBuildDirectory.join("ExplicitPrecompiledModules").str,
+                projectBuildDirectory.join("SwiftExplicitPrecompiledModules").str,
             ].sorted()
 
             let tester = try await testerForBasicPackageProject(tmpDirPath: tmpDirPath, configurationToBuild: configurationToBuild)

Tests/SWBCoreTests/SettingsTests.swift

@@ -396,7 +396,7 @@ import SWBMacro
                 if let project = settings.project {
                     #expect(settings.globalScope.evaluate(BuiltinMacros.OBJROOT) == Path("\(project.sourceRoot.str)/build"))
                     #expect(settings.globalScope.evaluate(BuiltinMacros.CONFIGURATION_BUILD_DIR) == Path("\(project.sourceRoot.str)/build/Config1"))
-                    #expect(settings.globalScope.evaluate(BuiltinMacros.CLANG_EXPLICIT_MODULES_OUTPUT_PATH) == "\(project.sourceRoot.str)/build/ExplicitPrecompiledModules")
+                    #expect(settings.globalScope.evaluate(BuiltinMacros.CLANG_EXPLICIT_MODULES_OUTPUT_PATH).str == "\(project.sourceRoot.str)/build/ExplicitPrecompiledModules")
                 }
 
                 // check that we get the right value for VERSION_INFO_STRING, which validates we parsed it correctly.

Tests/SWBTaskConstructionTests/TaskConstructionTests.swift

@@ -2136,6 +2136,8 @@ fileprivate struct TaskConstructionTests: CoreBasedTests {
                             .namePattern(.prefix("CreateBuildDirectory-/tmp/Test/aProject/build")),
                             .namePattern(.prefix("CreateBuildDirectory-/tmp/Test/aProject/build/Release\(runDestination.builtProductsDirSuffix)/BuiltProducts")),
                             .namePattern(.prefix("CreateBuildDirectory-/tmp/Test/aProject/build/EagerLinkingTBDs")),
+                            .namePattern(.prefix("CreateBuildDirectory-/tmp/Test/aProject/build/SwiftExplicitPrecompiledModules")),
+                            .namePattern(.prefix("CreateBuildDirectory-/tmp/Test/aProject/build/ExplicitPrecompiledModules")),
                             .namePattern(.and(.prefix("target"), .suffix("-begin-compiling"))),
                         ])
                     } else {
@@ -8260,6 +8262,30 @@ fileprivate struct TaskConstructionTests: CoreBasedTests {
                     task.checkInputs([.path("\(tmpDir.str)/build/a/b/c/d")])
                 }
 
+                results.checkTask(.matchRuleType("CreateBuildDirectory"), .matchRuleItem("\(tmpDir.str)/build/a/b/c/d/ExplicitPrecompiledModules")) { task in
+                    task.checkInputs([.path("\(tmpDir.str)/build/a/b/c/d")])
+                }
+
+                results.checkTask(.matchRuleType("CreateBuildDirectory"), .matchRuleItem("\(tmpDir.str)/build/a/b/c/d/SwiftExplicitPrecompiledModules")) { task in
+                    task.checkInputs([.path("\(tmpDir.str)/build/a/b/c/d")])
+                }
+
+                results.checkTask(.matchRuleType("CreateBuildDirectory"), .matchRuleItem("\(tmpDir.str)/build/a/b/ExplicitPrecompiledModules")) { task in
+                    task.checkInputs([.path("\(tmpDir.str)/build/a/b")])
+                }
+
+                results.checkTask(.matchRuleType("CreateBuildDirectory"), .matchRuleItem("\(tmpDir.str)/build/a/b/SwiftExplicitPrecompiledModules")) { task in
+                    task.checkInputs([.path("\(tmpDir.str)/build/a/b")])
+                }
+
+                results.checkTask(.matchRuleType("CreateBuildDirectory"), .matchRuleItem("\(tmpDir.str)/build/a/ExplicitPrecompiledModules")) { task in
+                    task.checkInputs([.path("\(tmpDir.str)/build/a")])
+                }
+
+                results.checkTask(.matchRuleType("CreateBuildDirectory"), .matchRuleItem("\(tmpDir.str)/build/a/SwiftExplicitPrecompiledModules")) { task in
+                    task.checkInputs([.path("\(tmpDir.str)/build/a")])
+                }
+
                 results.checkTask(.matchRuleType("CreateBuildDirectory"), .matchRuleItem("\(tmpDir.str)/build/a/b/c/d")) { task in
                     task.checkInputs([.path("\(tmpDir.str)/build/a/b")])
                 }