Add driver implementation WIP

Package.swift

@@ -32,6 +32,10 @@ let package = Package(
             name: "xcswiftc",
             dependencies: ["XCRemoteCache"]
         ),
+        .target(
+            name: "xcswift-frontend",
+            dependencies: ["XCRemoteCache"]
+        ),
         .target(
             name: "xclibtoolSupport",
             dependencies: ["XCRemoteCache"]
@@ -69,6 +73,7 @@ let package = Package(
             dependencies: [
                 "xcprebuild",
                 "xcswiftc",
+                "xcswift-frontend",
                 "xclibtool",
                 "xcpostbuild",
                 "xcprepare",

README.md

@@ -359,6 +359,7 @@ Note: This step is not required if at least one of these is true:
 | `custom_rewrite_envs` | A list of extra ENVs that should be used as placeholders in the dependency list. ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process. | `[]` | ⬜️ |
 | `irrelevant_dependencies_paths` | Regexes of files that should not be included in a list of dependencies. Warning! Add entries here with caution - excluding dependencies that are relevant might lead to a target overcaching. The regex can match either partially or fully the filepath, e.g. `\\.modulemap$` will exclude all `.modulemap` files. | `[]` | ⬜️ |
 | `gracefully_handle_missing_common_sha` | If true, do not fail `prepare` if cannot find the most recent common commits with the primary branch. That might be useful on CI, where a shallow clone is used and cloning depth is not big enough to fetch a commit from a primary branch | `false` | ⬜️ |
+| `enable_swift_driver_integration` | Enable experimental integration with swift driver, added in Xcode 14 | `false` | ⬜️ |
 
 ## Backend cache server
 

Rakefile

@@ -10,7 +10,7 @@ DERIVED_DATA_DIR = File.join('.build').freeze
 RELEASES_ROOT_DIR = File.join('releases').freeze
 
 EXECUTABLE_NAME = 'XCRemoteCache'
-EXECUTABLE_NAMES = ['xclibtool', 'xcpostbuild', 'xcprebuild', 'xcprepare', 'xcswiftc', 'xcld', 'xcldplusplus', 'xclipo']
+EXECUTABLE_NAMES = ['xclibtool', 'xcpostbuild', 'xcprebuild', 'xcprepare', 'xcswiftc', 'swiftc', 'xcswift-frontend', 'swift-frontend', 'xcld', 'xcldplusplus', 'xclipo']
 PROJECT_NAME = 'XCRemoteCache'
 
 SWIFTLINT_ENABLED = true
@@ -59,6 +59,10 @@ task :build, [:configuration, :arch, :sdks, :is_archive] do |task, args|
 
     # Path of the executable looks like: `.build/(debug|release)/XCRemoteCache`
     build_path_base = File.join(DERIVED_DATA_DIR, args.configuration)
+    # swift-frontent integration requires that the SWIFT_EXEC is `swiftc` so create
+    # a symbolic link between swiftc->xcswiftc and swift-frontend->xcswift-frontend
+    system("cd #{build_path_base} && ln -s xcswiftc swiftc")
+    system("cd #{build_path_base} && ln -s xcswift-frontend swift-frontend")
     sdk_build_paths = EXECUTABLE_NAMES.map {|e| File.join(build_path_base, e)}
 
     build_paths.push(sdk_build_paths)
@@ -130,7 +134,9 @@ def create_release_zip(build_paths)
   # Create and move files into the release directory
   mkdir_p release_dir
   build_paths.each {|p|
-    cp_r p, release_dir
+    # -r for recursive
+    # -P for copying symbolic link as is
+    system("cp -rP #{p} #{release_dir}")
   }
 
   output_artifact_basename = "#{PROJECT_NAME}.zip"
@@ -139,7 +145,8 @@ def create_release_zip(build_paths)
     # -X: no extras (uid, gid, file times, ...)
     # -x: exclude .DS_Store
     # -r: recursive
-    system("zip -X -x '*.DS_Store' -r #{output_artifact_basename} .") or abort "zip failure"
+    # -y: to store symbolic links (used for swiftc -> xcswiftc)
+    system("zip -X -x '*.DS_Store' -r -y #{output_artifact_basename} .") or abort "zip failure"
     # List contents of zip file
     system("unzip -l #{output_artifact_basename}") or abort "unzip failure"
   end

Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift

@@ -89,6 +89,9 @@ public struct PostbuildContext {
     var publicHeadersFolderPath: URL?
     /// XCRemoteCache is explicitly disabled
     let disabled: Bool
+    /// The LLBUILD_BUILD_ID ENV that describes the compilation identifier
+    /// it is used in the swift-frontend flow
+    let llbuildIdLockFile: URL
 }
 
 extension PostbuildContext {
@@ -149,5 +152,7 @@ extension PostbuildContext {
             publicHeadersFolderPath = builtProductsDir.appendingPathComponent(publicHeadersPath)
         }
         disabled = try env.readEnv(key: "XCRC_DISABLED") ?? false
+        let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID")
+        llbuildIdLockFile = XCSwiftFrontend.generateLlbuildIdSharedLockUrl(llbuildId: llbuildId, tmpDir: targetTempDir)
     }
 }

Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift

@@ -60,6 +60,7 @@ public class XCPostbuild {
             dependenciesWriter: FileDependenciesWriter.init,
             dependenciesReader: FileDependenciesReader.init,
             markerWriter: NoopMarkerWriter.init,
+            llbuildLockFile: context.llbuildIdLockFile,
             fileManager: fileManager
         )
 

Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift

@@ -48,6 +48,9 @@ public struct PrebuildContext {
     let overlayHeadersPath: URL
     /// XCRemoteCache is explicitly disabled
     let disabled: Bool
+    /// The LLBUILD_BUILD_ID ENV that describes the compilation identifier
+    /// it is used in the swift-frontend flow
+    let llbuildIdLockFile: URL
 }
 
 extension PrebuildContext {
@@ -72,5 +75,7 @@ extension PrebuildContext {
         /// Note: The file has yaml extension, even it is in the json format
         overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml")
         disabled = try env.readEnv(key: "XCRC_DISABLED") ?? false
+        let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID")
+        llbuildIdLockFile = XCSwiftFrontend.generateLlbuildIdSharedLockUrl(llbuildId: llbuildId, tmpDir: targetTempDir)
     }
 }

Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift

@@ -55,6 +55,7 @@ public class XCPrebuild {
             dependenciesWriter: FileDependenciesWriter.init,
             dependenciesReader: FileDependenciesReader.init,
             markerWriter: lazyMarkerWriterFactory,
+            llbuildLockFile: context.llbuildIdLockFile,
             fileManager: fileManager
         )
 

Sources/XCRemoteCache/Commands/Prepare/Integrate/IntegrateContext.swift

@@ -27,36 +27,44 @@ struct IntegrateContext {
     let configOverride: URL
     let fakeSrcRoot: URL
     let output: URL?
+    let buildSettingsAppenderOptions: BuildSettingsIntegrateAppenderOption
 }
 
 extension IntegrateContext {
     init(
         input: String,
-        repoRootPath: String,
+        config: XCRemoteCacheConfig,
         mode: Mode,
-        configOverridePath: String,
         env: [String: String],
         binariesDir: URL,
         fakeSrcRoot: String,
         outputPath: String?
     ) throws {
         projectPath = URL(fileURLWithPath: input)
         let srcRoot = projectPath.deletingLastPathComponent()
-        repoRoot = URL(fileURLWithPath: repoRootPath, relativeTo: srcRoot)
+        repoRoot = URL(fileURLWithPath: config.repoRoot, relativeTo: srcRoot)
         self.mode = mode
-        configOverride = URL(fileURLWithPath: configOverridePath, relativeTo: srcRoot)
+        configOverride = URL(fileURLWithPath: config.extraConfigurationFile, relativeTo: srcRoot)
         output = outputPath.flatMap(URL.init(fileURLWithPath:))
         self.fakeSrcRoot = URL(fileURLWithPath: fakeSrcRoot)
+        var swiftcBinaryName = "swiftc"
+        var buildSettingsAppenderOptions: BuildSettingsIntegrateAppenderOption = []
+        // Keep the legacy behaviour (supported in Xcode 14 and lower)
+        if !config.enableSwifDriverIntegration {
+            buildSettingsAppenderOptions.insert(.disableSwiftDriverIntegration)
+            swiftcBinaryName = "xcswiftc"
+        }
         binaries = XCRCBinariesPaths(
             prepare: binariesDir.appendingPathComponent("xcprepare"),
             cc: binariesDir.appendingPathComponent("xccc"),
-            swiftc: binariesDir.appendingPathComponent("xcswiftc"),
+            swiftc: binariesDir.appendingPathComponent(swiftcBinaryName),
             libtool: binariesDir.appendingPathComponent("xclibtool"),
             lipo: binariesDir.appendingPathComponent("xclipo"),
             ld: binariesDir.appendingPathComponent("xcld"),
             ldplusplus: binariesDir.appendingPathComponent("xcldplusplus"),
             prebuild: binariesDir.appendingPathComponent("xcprebuild"),
             postbuild: binariesDir.appendingPathComponent("xcpostbuild")
         )
+        self.buildSettingsAppenderOptions = buildSettingsAppenderOptions
     }
 }

Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift

@@ -82,9 +82,8 @@ public class XCIntegrate {
 
             let context = try IntegrateContext(
                 input: projectPath,
-                repoRootPath: config.repoRoot,
+                config: config,
                 mode: mode,
-                configOverridePath: config.extraConfigurationFile,
                 env: env,
                 binariesDir: binariesDir,
                 fakeSrcRoot: fakeSrcRoot,
@@ -98,15 +97,12 @@ public class XCIntegrate {
                 excludes: targetsExclude.integrateArrayArguments,
                 includes: targetsInclude.integrateArrayArguments
             )
-            let buildSettingsAppenderOptions: BuildSettingsIntegrateAppenderOption = [
-                .disableSwiftDriverIntegration,
-            ]
             let buildSettingsAppender = XcodeProjBuildSettingsIntegrateAppender(
                 mode: context.mode,
                 repoRoot: context.repoRoot,
                 fakeSrcRoot: context.fakeSrcRoot,
                 sdksExclude: sdksExclude.integrateArrayArguments,
-                options: buildSettingsAppenderOptions
+                options: context.buildSettingsAppenderOptions
             )
             let lldbPatcher: LLDBInitPatcher
             switch lldbMode {