diff --git a/Package.swift b/Package.swift index 426a40d9..00c88e43 100644 --- a/Package.swift +++ b/Package.swift @@ -32,6 +32,10 @@ let package = Package( name: "xcswiftc", dependencies: ["XCRemoteCache"] ), + .target( + name: "xcswift-frontend", + dependencies: ["XCRemoteCache"] + ), .target( name: "xclibtoolSupport", dependencies: ["XCRemoteCache"] @@ -69,6 +73,7 @@ let package = Package( dependencies: [ "xcprebuild", "xcswiftc", + "xcswift-frontend", "xclibtool", "xcpostbuild", "xcprepare", diff --git a/README.md b/README.md index 3e2dd1e2..72f9e8f9 100755 --- a/README.md +++ b/README.md @@ -359,6 +359,7 @@ Note: This step is not required if at least one of these is true: | `custom_rewrite_envs` | A list of extra ENVs that should be used as placeholders in the dependency list. ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process. | `[]` | ⬜️ | | `irrelevant_dependencies_paths` | Regexes of files that should not be included in a list of dependencies. Warning! Add entries here with caution - excluding dependencies that are relevant might lead to a target overcaching. The regex can match either partially or fully the filepath, e.g. `\\.modulemap$` will exclude all `.modulemap` files. | `[]` | ⬜️ | | `gracefully_handle_missing_common_sha` | If true, do not fail `prepare` if cannot find the most recent common commits with the primary branch. That might be useful on CI, where a shallow clone is used and cloning depth is not big enough to fetch a commit from a primary branch | `false` | ⬜️ | +| `enable_swift_driver_integration` | Enable experimental integration with swift driver, added in Xcode 14 | `false` | ⬜️ | ## Backend cache server diff --git a/Rakefile b/Rakefile index 3934c77f..45a4e867 100644 --- a/Rakefile +++ b/Rakefile @@ -10,7 +10,7 @@ DERIVED_DATA_DIR = File.join('.build').freeze RELEASES_ROOT_DIR = File.join('releases').freeze EXECUTABLE_NAME = 'XCRemoteCache' -EXECUTABLE_NAMES = ['xclibtool', 'xcpostbuild', 'xcprebuild', 'xcprepare', 'xcswiftc', 'xcld', 'xcldplusplus', 'xclipo'] +EXECUTABLE_NAMES = ['xclibtool', 'xcpostbuild', 'xcprebuild', 'xcprepare', 'xcswiftc', 'swiftc', 'xcswift-frontend', 'swift-frontend', 'xcld', 'xcldplusplus', 'xclipo'] PROJECT_NAME = 'XCRemoteCache' SWIFTLINT_ENABLED = true @@ -59,6 +59,10 @@ task :build, [:configuration, :arch, :sdks, :is_archive] do |task, args| # Path of the executable looks like: `.build/(debug|release)/XCRemoteCache` build_path_base = File.join(DERIVED_DATA_DIR, args.configuration) + # swift-frontent integration requires that the SWIFT_EXEC is `swiftc` so create + # a symbolic link between swiftc->xcswiftc and swift-frontend->xcswift-frontend + system("cd #{build_path_base} && ln -s xcswiftc swiftc") + system("cd #{build_path_base} && ln -s xcswift-frontend swift-frontend") sdk_build_paths = EXECUTABLE_NAMES.map {|e| File.join(build_path_base, e)} build_paths.push(sdk_build_paths) @@ -130,7 +134,9 @@ def create_release_zip(build_paths) # Create and move files into the release directory mkdir_p release_dir build_paths.each {|p| - cp_r p, release_dir + # -r for recursive + # -P for copying symbolic link as is + system("cp -rP #{p} #{release_dir}") } output_artifact_basename = "#{PROJECT_NAME}.zip" @@ -139,7 +145,8 @@ def create_release_zip(build_paths) # -X: no extras (uid, gid, file times, ...) # -x: exclude .DS_Store # -r: recursive - system("zip -X -x '*.DS_Store' -r #{output_artifact_basename} .") or abort "zip failure" + # -y: to store symbolic links (used for swiftc -> xcswiftc) + system("zip -X -x '*.DS_Store' -r -y #{output_artifact_basename} .") or abort "zip failure" # List contents of zip file system("unzip -l #{output_artifact_basename}") or abort "unzip failure" end diff --git a/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift b/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift index d71ea9ae..f781d617 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/PostbuildContext.swift @@ -89,6 +89,9 @@ public struct PostbuildContext { var publicHeadersFolderPath: URL? /// XCRemoteCache is explicitly disabled let disabled: Bool + /// The LLBUILD_BUILD_ID ENV that describes the compilation identifier + /// it is used in the swift-frontend flow + let llbuildIdLockFile: URL } extension PostbuildContext { @@ -149,5 +152,7 @@ extension PostbuildContext { publicHeadersFolderPath = builtProductsDir.appendingPathComponent(publicHeadersPath) } disabled = try env.readEnv(key: "XCRC_DISABLED") ?? false + let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID") + llbuildIdLockFile = XCSwiftFrontend.generateLlbuildIdSharedLockUrl(llbuildId: llbuildId, tmpDir: targetTempDir) } } diff --git a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift index cc597bf5..235f56c1 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift @@ -60,6 +60,7 @@ public class XCPostbuild { dependenciesWriter: FileDependenciesWriter.init, dependenciesReader: FileDependenciesReader.init, markerWriter: NoopMarkerWriter.init, + llbuildLockFile: context.llbuildIdLockFile, fileManager: fileManager ) diff --git a/Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift b/Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift index d36bcfaf..f1de7e05 100644 --- a/Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift +++ b/Sources/XCRemoteCache/Commands/Prebuild/PrebuildContext.swift @@ -48,6 +48,9 @@ public struct PrebuildContext { let overlayHeadersPath: URL /// XCRemoteCache is explicitly disabled let disabled: Bool + /// The LLBUILD_BUILD_ID ENV that describes the compilation identifier + /// it is used in the swift-frontend flow + let llbuildIdLockFile: URL } extension PrebuildContext { @@ -72,5 +75,7 @@ extension PrebuildContext { /// Note: The file has yaml extension, even it is in the json format overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml") disabled = try env.readEnv(key: "XCRC_DISABLED") ?? false + let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID") + llbuildIdLockFile = XCSwiftFrontend.generateLlbuildIdSharedLockUrl(llbuildId: llbuildId, tmpDir: targetTempDir) } } diff --git a/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift b/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift index 97fc759a..b4c8dd8c 100644 --- a/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift +++ b/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift @@ -55,6 +55,7 @@ public class XCPrebuild { dependenciesWriter: FileDependenciesWriter.init, dependenciesReader: FileDependenciesReader.init, markerWriter: lazyMarkerWriterFactory, + llbuildLockFile: context.llbuildIdLockFile, fileManager: fileManager ) diff --git a/Sources/XCRemoteCache/Commands/Prepare/Integrate/IntegrateContext.swift b/Sources/XCRemoteCache/Commands/Prepare/Integrate/IntegrateContext.swift index aec0450c..2acb7a6c 100644 --- a/Sources/XCRemoteCache/Commands/Prepare/Integrate/IntegrateContext.swift +++ b/Sources/XCRemoteCache/Commands/Prepare/Integrate/IntegrateContext.swift @@ -27,14 +27,14 @@ struct IntegrateContext { let configOverride: URL let fakeSrcRoot: URL let output: URL? + let buildSettingsAppenderOptions: BuildSettingsIntegrateAppenderOption } extension IntegrateContext { init( input: String, - repoRootPath: String, + config: XCRemoteCacheConfig, mode: Mode, - configOverridePath: String, env: [String: String], binariesDir: URL, fakeSrcRoot: String, @@ -42,15 +42,22 @@ extension IntegrateContext { ) throws { projectPath = URL(fileURLWithPath: input) let srcRoot = projectPath.deletingLastPathComponent() - repoRoot = URL(fileURLWithPath: repoRootPath, relativeTo: srcRoot) + repoRoot = URL(fileURLWithPath: config.repoRoot, relativeTo: srcRoot) self.mode = mode - configOverride = URL(fileURLWithPath: configOverridePath, relativeTo: srcRoot) + configOverride = URL(fileURLWithPath: config.extraConfigurationFile, relativeTo: srcRoot) output = outputPath.flatMap(URL.init(fileURLWithPath:)) self.fakeSrcRoot = URL(fileURLWithPath: fakeSrcRoot) + var swiftcBinaryName = "swiftc" + var buildSettingsAppenderOptions: BuildSettingsIntegrateAppenderOption = [] + // Keep the legacy behaviour (supported in Xcode 14 and lower) + if !config.enableSwifDriverIntegration { + buildSettingsAppenderOptions.insert(.disableSwiftDriverIntegration) + swiftcBinaryName = "xcswiftc" + } binaries = XCRCBinariesPaths( prepare: binariesDir.appendingPathComponent("xcprepare"), cc: binariesDir.appendingPathComponent("xccc"), - swiftc: binariesDir.appendingPathComponent("xcswiftc"), + swiftc: binariesDir.appendingPathComponent(swiftcBinaryName), libtool: binariesDir.appendingPathComponent("xclibtool"), lipo: binariesDir.appendingPathComponent("xclipo"), ld: binariesDir.appendingPathComponent("xcld"), @@ -58,5 +65,6 @@ extension IntegrateContext { prebuild: binariesDir.appendingPathComponent("xcprebuild"), postbuild: binariesDir.appendingPathComponent("xcpostbuild") ) + self.buildSettingsAppenderOptions = buildSettingsAppenderOptions } } diff --git a/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift b/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift index 4b435cc5..e649b01b 100644 --- a/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift +++ b/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift @@ -82,9 +82,8 @@ public class XCIntegrate { let context = try IntegrateContext( input: projectPath, - repoRootPath: config.repoRoot, + config: config, mode: mode, - configOverridePath: config.extraConfigurationFile, env: env, binariesDir: binariesDir, fakeSrcRoot: fakeSrcRoot, @@ -98,15 +97,12 @@ public class XCIntegrate { excludes: targetsExclude.integrateArrayArguments, includes: targetsInclude.integrateArrayArguments ) - let buildSettingsAppenderOptions: BuildSettingsIntegrateAppenderOption = [ - .disableSwiftDriverIntegration, - ] let buildSettingsAppender = XcodeProjBuildSettingsIntegrateAppender( mode: context.mode, repoRoot: context.repoRoot, fakeSrcRoot: context.fakeSrcRoot, sdksExclude: sdksExclude.integrateArrayArguments, - options: buildSettingsAppenderOptions + options: context.buildSettingsAppenderOptions ) let lldbPatcher: LLDBInitPatcher switch lldbMode { diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendArgInput.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendArgInput.swift new file mode 100644 index 00000000..61b9873e --- /dev/null +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendArgInput.swift @@ -0,0 +1,243 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +enum SwiftFrontendArgInputError: Error, Equatable { + // swift-frontend should either be compling or emiting a module + case bothCompilationAndEmitAction + // no .swift files have been passed as input files + case noCompilationInputs + // no -primary-file .swift files have been passed as input files + case noPrimaryFileCompilationInputs + // number of -emit-dependencies-path doesn't match compilation inputs + case dependenciesOuputCountDoesntMatch(expected: Int, parsed: Int) + // number of -serialize-diagnostics-path doesn't match compilation inputs + case diagnosticsOuputCountDoesntMatch(expected: Int, parsed: Int) + // number of -o doesn't match compilation inputs + case outputsOuputCountDoesntMatch(expected: Int, parsed: Int) + // number of -o for emit-module can be only 1 + case emitModulOuputCountIsNot1(parsed: Int) + // number of -emit-dependencies-path for emit-module can be 0 or 1 (generate or not) + case emitModuleDependenciesOuputCountIsHigherThan1(parsed: Int) + // number of -serialize-diagnostics-path for emit-module can be 0 or 1 (generate or not) + case emitModuleDiagnosticsOuputCountIsHigherThan1(parsed: Int) + // emit-module requires -emit-objc-header-path + case emitModuleMissingObjcHeaderPath + // -target is required + case emitMissingTarget + // -moduleName is required + case emiMissingModuleName +} + +public struct SwiftFrontendArgInput { + let compile: Bool + let emitModule: Bool + let objcHeaderOutput: String? + let moduleName: String? + let target: String? + let primaryInputPaths: [String] + let inputPaths: [String] + var outputPaths: [String] + var dependenciesPaths: [String] + // Extra params + // Diagnostics are not supported yet in the XCRemoteCache (cached artifacts assumes no warnings) + var diagnosticsPaths: [String] + // Unsed for now: + // .swiftsourceinfo and .swiftdoc will be placed next to the .swiftmodule + let sourceInfoPath: String? + let docPath: String? + let supplementaryOutputFileMap: String? + + /// Manual initializer implementation required to be public + public init( + compile: Bool, + emitModule: Bool, + objcHeaderOutput: String?, + moduleName: String?, + target: String?, + primaryInputPaths: [String], + inputPaths: [String], + outputPaths: [String], + dependenciesPaths: [String], + diagnosticsPaths: [String], + sourceInfoPath: String?, + docPath: String?, + supplementaryOutputFileMap: String? + ) { + self.compile = compile + self.emitModule = emitModule + self.objcHeaderOutput = objcHeaderOutput + self.moduleName = moduleName + self.target = target + self.primaryInputPaths = primaryInputPaths + self.inputPaths = inputPaths + self.outputPaths = outputPaths + self.dependenciesPaths = dependenciesPaths + self.diagnosticsPaths = diagnosticsPaths + self.sourceInfoPath = sourceInfoPath + self.docPath = docPath + self.supplementaryOutputFileMap = supplementaryOutputFileMap + } + + private func generateForCompilation( + config: XCRemoteCacheConfig, + target: String, + moduleName: String + ) throws -> SwiftcContext { + let primaryInputsCount = primaryInputPaths.count + + guard primaryInputsCount > 0 else { + throw SwiftFrontendArgInputError.noPrimaryFileCompilationInputs + } + guard [primaryInputsCount, 0].contains(dependenciesPaths.count) else { + throw SwiftFrontendArgInputError.dependenciesOuputCountDoesntMatch( + expected: primaryInputsCount, + parsed: dependenciesPaths.count + ) + } + guard [primaryInputsCount, 0].contains(diagnosticsPaths.count) else { + throw SwiftFrontendArgInputError.diagnosticsOuputCountDoesntMatch( + expected: primaryInputsCount, + parsed: diagnosticsPaths.count + ) + } + guard outputPaths.count == primaryInputsCount else { + throw SwiftFrontendArgInputError.outputsOuputCountDoesntMatch( + expected: primaryInputsCount, + parsed: outputPaths.count + ) + } + let primaryInputFilesURLs: [URL] = primaryInputPaths.map(URL.init(fileURLWithPath:)) + + let steps: SwiftcContext.SwiftcSteps = SwiftcContext.SwiftcSteps( + compileFilesScope: .subset(primaryInputFilesURLs), + emitModule: nil + ) + + let compilationFilesInputs = buildCompilationFilesInputs( + primaryInputsCount: primaryInputsCount, + primaryInputFilesURLs: primaryInputFilesURLs + ) + + return try .init( + config: config, + moduleName: moduleName, + steps: steps, + inputs: compilationFilesInputs, + target: target, + compilationFiles: .list(inputPaths), + exampleWorkspaceFilePath: outputPaths[0] + ) + } + + private func buildCompilationFilesInputs( + primaryInputsCount: Int, + primaryInputFilesURLs: [URL] + ) -> SwiftcContext.CompilationFilesInputs { + if let compimentaryFileMa = supplementaryOutputFileMap { + return .supplementaryFileMap(compimentaryFileMa) + } else { + return .map((0.. SwiftcContext { + guard outputPaths.count == 1 else { + throw SwiftFrontendArgInputError.emitModulOuputCountIsNot1(parsed: outputPaths.count) + } + guard let objcHeaderOutput = objcHeaderOutput else { + throw SwiftFrontendArgInputError.emitModuleMissingObjcHeaderPath + } + guard diagnosticsPaths.count <= 1 else { + throw SwiftFrontendArgInputError.emitModuleDiagnosticsOuputCountIsHigherThan1( + parsed: diagnosticsPaths.count + ) + } + guard dependenciesPaths.count <= 1 else { + throw SwiftFrontendArgInputError.emitModuleDependenciesOuputCountIsHigherThan1( + parsed: dependenciesPaths.count + ) + } + + let steps: SwiftcContext.SwiftcSteps = SwiftcContext.SwiftcSteps( + compileFilesScope: .none, + emitModule: SwiftcContext.SwiftcStepEmitModule( + objcHeaderOutput: URL(fileURLWithPath: objcHeaderOutput), + modulePathOutput: URL(fileURLWithPath: outputPaths[0]), + dependencies: dependenciesPaths.first.map(URL.init(fileURLWithPath:)) + ) + ) + return try .init( + config: config, + moduleName: moduleName, + steps: steps, + inputs: .map([:]), + target: target, + compilationFiles: .list(inputPaths), + exampleWorkspaceFilePath: objcHeaderOutput + ) + } + + func generateSwiftcContext(config: XCRemoteCacheConfig) throws -> SwiftcContext { + guard compile != emitModule else { + throw SwiftFrontendArgInputError.bothCompilationAndEmitAction + } + let inputPathsCount = inputPaths.count + guard inputPathsCount > 0 else { + throw SwiftFrontendArgInputError.noCompilationInputs + } + guard let target = target else { + throw SwiftFrontendArgInputError.emitMissingTarget + } + guard let moduleName = moduleName else { + throw SwiftFrontendArgInputError.emiMissingModuleName + } + + if compile { + return try generateForCompilation( + config: config, + target: target, + moduleName: moduleName + ) + } else { + return try generateForEmitModule( + config: config, + target: target, + moduleName: moduleName + ) + } + } +} diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift new file mode 100644 index 00000000..7ecc4d97 --- /dev/null +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/SwiftFrontendOrchestrator.swift @@ -0,0 +1,129 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +/// Manages the `swift-frontend` logic +protocol SwiftFrontendOrchestrator { + /// Executes the criticial secion according to the required order + /// - Parameter criticalSection: the block that should be synchronized + func run(criticalSection: () -> Void ) throws +} + +/// The default orchestrator that manages the order or swift-frontend invocations +/// For emit-module (the "first" process) action, it locks a shared file between all swift-frontend invcations, +/// verifies that the mocking can be done and continues the mocking/fallbacking along the lock release +/// For the compilation action, tries to ackquire a lock and waits until the "emit-module" makes a decision +/// if the compilation should be skipped and a "mocking" should used instead +class CommonSwiftFrontendOrchestrator { + /// Content saved to the shared file + /// Safe to use forced unwrapping + private static let emitModuleContent = "done".data(using: .utf8)! + + enum Action { + case emitModule + case compile + } + private let mode: SwiftcContext.SwiftcMode + private let action: Action + private let lockAccessor: ExclusiveFileAccessor + private let maxLockTimeout: TimeInterval + + init( + mode: SwiftcContext.SwiftcMode, + action: Action, + lockAccessor: ExclusiveFileAccessor, + maxLockTimeout: TimeInterval + ) { + self.mode = mode + self.action = action + self.lockAccessor = lockAccessor + self.maxLockTimeout = maxLockTimeout + } + + func run(criticalSection: () throws -> Void) throws { + guard case .consumer(commit: .available) = mode else { + // no need to lock anything - just allow fallbacking to the `swiftc or swift-frontend` + // if we face producer or a consumer where RC is disabled (we have already caught the + // cache miss) + try criticalSection() + return + } + try executeMockAttemp(criticalSection: criticalSection) + } + + private func executeMockAttemp(criticalSection: () throws -> Void) throws { + switch action { + case .emitModule: + try validateEmitModuleStep(criticalSection: criticalSection) + case .compile: + try waitForEmitModuleLock(criticalSection: criticalSection) + } + } + + + /// Fro emit-module, wraps the critical section with the shared lock so other processes (compilation) + /// have to wait until it finishes + /// Once the emit-module is done, the "magical" string is saved to the file and the lock is released + /// + /// Note: The design of wrapping the entire "emit-module" has a small performance downside if inside + /// the critical section, the code realizes that remote cache cannot be used + /// (in practice - a new file has been added) + /// None of compilation process (so with '-c' args) can continue until the entire emit-module logic finishes + /// Because it is expected to happen no that often and emit-module is usually quite fast, this makes the + /// implementation way simpler. If we ever want to optimize it, we should release the lock as early + /// as we know, the remote cache cannot be used. Then all other compilation process (-c) can run + /// in parallel with emit-module + private func validateEmitModuleStep(criticalSection: () throws -> Void) throws { + try lockAccessor.exclusiveAccess { handle in + defer { + handle.write(Self.self.emitModuleContent) + } + do { + try criticalSection() + } + } + } + + /// Locks a shared file in a loop until its content non-empty, which means the "parent" emit-module has finished + private func waitForEmitModuleLock(criticalSection: () throws -> Void) throws { + // emit-module process should really quickly retain a lock (it is always invoked + // by Xcode as a first process) + var executed = false + let startingDate = Date() + while !executed { + try lockAccessor.exclusiveAccess { handle in + if !handle.availableData.isEmpty { + // the file is not empty so the emit-module process is done with the "check" + try criticalSection() + executed = true + } + } + // When a max locking time is achieved, execute anyway + if !executed && Date().timeIntervalSince(startingDate) > self.maxLockTimeout { + errorLog(""" + Executing command \(action) without lock synchronization. That may be cause by the\ + crashed or extremly long emit-module. Contact XCRemoteCache authors about this error. + """) + try criticalSection() + executed = true + } + } + } +} diff --git a/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift new file mode 100644 index 00000000..0821cc7d --- /dev/null +++ b/Sources/XCRemoteCache/Commands/SwiftFrontend/XCSwiftFrontend.swift @@ -0,0 +1,92 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +public class XCSwiftFrontend: XCSwiftAbstract { + // don't lock individual compilation invocations for more than 10s + private static let MaxLockingTimeout: TimeInterval = 10 + private let env: [String: String] + + public init( + command: String, + inputArgs: SwiftFrontendArgInput, + env: [String: String], + dependenciesWriter: @escaping (URL, FileManager) -> DependenciesWriter, + touchFactory: @escaping (URL, FileManager) -> Touch + ) throws { + self.env = env + super.init( + command: command, + inputArgs: inputArgs, + dependenciesWriter: dependenciesWriter, + touchFactory: touchFactory + ) + } + + override func buildContext() throws -> (XCRemoteCacheConfig, SwiftcContext) { + let fileManager = FileManager.default + let config: XCRemoteCacheConfig + let context: SwiftcContext + + let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath) + config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager) + .readConfiguration() + context = try SwiftcContext(config: config, input: inputArgs) + // do not cache this context, as it is subject to change when + // the emit-module finds that the cached artifact cannot be used + return (config, context) + } + + override public func run() throws { + do { + /// The LLBUILD_BUILD_ID ENV that describes the swiftc (parent) invocation + let llbuildId: String = try env.readEnv(key: "LLBUILD_BUILD_ID") + let (_, context) = try buildContext() + + let sharedLockFileURL = XCSwiftFrontend.generateLlbuildIdSharedLockUrl( + llbuildId: llbuildId, + tmpDir: context.tempDir + ) + let sharedLock = ExclusiveFile(sharedLockFileURL, mode: .override) + + let action: CommonSwiftFrontendOrchestrator.Action = inputArgs.emitModule ? .emitModule : .compile + let swiftFrontendOrchestrator = CommonSwiftFrontendOrchestrator( + mode: context.mode, + action: action, + lockAccessor: sharedLock, + maxLockTimeout: Self.self.MaxLockingTimeout + ) + + try swiftFrontendOrchestrator.run(criticalSection: super.run) + } catch { + // Splitting into 2 invocations as os_log truncates a massage + defaultLog("Cannot correctly orchestrate the \(command) with params \(inputArgs)") + defaultLog("Cannot correctly orchestrate error: \(error)") + throw error + } + } +} + +extension XCSwiftFrontend { + /// The file is used to sycnhronize mutliple swift-frontend invocations + static func generateLlbuildIdSharedLockUrl(llbuildId: String, tmpDir: URL) -> URL { + return tmpDir.appendingPathComponent(llbuildId).appendingPathExtension("lock") + } +} diff --git a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift index 485c9d18..6cd89e08 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/SwiftcContext.swift @@ -159,4 +159,11 @@ public struct SwiftcContext { exampleWorkspaceFilePath: input.modulePathOutput ) } + + init( + config: XCRemoteCacheConfig, + input: SwiftFrontendArgInput + ) throws { + self = try input.generateSwiftcContext(config: config) + } } diff --git a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift index 4ef7b77f..52410d7c 100644 --- a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift +++ b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift @@ -17,6 +17,8 @@ // specific language governing permissions and limitations // under the License. +// swiftlint:disable file_length + import Foundation import Yams @@ -57,6 +59,8 @@ public struct XCRemoteCacheConfig: Encodable { var clangCommand: String = "clang" /// Command for a standard Swift compilation (swiftc) var swiftcCommand: String = "swiftc" + /// Command for a standard Swift frontend compilation (swift-frontend) + var swiftFrontendCommand: String = "swift-frontend" /// Path of the primary repository that produces cache artifacts var primaryRepo: String = "" /// Main (primary) branch that produces cache artifacts (default to 'master') @@ -151,6 +155,8 @@ public struct XCRemoteCacheConfig: Encodable { /// If true, do not fail `prepare` if cannot find the most recent common commits with the primary branch /// That might useful on CI, where a shallow clone is used var gracefullyHandleMissingCommonSha: Bool = false + /// Enable experimental integration with swift driver, added in Xcode 14 + var enableSwifDriverIntegration: Bool = false } extension XCRemoteCacheConfig { @@ -211,6 +217,7 @@ extension XCRemoteCacheConfig { merge.irrelevantDependenciesPaths = scheme.irrelevantDependenciesPaths ?? irrelevantDependenciesPaths merge.gracefullyHandleMissingCommonSha = scheme.gracefullyHandleMissingCommonSha ?? gracefullyHandleMissingCommonSha + merge.enableSwifDriverIntegration = scheme.enableSwifDriverIntegration ?? enableSwifDriverIntegration return merge } @@ -279,6 +286,7 @@ struct ConfigFileScheme: Decodable { let customRewriteEnvs: [String]? let irrelevantDependenciesPaths: [String]? let gracefullyHandleMissingCommonSha: Bool? + let enableSwifDriverIntegration: Bool? // Yams library doesn't support encoding strategy, see https://github.com/jpsim/Yams/issues/84 enum CodingKeys: String, CodingKey { @@ -330,6 +338,7 @@ struct ConfigFileScheme: Decodable { case customRewriteEnvs = "custom_rewrite_envs" case irrelevantDependenciesPaths = "irrelevant_dependencies_paths" case gracefullyHandleMissingCommonSha = "gracefully_handle_missing_common_sha" + case enableSwifDriverIntegration = "enable_swift_driver_integration" } } diff --git a/Sources/XCRemoteCache/Dependencies/CacheModeController.swift b/Sources/XCRemoteCache/Dependencies/CacheModeController.swift index db8871c3..ddc50290 100644 --- a/Sources/XCRemoteCache/Dependencies/CacheModeController.swift +++ b/Sources/XCRemoteCache/Dependencies/CacheModeController.swift @@ -48,6 +48,7 @@ class PhaseCacheModeController: CacheModeController { private let dependenciesWriter: DependenciesWriter private let dependenciesReader: DependenciesReader private let markerWriter: MarkerWriter + private let llbuildLockFile: URL private let fileManager: FileManager init( @@ -59,6 +60,7 @@ class PhaseCacheModeController: CacheModeController { dependenciesWriter: (URL, FileManager) -> DependenciesWriter, dependenciesReader: (URL, FileManager) -> DependenciesReader, markerWriter: (URL, FileManager) -> MarkerWriter, + llbuildLockFile: URL, fileManager: FileManager ) { @@ -69,10 +71,12 @@ class PhaseCacheModeController: CacheModeController { let discoveryURL = tempDir.appendingPathComponent(phaseDependencyPath) self.dependenciesWriter = dependenciesWriter(discoveryURL, fileManager) self.dependenciesReader = dependenciesReader(discoveryURL, fileManager) + self.llbuildLockFile = llbuildLockFile self.markerWriter = markerWriter(modeMarker, fileManager) } func enable(allowedInputFiles: [URL], dependencies: [URL]) throws { + try cleanupLlBuildLock() // marker file contains filepaths that contribute to the build products // and should invalidate all other target steps (swiftc,libtool etc.) let targetSensitiveFiles = dependencies + [modeMarker, Self.xcodeSelectLink] @@ -84,6 +88,7 @@ class PhaseCacheModeController: CacheModeController { } func disable() throws { + try cleanupLlBuildLock() guard !forceCached else { throw PhaseCacheModeControllerError.cannotUseRemoteCacheForForcedCacheMode } @@ -114,4 +119,14 @@ class PhaseCacheModeController: CacheModeController { } return false } + + private func cleanupLlBuildLock() throws { + if fileManager.fileExists(atPath: llbuildLockFile.path) { + do { + try fileManager.removeItem(at: llbuildLockFile) + } catch { + printWarning("Removing llbuild lock at \(llbuildLockFile.path) failed. Error: \(error)") + } + } + } } diff --git a/Sources/XCRemoteCache/FlowControl/RemoteCommitInfo.swift b/Sources/XCRemoteCache/FlowControl/RemoteCommitInfo.swift index 094b43ac..5dc38af1 100644 --- a/Sources/XCRemoteCache/FlowControl/RemoteCommitInfo.swift +++ b/Sources/XCRemoteCache/FlowControl/RemoteCommitInfo.swift @@ -27,7 +27,7 @@ enum RemoteCommitInfo: Equatable { extension RemoteCommitInfo { init(_ commit: String?) { switch commit { - case .some(let value) where !value.isEmpty : + case .some(let value) where !value.isEmpty: self = .available(commit: value) default: self = .unavailable diff --git a/Sources/XCRemoteCache/Utils/Array+Utils.swift b/Sources/XCRemoteCache/Utils/Array+Utils.swift new file mode 100644 index 00000000..67e2e75a --- /dev/null +++ b/Sources/XCRemoteCache/Utils/Array+Utils.swift @@ -0,0 +1,29 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +public extension Array { + func get(_ i: Index) -> Element? { + guard count > i else { + return nil + } + return self[i] + } +} diff --git a/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift b/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift new file mode 100644 index 00000000..05141f49 --- /dev/null +++ b/Sources/xcswift-frontend/XCSwiftcFrontendMain.swift @@ -0,0 +1,135 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import XCRemoteCache + +/// Wrapper for a `swift-frontend` that skips compilation and +/// produces empty output files (.o). As a compilation dependencies +/// (.d) file, it copies all dependency files from the prebuild marker file +/// Fallbacks to a standard `swift-frontend` when the ramote cache is not applicable (e.g. modified sources) +public class XCSwiftcFrontendMain { + // swiftlint:disable:next function_body_length cyclomatic_complexity + public func main() { + let env = ProcessInfo.processInfo.environment + let command = ProcessInfo().processName + let args = ProcessInfo().arguments + var compile = false + var emitModule = false + var objcHeaderOutput: String? + var moduleName: String? + var target: String? + var inputPaths: [String] = [] + var primaryInputPaths: [String] = [] + var outputPaths: [String] = [] + var dependenciesPaths: [String] = [] + var diagnosticsPaths: [String] = [] + var sourceInfoPath: String? + var docPath: String? + var supplementaryOutputFileMap: String? + + for i in 0.. Never { + let developerDir = env["DEVELOPER_DIR"]! + // limitation: always using the Xcode's toolchain + let swiftFrontendCommand = "\(developerDir)/Toolchains/XcodeDefault.xctoolchain/usr/bin/swift-frontend" + + let args = ProcessInfo().arguments + let paramList = [swiftFrontendCommand] + args.dropFirst() + let cargs = paramList.map { strdup($0) } + [nil] + execvp(swiftFrontendCommand, cargs) + + /// C-function `execv` returns only when the command fails + exit(1) + } +} diff --git a/Sources/xcswift-frontend/main.swift b/Sources/xcswift-frontend/main.swift new file mode 100644 index 00000000..0bdf4a1a --- /dev/null +++ b/Sources/xcswift-frontend/main.swift @@ -0,0 +1,22 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import XCRemoteCache + +XCSwiftcFrontendMain().main() diff --git a/Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift b/Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift index be78500b..097567fc 100644 --- a/Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/PostbuildContextTests.swift @@ -28,7 +28,7 @@ class PostbuildContextTests: FileXCTestCase { "TARGET_TEMP_DIR": "TARGET_TEMP_DIR", "DERIVED_FILE_DIR": "DERIVED_FILE_DIR", "ARCHS": "x86_64", - "OBJECT_FILE_DIR_normal": "/OBJECT_FILE_DIR_normal" , + "OBJECT_FILE_DIR_normal": "/OBJECT_FILE_DIR_normal", "CONFIGURATION": "CONFIGURATION", "PLATFORM_NAME": "PLATFORM_NAME", "XCODE_PRODUCT_BUILD_VERSION": "XCODE_PRODUCT_BUILD_VERSION", @@ -45,6 +45,7 @@ class PostbuildContextTests: FileXCTestCase { "DERIVED_SOURCES_DIR": "DERIVED_SOURCES_DIR", "CURRENT_VARIANT": "normal", "PUBLIC_HEADERS_FOLDER_PATH": "/usr/local/include", + "LLBUILD_BUILD_ID": "1", ] override func setUpWithError() throws { @@ -186,4 +187,17 @@ class PostbuildContextTests: FileXCTestCase { XCTAssertFalse(context.disabled) } + + func testFailsIfLlBuildIdEnvIsMissing() throws { + var envs = Self.SampleEnvs + envs.removeValue(forKey: "LLBUILD_BUILD_ID") + + XCTAssertThrowsError(try PostbuildContext(config, env: envs)) + } + + func testBuildsLockValidFileUrl() throws { + let context = try PostbuildContext(config, env: Self.SampleEnvs) + + XCTAssertEqual(context.llbuildIdLockFile, "TARGET_TEMP_DIR/1.lock") + } } diff --git a/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift b/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift index 31c34595..9c9f39b8 100644 --- a/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/PostbuildTests.swift @@ -57,7 +57,8 @@ class PostbuildTests: FileXCTestCase { overlayHeadersPath: "", irrelevantDependenciesPaths: [], publicHeadersFolderPath: nil, - disabled: false + disabled: false, + llbuildIdLockFile: "/file" ) private var network = RemoteNetworkClientImpl( NetworkClientFake(fileManager: .default), diff --git a/Tests/XCRemoteCacheTests/Commands/PrebuildContextTests.swift b/Tests/XCRemoteCacheTests/Commands/PrebuildContextTests.swift new file mode 100644 index 00000000..cd6318e4 --- /dev/null +++ b/Tests/XCRemoteCacheTests/Commands/PrebuildContextTests.swift @@ -0,0 +1,72 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +@testable import XCRemoteCache +import XCTest + +class PrebuildContextTests: FileXCTestCase { + private var config: XCRemoteCacheConfig! + private var remoteCommitFile: URL! + private static let SampleEnvs = [ + "TARGET_NAME": "TARGET_NAME", + "TARGET_TEMP_DIR": "TARGET_TEMP_DIR", + "DERIVED_FILE_DIR": "DERIVED_FILE_DIR", + "ARCHS": "x86_64", + "OBJECT_FILE_DIR_normal": "/OBJECT_FILE_DIR_normal", + "CONFIGURATION": "CONFIGURATION", + "PLATFORM_NAME": "PLATFORM_NAME", + "XCODE_PRODUCT_BUILD_VERSION": "XCODE_PRODUCT_BUILD_VERSION", + "TARGET_BUILD_DIR": "TARGET_BUILD_DIR", + "PRODUCT_MODULE_NAME": "PRODUCT_MODULE_NAME", + "EXECUTABLE_PATH": "EXECUTABLE_PATH", + "SRCROOT": "SRCROOT", + "DEVELOPER_DIR": "DEVELOPER_DIR", + "MACH_O_TYPE": "MACH_O_TYPE", + "DWARF_DSYM_FILE_SHOULD_ACCOMPANY_PRODUCT": "DWARF_DSYM_FILE_SHOULD_ACCOMPANY_PRODUCT", + "DWARF_DSYM_FOLDER_PATH": "DWARF_DSYM_FOLDER_PATH", + "DWARF_DSYM_FILE_NAME": "DWARF_DSYM_FILE_NAME", + "BUILT_PRODUCTS_DIR": "BUILT_PRODUCTS_DIR", + "DERIVED_SOURCES_DIR": "DERIVED_SOURCES_DIR", + "CURRENT_VARIANT": "normal", + "PUBLIC_HEADERS_FOLDER_PATH": "/usr/local/include", + "LLBUILD_BUILD_ID": "1", + ] + + override func setUpWithError() throws { + try super.setUpWithError() + let workingDir = try prepareTempDir() + remoteCommitFile = workingDir.appendingPathComponent("arc.rc") + _ = workingDir.appendingPathComponent("mpo") + config = XCRemoteCacheConfig(remoteCommitFile: remoteCommitFile.path, sourceRoot: workingDir.path) + config.recommendedCacheAddress = "http://test.com" + } + + func testFailsIfLlBuildIdEnvIsMissing() throws { + var envs = Self.SampleEnvs + envs.removeValue(forKey: "LLBUILD_BUILD_ID") + + XCTAssertThrowsError(try PrebuildContext(config, env: envs)) + } + + func testBuildsLockValidFileUrl() throws { + let context = try PrebuildContext(config, env: Self.SampleEnvs) + + XCTAssertEqual(context.llbuildIdLockFile, "TARGET_TEMP_DIR/1.lock") + } +} diff --git a/Tests/XCRemoteCacheTests/Commands/PrebuildTests.swift b/Tests/XCRemoteCacheTests/Commands/PrebuildTests.swift index 84bcea4b..56e08e6e 100644 --- a/Tests/XCRemoteCacheTests/Commands/PrebuildTests.swift +++ b/Tests/XCRemoteCacheTests/Commands/PrebuildTests.swift @@ -20,6 +20,7 @@ @testable import XCRemoteCache import XCTest +// swiftlint:disable file_length // swiftlint:disable:next type_body_length class PrebuildTests: FileXCTestCase { @@ -52,6 +53,13 @@ class PrebuildTests: FileXCTestCase { remoteNetwork = RemoteNetworkClientImpl(network, URLBuilderFake(remoteCacheURL)) remapper = DependenciesRemapperFake(baseURL: URL(fileURLWithPath: "/")) metaReader = JsonMetaReader(fileAccessor: FileManager.default) + setupNonCachedContext() + setupCachedContext() + organizer = ArtifactOrganizerFake(artifactRoot: artifactsRoot, unzippedExtension: "unzip") + globalCacheSwitcher = InMemoryGlobalCacheSwitcher() + } + + private func setupNonCachedContext() { contextNonCached = PrebuildContext( targetTempDir: sampleURL, productsDir: sampleURL, @@ -64,8 +72,12 @@ class PrebuildTests: FileXCTestCase { turnOffRemoteCacheOnFirstTimeout: true, targetName: "", overlayHeadersPath: "", - disabled: false + disabled: false, + llbuildIdLockFile: "/tmp/lock" ) + } + + private func setupCachedContext() { contextCached = PrebuildContext( targetTempDir: sampleURL, productsDir: sampleURL, @@ -78,10 +90,9 @@ class PrebuildTests: FileXCTestCase { turnOffRemoteCacheOnFirstTimeout: true, targetName: "", overlayHeadersPath: "", - disabled: false + disabled: false, + llbuildIdLockFile: "/tmp/lock" ) - organizer = ArtifactOrganizerFake(artifactRoot: artifactsRoot, unzippedExtension: "unzip") - globalCacheSwitcher = InMemoryGlobalCacheSwitcher() } override func tearDownWithError() throws { @@ -244,7 +255,8 @@ class PrebuildTests: FileXCTestCase { turnOffRemoteCacheOnFirstTimeout: true, targetName: "", overlayHeadersPath: "", - disabled: false + disabled: false, + llbuildIdLockFile: "/tmp/lock" ) let prebuild = Prebuild( @@ -276,7 +288,8 @@ class PrebuildTests: FileXCTestCase { turnOffRemoteCacheOnFirstTimeout: true, targetName: "", overlayHeadersPath: "", - disabled: false + disabled: false, + llbuildIdLockFile: "/tmp/lock" ) metaContent = try generateMeta(fingerprint: generator.generate(), filekey: "1") let downloadedArtifactPackage = artifactsRoot.appendingPathComponent("1") @@ -340,7 +353,8 @@ class PrebuildTests: FileXCTestCase { turnOffRemoteCacheOnFirstTimeout: false, targetName: "", overlayHeadersPath: "", - disabled: false + disabled: false, + llbuildIdLockFile: "/tmp/lock" ) try globalCacheSwitcher.enable(sha: "1") let prebuild = Prebuild( @@ -372,7 +386,8 @@ class PrebuildTests: FileXCTestCase { turnOffRemoteCacheOnFirstTimeout: true, targetName: "", overlayHeadersPath: "", - disabled: true + disabled: true, + llbuildIdLockFile: "/tmp/lock" ) let prebuild = Prebuild( diff --git a/Tests/XCRemoteCacheTests/Commands/Prepare/Integrate/IntegrateContextTests.swift b/Tests/XCRemoteCacheTests/Commands/Prepare/Integrate/IntegrateContextTests.swift new file mode 100644 index 00000000..9e144005 --- /dev/null +++ b/Tests/XCRemoteCacheTests/Commands/Prepare/Integrate/IntegrateContextTests.swift @@ -0,0 +1,67 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +@testable import XCRemoteCache +import XCTest + +class IntegrateTests: FileXCTestCase { + private var config: XCRemoteCacheConfig! + private var remoteCommitFile: URL! + + override func setUpWithError() throws { + try super.setUpWithError() + let workingDir = try prepareTempDir() + remoteCommitFile = workingDir.appendingPathComponent("arc.rc") + _ = workingDir.appendingPathComponent("mpo") + config = XCRemoteCacheConfig(remoteCommitFile: remoteCommitFile.path, sourceRoot: workingDir.path) + config.recommendedCacheAddress = "http://test.com" + } + + + func tesFallbacksToNoDriverByDefault() throws { + let context = try IntegrateContext( + input: "project.xcodeproj", + config: config, + mode: .producer, + env: [:], + binariesDir: "/binaries", + fakeSrcRoot: "/src", + outputPath: "/output" + ) + + XCTAssertEqual(context.buildSettingsAppenderOptions, [.disableSwiftDriverIntegration]) + XCTAssertEqual(context.binaries.swiftc, "/binaries/xcswiftc") + } + + func testEnablesDriverOnRequest() throws { + config.enableSwifDriverIntegration = true + let context = try IntegrateContext( + input: "project.xcodeproj", + config: config, + mode: .producer, + env: [:], + binariesDir: "/binaries", + fakeSrcRoot: "/src", + outputPath: "/output" + ) + + XCTAssertEqual(context.buildSettingsAppenderOptions, []) + XCTAssertEqual(context.binaries.swiftc, "/binaries/swiftc") + } +} diff --git a/Tests/XCRemoteCacheTests/Commands/SwiftFrontend/SwiftFrontendArgInputTests.swift b/Tests/XCRemoteCacheTests/Commands/SwiftFrontend/SwiftFrontendArgInputTests.swift new file mode 100644 index 00000000..26237688 --- /dev/null +++ b/Tests/XCRemoteCacheTests/Commands/SwiftFrontend/SwiftFrontendArgInputTests.swift @@ -0,0 +1,362 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +@testable import XCRemoteCache +import XCTest + +class SwiftFrontendArgInputTests: FileXCTestCase { + private var compile: Bool = true + private var emitModule: Bool = false + private var objcHeaderOutput: String? + private var moduleName: String? + private var target: String? + private var primaryInputPaths: [String] = [] + private var inputPaths: [String] = [] + private var outputPaths: [String] = [] + private var dependenciesPaths: [String] = [] + private var diagnosticsPaths: [String] = [] + private var sourceInfoPath: String? + private var docPath: String? + private var supplementaryOutputFileMap: String? + + private var config: XCRemoteCacheConfig! + private var input: SwiftFrontendArgInput! + + override func setUpWithError() throws { + try super.setUpWithError() + let workingDir = try prepareTempDir() + let remoteCommitFile = workingDir.appendingPathComponent("arc.rc") + config = XCRemoteCacheConfig(remoteCommitFile: remoteCommitFile.path, sourceRoot: workingDir.path) + config.recommendedCacheAddress = "http://test.com" + + buildInput() + } + + private func buildInput() { + input = SwiftFrontendArgInput( + compile: compile, + emitModule: emitModule, + objcHeaderOutput: objcHeaderOutput, + moduleName: moduleName, + target: target, + primaryInputPaths: primaryInputPaths, + inputPaths: inputPaths, + outputPaths: outputPaths, + dependenciesPaths: dependenciesPaths, + diagnosticsPaths: diagnosticsPaths, + sourceInfoPath: sourceInfoPath, + docPath: docPath, + supplementaryOutputFileMap: supplementaryOutputFileMap) + } + + private func assertGenerationError(_ expectedError: SwiftFrontendArgInputError) { + XCTAssertThrowsError(try input.generateSwiftcContext(config: config)) { error in + guard let e = error as? SwiftFrontendArgInputError else { + XCTFail("Received invalid error \(error). Expected: \(expectedError)") + return + } + XCTAssertEqual(e, expectedError) + } + } + + func testFailsForNoStep() throws { + compile = false + emitModule = false + buildInput() + + assertGenerationError(SwiftFrontendArgInputError.bothCompilationAndEmitAction) + } + + func testFailsIfNoCompilationFiles() throws { + buildInput() + + assertGenerationError(SwiftFrontendArgInputError.noCompilationInputs) + } + + func testFailsIfNoTarget() throws { + inputPaths = ["/file1"] + buildInput() + + assertGenerationError(SwiftFrontendArgInputError.emitMissingTarget) + } + + func testFailsIfNoModuleName() throws { + inputPaths = ["/file1"] + target = "Target" + buildInput() + + assertGenerationError(SwiftFrontendArgInputError.emiMissingModuleName) + } + + func testFailsIfNoCompileHasNoPrimaryInputs() throws { + inputPaths = ["/file1"] + target = "Target" + moduleName = "Module" + buildInput() + + assertGenerationError(SwiftFrontendArgInputError.noPrimaryFileCompilationInputs) + } + + func testFailsIfDependenciesAreMissing() throws { + inputPaths = ["/file1","/file2","/file3"] + target = "Target" + moduleName = "Module" + primaryInputPaths = ["/file1","/file2"] + dependenciesPaths = ["/file1.d"] + buildInput() + + assertGenerationError(SwiftFrontendArgInputError.dependenciesOuputCountDoesntMatch(expected: 2, parsed: 1)) + } + + func testDoesntFailForMissingDependenciesIfNoDependencies() throws { + inputPaths = ["/file1","/file2","/file3"] + target = "Target" + moduleName = "Module" + primaryInputPaths = ["/file1","/file2"] + dependenciesPaths = [] + buildInput() + + assertGenerationError(SwiftFrontendArgInputError.outputsOuputCountDoesntMatch(expected: 2, parsed: 0)) + } + + func testFailsIfDiagnosticsAreMissing() throws { + inputPaths = ["/file1","/file2","/file3"] + target = "Target" + moduleName = "Module" + primaryInputPaths = ["/file1","/file2"] + diagnosticsPaths = ["/file1.d"] + buildInput() + + assertGenerationError(SwiftFrontendArgInputError.diagnosticsOuputCountDoesntMatch(expected: 2, parsed: 1)) + } + + func testDoesntFailForMissingDdiagnosticsIfNoDiagnostics() throws { + inputPaths = ["/file1","/file2","/file3"] + target = "Target" + moduleName = "Module" + primaryInputPaths = ["/file1","/file2"] + diagnosticsPaths = [] + buildInput() + + assertGenerationError(SwiftFrontendArgInputError.outputsOuputCountDoesntMatch(expected: 2, parsed: 0)) + } + + func testFailsIfOutputsAreMissing() throws { + inputPaths = ["/file1","/file2","/file3"] + target = "Target" + moduleName = "Module" + primaryInputPaths = ["/file1","/file2"] + outputPaths = ["/file1.o"] + buildInput() + + assertGenerationError(SwiftFrontendArgInputError.outputsOuputCountDoesntMatch(expected: 2, parsed: 1)) + } + + func testSetsCompilationSubsetForCompilation() throws { + inputPaths = ["/file1","/file2","/file3"] + target = "Target" + moduleName = "Module" + primaryInputPaths = ["/file1"] + outputPaths = ["/file1.o"] + buildInput() + + let context = try input.generateSwiftcContext(config: config) + + XCTAssertEqual(context.steps, .init( + compileFilesScope: .subset(["/file1"]), + emitModule: .none + )) + } + + func testBuildCompilationFilesInputs() throws { + inputPaths = ["/file1","/file2","/file3"] + target = "Target" + moduleName = "Module" + primaryInputPaths = ["/file1"] + outputPaths = ["/file1.o"] + dependenciesPaths = ["/file1.d"] + buildInput() + + let context = try input.generateSwiftcContext(config: config) + + XCTAssertEqual(context.inputs, .map([ + "/file1": SwiftFileCompilationInfo( + file: "/file1", + dependencies: "/file1.d", + object: "/file1.o", + swiftDependencies: nil + ) + ]) + ) + } + + func testRecognizesArchFromOuputFirstPaths() throws { + inputPaths = ["/file1","/file2","/file3"] + target = "Target" + moduleName = "Module" + primaryInputPaths = ["/file1"] + outputPaths = ["/TARGET_TEMP_DIR/Object-normal/arm64/file1.o"] + dependenciesPaths = ["/file1.d"] + buildInput() + + let context = try input.generateSwiftcContext(config: config) + + XCTAssertEqual(context.arch, "arm64") + } + + func testPassesExtraParams() throws { + inputPaths = ["/file1","/file2","/file3"] + target = "Target" + moduleName = "Module" + primaryInputPaths = ["/file1"] + outputPaths = ["/file1.o"] + dependenciesPaths = ["/file1.d"] + buildInput() + + let context = try input.generateSwiftcContext(config: config) + + XCTAssertEqual(context.moduleName, "Module") + XCTAssertEqual(context.target, "Target") + XCTAssertEqual(context.compilationFiles, .list(inputPaths)) + XCTAssertEqual(context.mode, .consumer(commit: .unavailable)) + } + + func testEmitModuleFailsForMissingOutput() throws { + emitModule = true + compile = false + inputPaths = ["/file1","/file2","/file3"] + target = "Target" + moduleName = "Module" + outputPaths = [] + buildInput() + + assertGenerationError(SwiftFrontendArgInputError.emitModulOuputCountIsNot1(parsed: 0)) + } + + func testEmitModuleFailsForMissingObjcHeader() throws { + emitModule = true + compile = false + inputPaths = ["/file1","/file2","/file3"] + target = "Target" + moduleName = "Module" + outputPaths = ["/Module.swiftmodule"] + buildInput() + + assertGenerationError(SwiftFrontendArgInputError.emitModuleMissingObjcHeaderPath) + } + + func testEmitModuleFailsForExcessiveDiagnostics() throws { + emitModule = true + compile = false + inputPaths = ["/file1","/file2","/file3"] + target = "Target" + moduleName = "Module" + outputPaths = ["/Module.swiftmodule"] + objcHeaderOutput = "/file-Swift.h" + diagnosticsPaths = ["/file.diag", "/file2.diag"] + buildInput() + + assertGenerationError(SwiftFrontendArgInputError.emitModuleDiagnosticsOuputCountIsHigherThan1(parsed: 2)) + } + + func testEmitModuleFailsForExcessiveDependencies() throws { + emitModule = true + compile = false + inputPaths = ["/file1","/file2","/file3"] + target = "Target" + moduleName = "Module" + outputPaths = ["/Module.swiftmodule"] + objcHeaderOutput = "/file-Swift.h" + dependenciesPaths = ["/file.d", "/file2.d"] + buildInput() + + assertGenerationError(SwiftFrontendArgInputError.emitModuleDependenciesOuputCountIsHigherThan1(parsed: 2)) + } + + func testEmitModuleSetsStep() throws { + emitModule = true + compile = false + inputPaths = ["/file1","/file2","/file3"] + target = "Target" + moduleName = "Module" + outputPaths = ["/Module.swiftmodule"] + objcHeaderOutput = "/file-Swift.h" + diagnosticsPaths = ["/file.dia"] + dependenciesPaths = ["/file.d"] + buildInput() + + let context = try input.generateSwiftcContext(config: config) + + XCTAssertEqual(context.steps, .init( + compileFilesScope: .none, + emitModule: .init( + objcHeaderOutput: "/file-Swift.h", + modulePathOutput: "/Module.swiftmodule", + dependencies: "/file.d")) + ) + } + + func testEmitModuleSetsAllIntpus() throws { + emitModule = true + compile = false + inputPaths = ["/file1","/file2","/file3"] + target = "Target" + moduleName = "Module" + outputPaths = ["/Module.swiftmodule"] + objcHeaderOutput = "/file-Swift.h" + buildInput() + + let context = try input.generateSwiftcContext(config: config) + + XCTAssertEqual(context.compilationFiles, .list(inputPaths)) + } + + func testEmitModuleRecognizesArchFromObjCHeader() throws { + emitModule = true + compile = false + inputPaths = ["/file1","/file2","/file3"] + target = "Target" + moduleName = "Module" + outputPaths = ["file.swiftmodule"] + objcHeaderOutput = "/TARGET_TEMP_DIR/Object-normal/arm64/file-Swift.h" + buildInput() + + let context = try input.generateSwiftcContext(config: config) + + XCTAssertEqual(context.arch, "arm64") + } + + func testEmitModulePassesExtraParams() throws { + emitModule = true + compile = false + inputPaths = ["/file1","/file2","/file3"] + target = "Target" + moduleName = "Module" + outputPaths = ["/Module.swiftmodule"] + objcHeaderOutput = "/file-Swift.h" + buildInput() + + let context = try input.generateSwiftcContext(config: config) + + XCTAssertEqual(context.moduleName, "Module") + XCTAssertEqual(context.target, "Target") + XCTAssertEqual(context.compilationFiles, .list(inputPaths)) + XCTAssertEqual(context.mode, .consumer(commit: .unavailable)) + } +} diff --git a/Tests/XCRemoteCacheTests/Commands/SwiftFrontendOrchestratorTests.swift b/Tests/XCRemoteCacheTests/Commands/SwiftFrontendOrchestratorTests.swift new file mode 100644 index 00000000..18c07f11 --- /dev/null +++ b/Tests/XCRemoteCacheTests/Commands/SwiftFrontendOrchestratorTests.swift @@ -0,0 +1,155 @@ +// Copyright (c) 2023 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +@testable import XCRemoteCache +import XCTest + + +final class SwiftFrontendOrchestratorTests: FileXCTestCase { + private let prohibitedAccessor = DisallowedExclusiveFileAccessor() + private var nonEmptyFile: URL! + private let maxLocking: TimeInterval = 10 + + override func setUp() async throws { + nonEmptyFile = try prepareTempDir().appendingPathComponent("lock.lock") + try fileManager.write(toPath: nonEmptyFile.path, contents: "Done".data(using: .utf8)) + } + func testRunsCriticalSectionImmediatellyForProducer() throws { + let orchestrator = CommonSwiftFrontendOrchestrator( + mode: .producer, + action: .compile, + lockAccessor: prohibitedAccessor, + maxLockTimeout: maxLocking + ) + + var invoked = false + try orchestrator.run { + invoked = true + } + XCTAssertTrue(invoked) + } + + func testRunsCriticalSectionImmediatellyForDisabledConsumer() throws { + let orchestrator = CommonSwiftFrontendOrchestrator( + mode: .consumer(commit: .unavailable), + action: .compile, + lockAccessor: prohibitedAccessor, + maxLockTimeout: maxLocking + ) + + var invoked = false + try orchestrator.run { + invoked = true + } + XCTAssertTrue(invoked) + } + + func testRunsEmitModuleLogicInAnExclusiveLock() throws { + let lock = FakeExclusiveFileAccessor() + let orchestrator = CommonSwiftFrontendOrchestrator( + mode: .consumer(commit: .available(commit: "")), + action: .emitModule, + lockAccessor: lock, + maxLockTimeout: maxLocking + ) + + var invoked = false + try orchestrator.run { + XCTAssertTrue(lock.isLocked) + invoked = true + } + XCTAssertTrue(invoked) + } + + func testCompilationInvokesCriticalSectionOnlyForNonEmptyLockFile() throws { + let lock = FakeExclusiveFileAccessor(pattern: [.empty, .nonEmpty(nonEmptyFile)]) + let orchestrator = CommonSwiftFrontendOrchestrator( + mode: .consumer(commit: .available(commit: "")), + action: .compile, + lockAccessor: lock, + maxLockTimeout: maxLocking + ) + + var invoked = false + try orchestrator.run { + XCTAssertTrue(lock.isLocked) + invoked = true + } + XCTAssertTrue(invoked) + } + + func testExecutesActionWithoutLockIfLockingFileIsEmptyForALongTime() throws { + let lock = FakeExclusiveFileAccessor(pattern: []) + let orchestrator = CommonSwiftFrontendOrchestrator( + mode: .consumer(commit: .available(commit: "")), + action: .compile, + lockAccessor: lock, + maxLockTimeout: 0 + ) + + var invoked = false + try orchestrator.run { + XCTAssertFalse(lock.isLocked) + invoked = true + } + XCTAssertTrue(invoked) + } +} + +private class DisallowedExclusiveFileAccessor: ExclusiveFileAccessor { + func exclusiveAccess(block: (FileHandle) throws -> (T)) throws -> T { + throw "Invoked ProhibitedExclusiveFileAccessor" + } +} + +// Thread-unsafe, in-memory lock +private class FakeExclusiveFileAccessor: ExclusiveFileAccessor { + private(set) var isLocked = false + private var pattern: [LockFileContent] + + enum LockFileContent { + case empty + case nonEmpty(URL) + + func fileHandle() throws -> FileHandle { + switch self { + case .empty: return FileHandle.nullDevice + case .nonEmpty(let url): return try FileHandle(forReadingFrom: url) + } + } + } + + init(pattern: [LockFileContent] = []) { + // keep in the reversed order to always pop + self.pattern = pattern.reversed() + } + + func exclusiveAccess(block: (FileHandle) throws -> (T)) throws -> T { + if isLocked { + throw "FakeExclusiveFileAccessor lock is already locked" + } + defer { + isLocked = false + } + isLocked = true + let fileHandle = try (pattern.popLast() ?? .empty).fileHandle() + return try block(fileHandle) + } + +} diff --git a/Tests/XCRemoteCacheTests/Commands/SwiftcTests+Fronend.swift b/Tests/XCRemoteCacheTests/Commands/SwiftcTests+Fronend.swift new file mode 100644 index 00000000..6f6eab4d --- /dev/null +++ b/Tests/XCRemoteCacheTests/Commands/SwiftcTests+Fronend.swift @@ -0,0 +1,88 @@ +// Copyright (c) 2021 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +@testable import XCRemoteCache +import XCTest + +// swiftlint:disable:next type_body_length +class SwiftcTests_Frontend: FileXCTestCase { + private let dummyURL = URL(fileURLWithPath: "") + + private var inputFileListReader: ListReader! + private var markerReader: ListReader! + private var allowedFilesListScanner: FileListScanner! + private var artifactOrganizer: ArtifactOrganizer! + private var swiftcInputReader: SwiftcInputReader! + private var config: XCRemoteCacheConfig! + private var input: SwiftFrontendArgInput! + private var context: SwiftcContext! + private var markerWriter: MarkerWriterSpy! + private var productsGenerator: SwiftcProductsGeneratorSpy! + private var dependenciesWriterSpy: DependenciesWriterSpy! + private var dependenciesWriterFactory: ((URL, FileManager) -> DependenciesWriter)! + private var touchFactory: ((URL, FileManager) -> Touch)! + private var workingDir: URL! + private var remoteCommitLocation: URL! + private let sampleRemoteCommit = "bdb321" + + + override func setUpWithError() throws { + try super.setUpWithError() + workingDir = try prepareTempDir() + let modulePathOutput = workingDir.appendingPathComponent("Objects-normal") + .appendingPathComponent("archTest") + .appendingPathComponent("Target.swiftmodule") + try FileManager.default.createDirectory(at: workingDir, withIntermediateDirectories: true, attributes: nil) + + inputFileListReader = ListReaderFake(files: []) + markerReader = ListReaderFake(files: []) + allowedFilesListScanner = FileListScannerFake(files: []) + artifactOrganizer = ArtifactOrganizerFake() + swiftcInputReader = SwiftcInputReaderStub() + config = XCRemoteCacheConfig(remoteCommitFile: "arc.rc", sourceRoot: workingDir.path) + // SwiftcContext reads remoteCommit from a file so writing to a temporary file `sampleRemoteCommit` + remoteCommitLocation = URL(fileURLWithPath: config.sourceRoot).appendingPathComponent("arc.rc") + try sampleRemoteCommit.write(to: remoteCommitLocation, atomically: true, encoding: .utf8) + + input = SwiftFrontendArgInput( + compile: true, + emitModule: false, + objcHeaderOutput: nil, + moduleName: "Module", + target: "Target", + primaryInputPaths: [], + inputPaths: [], + outputPaths: [], + dependenciesPaths: [], + diagnosticsPaths: [], + sourceInfoPath: nil, + docPath: nil, + supplementaryOutputFileMap: nil + ) + context = try SwiftcContext(config: config, input: input) + markerWriter = MarkerWriterSpy() + productsGenerator = SwiftcProductsGeneratorSpy( + generatedDestination: SwiftcProductsGeneratorOutput(swiftmoduleDir: "", objcHeaderFile: "") + ) + let dependenciesWriterSpy = DependenciesWriterSpy() + self.dependenciesWriterSpy = dependenciesWriterSpy + dependenciesWriterFactory = { [dependenciesWriterSpy] _, _ in dependenciesWriterSpy } + touchFactory = { _, _ in TouchSpy() } + } // swiftlint:disable:next file_length +} diff --git a/Tests/XCRemoteCacheTests/Dependencies/PhaseCacheModeControllerTests.swift b/Tests/XCRemoteCacheTests/Dependencies/PhaseCacheModeControllerTests.swift index 3ad44a4b..8a14b71e 100644 --- a/Tests/XCRemoteCacheTests/Dependencies/PhaseCacheModeControllerTests.swift +++ b/Tests/XCRemoteCacheTests/Dependencies/PhaseCacheModeControllerTests.swift @@ -34,6 +34,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: FileDependenciesWriter.init, dependenciesReader: { _, _ in dependenciesReader }, markerWriter: FileMarkerWriter.init, + llbuildLockFile: "/file", fileManager: FileManager.default ) @@ -51,6 +52,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: FileDependenciesWriter.init, dependenciesReader: { _, _ in dependenciesReader }, markerWriter: FileMarkerWriter.init, + llbuildLockFile: "/tmp/lock", fileManager: FileManager.default ) @@ -68,6 +70,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: FileDependenciesWriter.init, dependenciesReader: { _, _ in dependenciesReader }, markerWriter: FileMarkerWriter.init, + llbuildLockFile: "/tmp/lock", fileManager: FileManager.default ) @@ -85,6 +88,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: { _, _ in dependenciesWriter }, dependenciesReader: { _, _ in DependenciesReaderFake(dependencies: [:]) }, markerWriter: { _, _ in MarkerWriterSpy() }, + llbuildLockFile: "/tmp/lock", fileManager: FileManager.default ) @@ -105,6 +109,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: { _, _ in dependenciesWriter }, dependenciesReader: { _, _ in DependenciesReaderFake(dependencies: [:]) }, markerWriter: { _, _ in MarkerWriterSpy() }, + llbuildLockFile: "/tmp/lock", fileManager: FileManager.default ) @@ -125,6 +130,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: { _, _ in DependenciesWriterSpy() }, dependenciesReader: { _, _ in DependenciesReaderFake(dependencies: [:]) }, markerWriter: { _, _ in markerWriter }, + llbuildLockFile: "/tmp/lock", fileManager: FileManager.default ) @@ -142,6 +148,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: { _, _ in DependenciesWriterSpy() }, dependenciesReader: { _, _ in DependenciesReaderFake(dependencies: [:]) }, markerWriter: { _, _ in MarkerWriterSpy() }, + llbuildLockFile: "/tmp/lock", fileManager: FileManager.default ) @@ -163,6 +170,7 @@ class PhaseCacheModeControllerTests: XCTestCase { dependenciesWriter: { _, _ in dependenciesWriter }, dependenciesReader: { _, _ in DependenciesReaderFake(dependencies: [:]) }, markerWriter: { _, _ in markerWriterSpy }, + llbuildLockFile: "/tmp/lock", fileManager: FileManager.default ) diff --git a/cocoapods-plugin/lib/cocoapods-xcremotecache/command/hooks.rb b/cocoapods-plugin/lib/cocoapods-xcremotecache/command/hooks.rb index 7a270f77..3c1c3110 100644 --- a/cocoapods-plugin/lib/cocoapods-xcremotecache/command/hooks.rb +++ b/cocoapods-plugin/lib/cocoapods-xcremotecache/command/hooks.rb @@ -123,7 +123,8 @@ def self.enable_xcremotecache( exclude_build_configurations, final_target, fake_src_root, - exclude_sdks_configurations + exclude_sdks_configurations, + enable_swift_driver_integration ) srcroot_relative_xc_location = parent_dir(xc_location, repo_distance) # location of the entrite CocoaPods project, relative to SRCROOT @@ -137,14 +138,15 @@ def self.enable_xcremotecache( elsif mode == 'producer' || mode == 'producer-fast' config.build_settings.delete('CC') if config.build_settings.key?('CC') end - reset_build_setting(config.build_settings, 'SWIFT_EXEC', "$SRCROOT/#{srcroot_relative_xc_location}/xcswiftc", exclude_sdks_configurations) + swiftc_name = enable_swift_driver_integration ? 'swiftc' : 'xcswiftc' + reset_build_setting(config.build_settings, 'SWIFT_EXEC', "$SRCROOT/#{srcroot_relative_xc_location}/#{swiftc_name}", exclude_sdks_configurations) reset_build_setting(config.build_settings, 'LIBTOOL', "$SRCROOT/#{srcroot_relative_xc_location}/xclibtool", exclude_sdks_configurations) # Setting LIBTOOL to '' breaks SwiftDriver intengration so resetting it to the original value 'libtool' for all excluded configurations add_build_setting_for_sdks(config.build_settings, 'LIBTOOL', 'libtool', exclude_sdks_configurations) reset_build_setting(config.build_settings, 'LD', "$SRCROOT/#{srcroot_relative_xc_location}/xcld", exclude_sdks_configurations) reset_build_setting(config.build_settings, 'LDPLUSPLUS', "$SRCROOT/#{srcroot_relative_xc_location}/xcldplusplus", exclude_sdks_configurations) reset_build_setting(config.build_settings, 'LIPO', "$SRCROOT/#{srcroot_relative_xc_location}/xclipo", exclude_sdks_configurations) - reset_build_setting(config.build_settings, 'SWIFT_USE_INTEGRATED_DRIVER', 'NO', exclude_sdks_configurations) + reset_build_setting(config.build_settings, 'SWIFT_USE_INTEGRATED_DRIVER', 'NO', exclude_sdks_configurations) unless enable_swift_driver_integration reset_build_setting(config.build_settings, 'XCREMOTE_CACHE_FAKE_SRCROOT', fake_src_root, exclude_sdks_configurations) reset_build_setting(config.build_settings, 'XCRC_PLATFORM_PREFERRED_ARCH', "$(LINK_FILE_LIST_$(CURRENT_VARIANT)_$(PLATFORM_PREFERRED_ARCH):dir:standardizepath:file:default=arm64)", exclude_sdks_configurations) @@ -498,6 +500,7 @@ def self.save_lldbinit_rewrite(user_proj_directory,fake_src_root) check_platform = @@configuration['check_platform'] fake_src_root = @@configuration['fake_src_root'] exclude_sdks_configurations = @@configuration['exclude_sdks_configurations'] || [] + enable_swift_driver_integration = @@configuration['enable_swift_driver_integration'] || false xccc_location_absolute = "#{user_proj_directory}/#{xccc_location}" xcrc_location_absolute = "#{user_proj_directory}/#{xcrc_location}" @@ -521,7 +524,7 @@ def self.save_lldbinit_rewrite(user_proj_directory,fake_src_root) next if target.name.start_with?("Pods-") next if target.name.end_with?("Tests") next if exclude_targets.include?(target.name) - enable_xcremotecache(target, 1, xcrc_location, xccc_location, mode, exclude_build_configurations, final_target,fake_src_root, exclude_sdks_configurations) + enable_xcremotecache(target, 1, xcrc_location, xccc_location, mode, exclude_build_configurations, final_target,fake_src_root, exclude_sdks_configurations, enable_swift_driver_integration) end # Create .rcinfo into `Pods` directory as that .xcodeproj reads configuration from .xcodeproj location @@ -534,7 +537,7 @@ def self.save_lldbinit_rewrite(user_proj_directory,fake_src_root) next if target.source_build_phase.files_references.empty? next if target.name.end_with?("Tests") next if exclude_targets.include?(target.name) - enable_xcremotecache(target, 1, xcrc_location, xccc_location, mode, exclude_build_configurations, final_target,fake_src_root, exclude_sdks_configurations) + enable_xcremotecache(target, 1, xcrc_location, xccc_location, mode, exclude_build_configurations, final_target,fake_src_root, exclude_sdks_configurations, enable_swift_driver_integration) end generated_project.save() end @@ -575,7 +578,7 @@ def self.save_lldbinit_rewrite(user_proj_directory,fake_src_root) # Attach XCRC to the app targets user_project.targets.each do |target| next if exclude_targets.include?(target.name) - enable_xcremotecache(target, 0, xcrc_location, xccc_location, mode, exclude_build_configurations, final_target,fake_src_root, exclude_sdks_configurations) + enable_xcremotecache(target, 0, xcrc_location, xccc_location, mode, exclude_build_configurations, final_target,fake_src_root, exclude_sdks_configurations, enable_swift_driver_integration) end # Set Target sourcemap diff --git a/cocoapods-plugin/lib/cocoapods-xcremotecache/gem_version.rb b/cocoapods-plugin/lib/cocoapods-xcremotecache/gem_version.rb index ddf31972..b2acb8e4 100644 --- a/cocoapods-plugin/lib/cocoapods-xcremotecache/gem_version.rb +++ b/cocoapods-plugin/lib/cocoapods-xcremotecache/gem_version.rb @@ -13,5 +13,5 @@ # limitations under the License. module CocoapodsXcremotecache - VERSION = "0.0.16" + VERSION = "0.0.17" end diff --git a/e2eTests/StandaloneSampleApp/.rcinfo b/e2eTests/StandaloneSampleApp/.rcinfo index 60030012..3cd0f26b 100644 --- a/e2eTests/StandaloneSampleApp/.rcinfo +++ b/e2eTests/StandaloneSampleApp/.rcinfo @@ -1,8 +1,9 @@ --- -cache_addresses: +cache_addresses: - 'http://localhost:8080/cache/pods' primary_repo: '.' primary_branch: 'e2e-test-branch' mode: 'consumer' final_target': XCRemoteCacheSample' artifact_maximum_age: 0 # do not use local cache in ~/Library/Caches/XCRemoteCache +enable_swift_driver_integration: true diff --git a/tasks/e2e.rb b/tasks/e2e.rb index 9aa9dbd6..029299f2 100644 --- a/tasks/e2e.rb +++ b/tasks/e2e.rb @@ -25,7 +25,8 @@ 'primary_branch' => GIT_BRANCH, 'mode' => 'consumer', 'final_target' => 'XCRemoteCacheSample', - 'artifact_maximum_age' => 0 + 'artifact_maximum_age' => 0, + 'enable_swift_driver_integration' => true }.freeze DEFAULT_EXPECTATIONS = { 'misses' => 0,