Support xcode15 test

.github/workflows/ci.yaml

@@ -13,12 +13,18 @@ jobs:
           args: --strict
 
   macOS:
-    runs-on: macos-12
+    runs-on: macos-13
     env:
-      XCODE_VERSION: ${{ '14.2' }}
+      XCODE_VERSION: ${{ '15.0' }}
     steps:
+      - name: Install nginx
+        run: "brew install nginx"
       - name: Select Xcode
         run: "sudo xcode-select -s /Applications/Xcode_$XCODE_VERSION.app"
+      - name: "Install iOS SDK"
+        run: "xcodebuild -downloadPlatform iOS"
+      - name: "Install watchOS SDK"
+        run: "xcodebuild -downloadPlatform watchOS"
       - name: Checkout
         uses: actions/checkout@v1
       - name: Build and Run

Package.swift

@@ -67,6 +67,10 @@ let package = Package(
             name: "xcldplusplus",
             dependencies: ["XCRemoteCache"]
         ),
+        .target(
+            name: "xcactool",
+            dependencies: ["XCRemoteCache"]
+        ),
         .target(
             // Wrapper target that builds all binaries but does nothing in runtime
             name: "Aggregator",
@@ -80,6 +84,7 @@ let package = Package(
                 "xcld",
                 "xcldplusplus",
                 "xclipo",
+                "xcactool"
             ]
         ),
         .testTarget(

README.md

@@ -323,6 +323,7 @@ Note: This step is not required if at least one of these is true:
 | `mode_marker_path` | Path, relative to `$TARGET_TEMP_DIR`, of a maker file to enable or disable the remote cache for a given target. Includes a list of all allowed input files to use remote cache | `rc.enabled` | ⬜️ |
 | `clang_command` | Command for a standard C compilation fallback | `clang` | ⬜️ |
 | `swiftc_command` | Command for a standard Swift compilation fallback | `swiftc` | ⬜️ |
+| `actool_command` | Command for the assets catalog tool. _By default uses a symlink that points to the default Xcode location's actool_ | `/var/db/xcode_select_link/usr/bin/actool` | ⬜️ |
 | `primary_repo` | Address of the primary git repository that produces cache artifacts (case-sensitive) | N/A | ✅ |
 | `primary_branch` | The main (primary) branch on the `primary_repo` that produces cache artifacts | `master` | ⬜️ |
 | `repo_root` | The path to the git repo root | `"."` | ⬜️ |

Rakefile

@@ -10,7 +10,7 @@ DERIVED_DATA_DIR = File.join('.build').freeze
 RELEASES_ROOT_DIR = File.join('releases').freeze
 
 EXECUTABLE_NAME = 'XCRemoteCache'
-EXECUTABLE_NAMES = ['xclibtool', 'xcpostbuild', 'xcprebuild', 'xcprepare', 'xcswiftc', 'swiftc', 'xcswift-frontend', 'swift-frontend', 'xcld', 'xcldplusplus', 'xclipo']
+EXECUTABLE_NAMES = ['xclibtool', 'xcpostbuild', 'xcprebuild', 'xcprepare', 'xcswiftc', 'swiftc', 'xcswift-frontend', 'swift-frontend', 'xcld', 'xcldplusplus', 'xclipo', 'xcactool', 'actool']
 PROJECT_NAME = 'XCRemoteCache'
 
 SWIFTLINT_ENABLED = true
@@ -60,9 +60,10 @@ task :build, [:configuration, :arch, :sdks, :is_archive] do |task, args|
     # Path of the executable looks like: `.build/(debug|release)/XCRemoteCache`
     build_path_base = File.join(DERIVED_DATA_DIR, args.configuration)
     # swift-frontent integration requires that the SWIFT_EXEC is `swiftc` so create
-    # a symbolic link between swiftc->xcswiftc and swift-frontend->xcswift-frontend
+    # a symbolic link between swiftc->xcswiftc and swift-frontend->xcswift-frontend etc.
     system("cd #{build_path_base} && ln -s xcswiftc swiftc")
     system("cd #{build_path_base} && ln -s xcswift-frontend swift-frontend")
+    system("cd #{build_path_base} && ln -s xcactool actool")
     sdk_build_paths = EXECUTABLE_NAMES.map {|e| File.join(build_path_base, e)}
 
     build_paths.push(sdk_build_paths)

Sources/XCRemoteCache/Artifacts/ArtifactMetaUpdater.swift

@@ -0,0 +1,73 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+enum ArtifactMetaProcessorError: Error {
+    /// The prebuild plugin execution was called but the local
+    /// path to the artifact directory is unknown
+    /// Might happen that the artifact processor didn't invoke
+    /// a request to process an artifact
+    case artifactLocationIsUnknown
+}
+
+/// Processes downloaded artifact by placing an up-to-date and remapped meta file
+/// in the artifact directory
+class ArtifactMetaUpdater: ArtifactProcessor {
+    private var artifactLocation: URL?
+    private let metaWriter: MetaWriter
+    private let fileRemapper: FileDependenciesRemapper
+
+    init(fileRemapper: FileDependenciesRemapper, metaWriter: MetaWriter) {
+        self.metaWriter = metaWriter
+        self.fileRemapper = fileRemapper
+    }
+
+    /// Remembers the artifact location, used later in the plugin
+    /// - Parameter url: artifact's root directory
+    func process(rawArtifact url: URL) throws {
+        // Storing the location of the just downloaded artifact
+        // Note, the location might include a meta (generated by producer
+        // while compiling and building an artifact), which might be outdated
+        // (e.g. a new schema has been applied to the meta format, while artifacts
+        // format is backward-compatible)
+        artifactLocation = url
+    }
+
+    func process(localArtifact url: URL) throws {
+        // No need to do anything in the postbuild
+    }
+}
+
+extension ArtifactMetaUpdater: ArtifactConsumerPrebuildPlugin {
+
+    /// Overrides the meta json file in the downloaded artifact with a meta including
+    /// remapped paths
+    func run(meta: MainArtifactMeta) throws {
+        guard let artifactLocation = artifactLocation else {
+            throw ArtifactMetaProcessorError.artifactLocationIsUnknown
+        }
+        _ = try metaWriter.write(meta, locationDir: artifactLocation)
+        // TODO: extract the meta json location logic to a shared class
+        let metaURL = artifactLocation
+            .appendingPathComponent(meta.fileKey)
+            .appendingPathExtension("json")
+        try fileRemapper.remap(fromGeneric: metaURL)
+    }
+}

Sources/XCRemoteCache/Artifacts/ArtifactOrganizer.swift

@@ -47,6 +47,8 @@ protocol ArtifactOrganizer {
 }
 
 class ZipArtifactOrganizer: ArtifactOrganizer {
+    static let activeArtifactLocation = "active"
+
     private let cacheDir: URL
     // all processors that should "prepare" the unzipped raw artifact
     private let artifactProcessors: [ArtifactProcessor]
@@ -63,7 +65,7 @@ class ZipArtifactOrganizer: ArtifactOrganizer {
     }
 
     func getActiveArtifactLocation() -> URL {
-        return cacheDir.appendingPathComponent("active")
+        return cacheDir.appendingPathComponent(Self.self.activeArtifactLocation)
     }
 
     func getActiveArtifactFilekey() throws -> String {
@@ -90,20 +92,27 @@ class ZipArtifactOrganizer: ArtifactOrganizer {
         let destinationURL = artifact.deletingPathExtension()
         guard fileManager.fileExists(atPath: destinationURL.path) == false else {
             infoLog("Skipping artifact, already existing at \(destinationURL)")
+            try runArtifactProcessors(artifactLocation: destinationURL)
             return destinationURL
         }
-        // Uzipping to a temp file first to never leave the uncompleted zip in the final location
+        // Unzipping to a temp file first to never leave the uncompleted zip in the final location
         // when the command was interrupted (internal crash or `kill -9` signal)
         let tempDestination = destinationURL.appendingPathExtension("tmp")
         try Zip.unzipFile(artifact, destination: tempDestination, overwrite: true, password: nil)
 
-        try artifactProcessors.forEach { processor in
-            try processor.process(rawArtifact: tempDestination)
-        }
         try fileManager.moveItem(at: tempDestination, to: destinationURL)
+        try runArtifactProcessors(artifactLocation: destinationURL)
         return destinationURL
     }
 
+    /// Iterates all processor when an artifact has been just downloaded or reused from already downloaded
+    /// and previously processed location
+    private func runArtifactProcessors(artifactLocation: URL) throws {
+        try artifactProcessors.forEach { processor in
+            try processor.process(rawArtifact: artifactLocation)
+        }
+    }
+
     func activate(extractedArtifact: URL) throws {
         let activeLocationURL = getActiveArtifactLocation()
         try fileManager.spt_forceSymbolicLink(at: activeLocationURL, withDestinationURL: extractedArtifact)

Sources/XCRemoteCache/Artifacts/MetaPathProvider.swift

@@ -0,0 +1,56 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+enum MetaPathProviderError: Error {
+    /// Generic error when a meta path cannot be provided
+    case failed(message: String)
+}
+
+protocol MetaPathProvider {
+    /// Returns the location of the meta file on disk
+    func getMetaPath() throws -> URL
+}
+
+/// Finds the location of the meta in the unzipped artifact.
+/// Assumes the artifact contains only a single .json file:
+/// a meta file with filename equal to the fileKey
+class ArtifactMetaPathProvider: MetaPathProvider {
+    private let artifactLocation: URL
+    private let dirScanner: DirScanner
+
+    init(
+        artifactLocation: URL,
+        dirScanner: DirScanner
+    ) {
+        self.artifactLocation = artifactLocation
+        self.dirScanner = dirScanner
+    }
+
+    func getMetaPath() throws -> URL {
+        let items = try dirScanner.items(at: artifactLocation)
+        guard let meta = items.first(where: { $0.pathExtension == "json" }) else {
+            throw MetaPathProviderError.failed(
+                message: "artifact \(artifactLocation) doesn't contain expected .json with a meta"
+            )
+        }
+        return meta
+    }
+}

Sources/XCRemoteCache/Commands/Actool/ACTool.swift

@@ -0,0 +1,81 @@
+// Copyright (c) 2023 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+enum ACToolResult: Equatable {
+    /// the generated actool interfaces matches the one observed on a producer side
+    case cacheHit(dependencies: [URL])
+    /// the generated interface is different - leads to a cache miss
+    case cacheMiss
+}
+
+class ACTool {
+    private let markerReader: ListReader
+    private let markerWriter: MarkerWriter
+    private let metaReader: MetaReader
+    private let fingerprintAccumulator: FingerprintAccumulator
+    private let metaPathProvider: MetaPathProvider
+
+    init(
+        markerReader: ListReader,
+        markerWriter: MarkerWriter,
+        metaReader: MetaReader,
+        fingerprintAccumulator: FingerprintAccumulator,
+        metaPathProvider: MetaPathProvider
+    ) {
+        self.markerReader = markerReader
+        self.markerWriter = markerWriter
+        self.metaReader = metaReader
+        self.fingerprintAccumulator = fingerprintAccumulator
+        self.metaPathProvider = metaPathProvider
+    }
+
+    func run() throws -> ACToolResult {
+        // 1. do nothing if the RC is disabled
+        guard markerReader.canRead() else {
+            return .cacheMiss
+        }
+        let dependencies = try markerReader.listFilesURLs()
+
+        // 2. Read meta's sources files & fingerprint
+        let metaPath = try metaPathProvider.getMetaPath()
+        let meta = try metaReader.read(localFile: metaPath)
+        // 3. Compare local vs meta's fingerprint
+        let localFingerprint = try computeFingerprints(meta.assetsSources)
+        // 4. Disable RC if the is fingerprint doesn't match
+        return (localFingerprint == meta.assetsSourcesFingerprint ? .cacheHit(dependencies: dependencies) : .cacheMiss)
+    }
+
+    private func computeFingerprints(_ paths: [String]) throws -> RawFingerprint {
+        // TODO:
+        // 1. transform to URL
+        // 2. build (in the same order the fingerprint)
+        fingerprintAccumulator.reset()
+        for path in paths {
+            let file = URL(fileURLWithPath: path)
+            do {
+                try fingerprintAccumulator.append(file)
+            } catch FingerprintAccumulatorError.missingFile(let content) {
+                printWarning("File at \(content.path) was not found on disc. Calculating fingerprint without it.")
+            }
+        }
+        return try fingerprintAccumulator.generate()
+    }
+}