tar: Use OutputStream

With OutputStream, the image can be saved to a memory buffer (this
is the default for Archive) or directly to a file on disk.

Author: euanh

Sources/Tar/tar.swift

@@ -12,6 +12,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+import Foundation
+
 // This file defines a basic tar writer which produces POSIX tar files.
 // This avoids the need to depend on a system-provided tar binary.
 //
@@ -342,44 +344,44 @@ public func tar(_ bytes: [UInt8], filename: String = "app") throws -> [UInt8] {
 }
 
 /// Represents a tar archive
-public struct Archive {
+public struct Archive: ~Copyable {
     /// The files, directories and other members of the archive
-    var members: [ArchiveMember]
+    var output: OutputStream
 
     /// Creates an empty Archive
     public init() {
-        members = []
+        output = OutputStream.toMemory()
+        output.open()
+        output.schedule(in: .current, forMode: .default)  // is this needed?
     }
 
-    /// Appends a member to the archive
-    /// Parameters:
-    /// - member: The member to append
-    public mutating func append(_ member: ArchiveMember) {
-        self.members.append(member)
+    deinit {
+        output.close()
     }
 
-    /// Returns a new archive made by appending a member to the receiver
+    /// Appends a member to the archive
     /// Parameters:
     /// - member: The member to append
-    /// Returns: A new archive made by appending `member` to the receiver.
-    public func appending(_ member: ArchiveMember) -> Self {
-        var ret = self
-        ret.members += [member]
-        return ret
+    public func append(_ member: ArchiveMember) {
+        print("append: \(member.bytes.count)")
+        let written = output.write(member.bytes, maxLength: member.bytes.count)
+        print("append wrote: \(written)")
+        if written != member.bytes.count {
+            print("count: \(member.bytes.count), written: \(written)")
+            fatalError()
+        }
     }
 
     /// The serialized byte representation of the archive, including padding and end-of-archive marker.
     public var bytes: [UInt8] {
-        var ret: [UInt8] = []
-        for member in members {
-            ret.append(contentsOf: member.bytes)
+        guard let data = output.property(forKey: .dataWrittenToMemoryStreamKey) as? Data else {
+            fatalError("retrieving memory stream contents")
         }
+        print("bytes returned: \(data.count)")
 
         // Append the end of file marker
         let marker = [UInt8](repeating: 0, count: 2 * blockSize)
-        ret.append(contentsOf: marker)
-
-        return ret
+        return [UInt8](data) + marker
     }
 }
 
@@ -416,32 +418,15 @@ extension Archive {
     /// - name: File name
     /// - prefix: Path prefix
     /// - data: File contents
-    public mutating func appendFile(name: String, prefix: String = "", data: [UInt8]) throws {
+    public func appendFile(name: String, prefix: String = "", data: [UInt8]) throws {
         try append(.init(header: .init(name: name, size: data.count, prefix: prefix), data: data))
     }
 
-    /// Adds a new file member at the end of the archive
-    /// parameters:
-    /// - name: File name
-    /// - prefix: Path prefix
-    /// - data: File contents
-    public func appendingFile(name: String, prefix: String = "", data: [UInt8]) throws -> Self {
-        try appending(.init(header: .init(name: name, size: data.count, prefix: prefix), data: data))
-    }
-
     /// Adds a new directory member at the end of the archive
     /// parameters:
     /// - name: Directory name
     /// - prefix: Path prefix
-    public mutating func appendDirectory(name: String, prefix: String = "") throws {
+    public func appendDirectory(name: String, prefix: String = "") throws {
         try append(.init(header: .init(name: name, typeflag: .DIRTYPE, prefix: prefix)))
     }
-
-    /// Adds a new directory member at the end of the archive
-    /// parameters:
-    /// - name: Directory name
-    /// - prefix: Path prefix
-    public func appendingDirectory(name: String, prefix: String = "") throws -> Self {
-        try self.appending(.init(header: .init(name: name, typeflag: .DIRTYPE, prefix: prefix)))
-    }
 }

Sources/containertool/Extensions/Archive+appending.swift

@@ -30,30 +30,28 @@ extension Archive {
     /// Parameters:
     /// - root: The path to the file or directory to add.
     /// Returns:  A new archive made by appending `root` to the receiver.
-    public func appendingRecursively(atPath root: String) throws -> Self {
+    public func appendRecursively(atPath root: String) throws {
         let url = URL(fileURLWithPath: root)
         if url.isDirectory {
-            return try self.appendingDirectoryTree(at: url)
+            try self.appendDirectoryTree(at: url)
         } else {
-            return try self.appendingFile(at: url)
+            try self.appendFile(at: url)
         }
     }
 
     /// Append a single file to the archive.
     /// Parameters:
     /// - path: The path to the file to add.
     /// Returns:  A new archive made by appending `path` to the receiver.
-    func appendingFile(at path: URL) throws -> Self {
-        try self.appendingFile(name: path.lastPathComponent, data: try [UInt8](Data(contentsOf: path)))
+    func appendFile(at path: URL) throws {
+        try self.appendFile(name: path.lastPathComponent, data: try [UInt8](Data(contentsOf: path)))
     }
 
     /// Recursively append a single directory tree to the archive.
     /// Parameters:
     /// - root: The path to the directory to add.
     /// Returns:  A new archive made by appending `root` to the receiver.
-    func appendingDirectoryTree(at root: URL) throws -> Self {
-        var ret = self
-
+    func appendDirectoryTree(at root: URL) throws {
         guard let enumerator = FileManager.default.enumerator(atPath: root.path) else {
             throw ("Unable to read \(root.path)")
         }
@@ -69,16 +67,14 @@ extension Archive {
             switch filetype {
             case .typeRegular:
                 let resource = try [UInt8](Data(contentsOf: root.appending(path: subpath)))
-                try ret.appendFile(name: subpath, prefix: root.lastPathComponent, data: resource)
+                try self.appendFile(name: subpath, prefix: root.lastPathComponent, data: resource)
 
             case .typeDirectory:
-                try ret.appendDirectory(name: subpath, prefix: root.lastPathComponent)
+                try self.appendDirectory(name: subpath, prefix: root.lastPathComponent)
 
             default:
                 throw "Resource file \(subpath) of type \(filetype) is not supported"
             }
         }
-
-        return ret
     }
 }

Sources/containertool/Extensions/RegistryClient+publish.swift

@@ -49,10 +49,11 @@ func publishContainerImage<Source: ImageSource, Destination: ImageDestination>(
 
     var resourceLayers: [(descriptor: ContentDescriptor, diffID: ImageReference.Digest)] = []
     for resourceDir in resources {
-        let resourceTardiff = try Archive().appendingRecursively(atPath: resourceDir).bytes
+        let resourceTardiff = Archive()
+        try resourceTardiff.appendRecursively(atPath: resourceDir)
         let resourceLayer = try await destination.uploadLayer(
             repository: destinationImage.repository,
-            contents: resourceTardiff
+            contents: resourceTardiff.bytes
         )
 
         if verbose {
@@ -64,9 +65,12 @@ func publishContainerImage<Source: ImageSource, Destination: ImageDestination>(
 
     // MARK: Upload the application layer
 
+    let applicationTardiff = Archive()
+    try applicationTardiff.appendFile(at: executableURL)
+
     let applicationLayer = try await destination.uploadLayer(
         repository: destinationImage.repository,
-        contents: try Archive().appendingFile(at: executableURL).bytes
+        contents: applicationTardiff.bytes
     )
     if verbose {
         log("application layer: \(applicationLayer.descriptor.digest) (\(applicationLayer.descriptor.size) bytes)")

Tests/TarTests/TarUnitTests.swift

@@ -339,12 +339,13 @@ let trailer = [UInt8](repeating: 0, count: trailerSize)
         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     ]
 
-    @Test func testAppendingEmptyFile() async throws {
-        let archive = try Archive().appendingFile(name: "emptyfile", data: []).bytes
+    @Test func testAppendEmptyFile() async throws {
+        let archive = Archive()
+        try archive.appendFile(name: "emptyfile", data: [])
 
         // Expecting: member header, no file content, 2-block end of archive marker
-        #expect(archive.count == headerSize + trailerSize)
-        #expect(archive == emptyFile + trailer)
+        #expect(archive.bytes.count == headerSize + trailerSize)
+        #expect(archive.bytes == emptyFile + trailer)
     }
 
     let helloFile: [UInt8] =
@@ -456,21 +457,12 @@ let trailer = [UInt8](repeating: 0, count: trailerSize)
         ]
 
     @Test func testAppendFile() async throws {
-        var archive = Archive()
+        let archive = Archive()
         try archive.appendFile(name: "hellofile", data: [UInt8]("hello".utf8))
-        let output = archive.bytes
 
         // Expecting: member header, file content, 2-block end of archive marker
-        #expect(output.count == headerSize + blockSize + trailerSize)
-        #expect(output == helloFile + trailer)
-    }
-
-    @Test func testAppendingFile() async throws {
-        let archive = try Archive().appendingFile(name: "hellofile", data: [UInt8]("hello".utf8)).bytes
-
-        // Expecting: member header, file content, 2-block end of archive marker
-        #expect(archive.count == headerSize + blockSize + trailerSize)
-        #expect(archive == helloFile + trailer)
+        #expect(archive.bytes.count == headerSize + blockSize + trailerSize)
+        #expect(archive.bytes == helloFile + trailer)
     }
 
     let directoryWithPrefix: [UInt8] = [
@@ -547,47 +539,35 @@ let trailer = [UInt8](repeating: 0, count: trailerSize)
     ]
 
     @Test func testAppendDirectory() async throws {
-        var archive = Archive()
+        let archive = Archive()
         try archive.appendDirectory(name: "directory", prefix: "prefix")
-        let output = archive.bytes
-
-        // Expecting: member header, no content, 2-block end of archive marker
-        #expect(output.count == headerSize + trailerSize)
-        #expect(output == directoryWithPrefix + trailer)
-    }
-
-    @Test func testAppendingDirectory() async throws {
-        let archive = try Archive().appendingDirectory(name: "directory", prefix: "prefix").bytes
 
         // Expecting: member header, no content, 2-block end of archive marker
-        #expect(archive.count == headerSize + trailerSize)
-        #expect(archive == directoryWithPrefix + trailer)
+        #expect(archive.bytes.count == headerSize + trailerSize)
+        #expect(archive.bytes == directoryWithPrefix + trailer)
     }
 
     @Test func testAppendFilesAndDirectories() async throws {
-        var archive = Archive()
+        let archive = Archive()
         try archive.appendFile(name: "hellofile", data: [UInt8]("hello".utf8))
         try archive.appendFile(name: "emptyfile", data: [UInt8]())
         try archive.appendDirectory(name: "directory", prefix: "prefix")
 
-        let output = archive.bytes
-
         // Expecting: file member header, file content, file member header, no file content,
         //     directory member header, 2-block end of archive marker
-        #expect(output.count == headerSize + blockSize + headerSize + headerSize + trailerSize)
-        #expect(output == helloFile + emptyFile + directoryWithPrefix + trailer)
+        #expect(archive.bytes.count == headerSize + blockSize + headerSize + headerSize + trailerSize)
+        #expect(archive.bytes == helloFile + emptyFile + directoryWithPrefix + trailer)
     }
 
     @Test func testAppendingFilesAndDirectories() async throws {
-        let archive = try Archive()
-            .appendingFile(name: "hellofile", data: [UInt8]("hello".utf8))
-            .appendingFile(name: "emptyfile", data: [UInt8]())
-            .appendingDirectory(name: "directory", prefix: "prefix")
-            .bytes
+        let archive = Archive()
+        try archive.appendFile(name: "hellofile", data: [UInt8]("hello".utf8))
+        try archive.appendFile(name: "emptyfile", data: [UInt8]())
+        try archive.appendDirectory(name: "directory", prefix: "prefix")
 
         // Expecting: file member header, file content, file member header, no file content,
         //     directory member header, 2-block end of archive marker
-        #expect(archive.count == headerSize + blockSize + headerSize + headerSize + trailerSize)
-        #expect(archive == helloFile + emptyFile + directoryWithPrefix + trailer)
+        #expect(archive.bytes.count == headerSize + blockSize + headerSize + headerSize + trailerSize)
+        #expect(archive.bytes == helloFile + emptyFile + directoryWithPrefix + trailer)
     }
 }