Add a modeled command-line for better checking (#330)

Swiftly has many different commands that it invokes in different
parts of its code base, supporting tooling, and tests. This list
includes tar, git, pkgutil, and dscl among oseveral others.

For the most part these tools are called using arrays that are
manually assembled at the call site. In order to exercise that
callsite in tests the test must necessarily become "medium" size
because it involves integration testing with that tool, instead of
"small", which is where most tests should be. A small test of that
array would verify that the length has the correct arguments,
number, spelling, and order. Also, there are various types of string
interpolation in these arrays, which limit the type-safety.

Introduce a new way of modeling command-line invocations using Swift
types. Commands and subcommands are modeled using structs. Use
functions to assemble the structs, and methods to assemble structs
for subcommands. Arguments become function arguments, with default
values if they are optional. Flags, and options will be described
using enum cases with carrying values for options.

Some commands are runnable, while others are not. Introduce a
Runnable protocol that marks runnable commands, and provides a
default implementation of a run() method to run the assembled command.

Other commands can produce output. Introduce an Output protocol and
default implementation that runs the command and receives the output
as a string that the callsite can use.

Some callsites make decisions based on whether a tool is present
or not. Provide a convenient way to verify whether a command exists
on the system. Use this mechanism to skip tests when they are run
in an environment that doesn't have the tool.

Pick one command, dscl, to use with the new modeled command-line.
Write small tests that don't invoke the tool. Instead they
verify the configuration that is produced with different typical
inputs from swiftly. Add a single medium size test that runs dscl
from a modeled command, enabled only if dscl exists on the system.

Add tags for medium, and large tests. Mark existing large tests,
which is the HTTPClientTests. Leave all of the remaining tests as
untagged, (i.e. small). Mark the CommandLineTest that invokes the
dscl tool as medium.

Author: cmcgee1024

Package.swift

@@ -117,6 +117,9 @@ let package = Package(
         .executableTarget(
             name: "build-swiftly-release",
             dependencies: [
+                .target(name: "SwiftlyCore"),
+                .target(name: "LinuxPlatform", condition: .when(platforms: [.linux])),
+                .target(name: "MacOSPlatform", condition: .when(platforms: [.macOS])),
                 .product(name: "ArgumentParser", package: "swift-argument-parser"),
             ],
             path: "Tools/build-swiftly-release"

Sources/MacOSPlatform/MacOS.swift

@@ -2,6 +2,7 @@ import Foundation
 import SwiftlyCore
 import SystemPackage
 
+typealias sys = SwiftlyCore.SystemCommand
 typealias fs = SwiftlyCore.FileSystem
 
 public struct SwiftPkgInfo: Codable {
@@ -191,14 +192,8 @@ public struct MacOS: Platform {
     }
 
     public func getShell() async throws -> String {
-        if let directoryInfo = try await runProgramOutput("dscl", ".", "-read", "\(fs.home)") {
-            for line in directoryInfo.components(separatedBy: "\n") {
-                if line.hasPrefix("UserShell: ") {
-                    if case let comps = line.components(separatedBy: ": "), comps.count == 2 {
-                        return comps[1]
-                    }
-                }
-            }
+        for (key, value) in try await sys.dscl(datasource: ".").read(path: fs.home, keys: "UserShell").properties(self) {
+            return value
         }
 
         // Fall back to zsh on macOS

Sources/SwiftlyCore/Commands.swift

@@ -0,0 +1,92 @@
+import Foundation
+import SystemPackage
+
+public enum SystemCommand {}
+
+// This file contains a set of system commands that's used by Swiftly and its related tests and tooling
+
+// Directory Service command line utility for macOS
+// See dscl(1) for details
+extension SystemCommand {
+    public static func dscl(executable: Executable = DsclCommand.defaultExecutable, datasource: String? = nil) -> DsclCommand {
+        DsclCommand(executable: executable, datasource: datasource)
+    }
+
+    public struct DsclCommand {
+        public static var defaultExecutable: Executable { .name("dscl") }
+
+        var executable: Executable
+        var datasource: String?
+
+        internal init(
+            executable: Executable,
+            datasource: String?
+        ) {
+            self.executable = executable
+            self.datasource = datasource
+        }
+
+        func config() -> Configuration {
+            var args: [String] = []
+
+            if let datasource = self.datasource {
+                args += [datasource]
+            }
+
+            return Configuration(
+                executable: self.executable,
+                arguments: Arguments(args),
+                environment: .inherit
+            )
+        }
+
+        public func read(path: FilePath? = nil, keys: [String]) -> ReadCommand {
+            ReadCommand(dscl: self, path: path, keys: keys)
+        }
+
+        public func read(path: FilePath? = nil, keys: String...) -> ReadCommand {
+            self.read(path: path, keys: keys)
+        }
+
+        public struct ReadCommand: Output {
+            var dscl: DsclCommand
+            var path: FilePath?
+            var keys: [String]
+
+            internal init(dscl: DsclCommand, path: FilePath?, keys: [String]) {
+                self.dscl = dscl
+                self.path = path
+                self.keys = keys
+            }
+
+            public func config() -> Configuration {
+                var c = self.dscl.config()
+
+                var args = c.arguments.storage.map(\.description) + ["-read"]
+
+                if let path = self.path {
+                    args += [path.string] + self.keys
+                }
+
+                c.arguments = .init(args)
+
+                return c
+            }
+        }
+    }
+}
+
+extension SystemCommand.DsclCommand.ReadCommand {
+    public func properties(_ p: Platform) async throws -> [(key: String, value: String)] {
+        let output = try await self.output(p)
+        guard let output else { return [] }
+
+        var props: [(key: String, value: String)] = []
+        for line in output.components(separatedBy: "\n") {
+            if case let comps = line.components(separatedBy: ": "), comps.count == 2 {
+                props.append((key: comps[0], value: comps[1]))
+            }
+        }
+        return props
+    }
+}

Sources/SwiftlyCore/ModeledCommandLine.swift

@@ -0,0 +1,177 @@
+import Foundation
+import SystemPackage
+
+public enum CommandLineError: Error {
+    case invalidArgs
+    case errorExit(exitCode: Int32, program: String)
+    case unknownVersion
+}
+
+// This section is a clone of the Configuration type from the new Subprocess package, until we can depend on that package.
+public struct Configuration: Sendable {
+    /// The executable to run.
+    public var executable: Executable
+    /// The arguments to pass to the executable.
+    public var arguments: Arguments
+    /// The environment to use when running the executable.
+    public var environment: Environment
+}
+
+public struct Executable: Sendable, Hashable {
+    internal enum Storage: Sendable, Hashable {
+        case executable(String)
+        case path(FilePath)
+    }
+
+    internal let storage: Storage
+
+    private init(_config: Storage) {
+        self.storage = _config
+    }
+
+    /// Locate the executable by its name.
+    /// `Subprocess` will use `PATH` value to
+    /// determine the full path to the executable.
+    public static func name(_ executableName: String) -> Self {
+        .init(_config: .executable(executableName))
+    }
+
+    /// Locate the executable by its full path.
+    /// `Subprocess` will use this  path directly.
+    public static func path(_ filePath: FilePath) -> Self {
+        .init(_config: .path(filePath))
+    }
+}
+
+public struct Environment: Sendable, Hashable {
+    internal enum Configuration: Sendable, Hashable {
+        case inherit([String: String])
+        case custom([String: String])
+    }
+
+    internal let config: Configuration
+
+    init(config: Configuration) {
+        self.config = config
+    }
+
+    /// Child process should inherit the same environment
+    /// values from its parent process.
+    public static var inherit: Self {
+        .init(config: .inherit([:]))
+    }
+
+    /// Override the provided `newValue` in the existing `Environment`
+    public func updating(_ newValue: [String: String]) -> Self {
+        .init(config: .inherit(newValue))
+    }
+
+    /// Use custom environment variables
+    public static func custom(_ newValue: [String: String]) -> Self {
+        .init(config: .custom(newValue))
+    }
+}
+
+internal enum StringOrRawBytes: Sendable, Hashable {
+    case string(String)
+
+    var stringValue: String? {
+        switch self {
+        case let .string(string):
+            return string
+        }
+    }
+
+    var description: String {
+        switch self {
+        case let .string(string):
+            return string
+        }
+    }
+
+    var count: Int {
+        switch self {
+        case let .string(string):
+            return string.count
+        }
+    }
+
+    func hash(into hasher: inout Hasher) {
+        // If Raw bytes is valid UTF8, hash it as so
+        switch self {
+        case let .string(string):
+            hasher.combine(string)
+        }
+    }
+}
+
+public struct Arguments: Sendable, ExpressibleByArrayLiteral, Hashable {
+    public typealias ArrayLiteralElement = String
+
+    internal let storage: [StringOrRawBytes]
+
+    /// Create an Arguments object using the given literal values
+    public init(arrayLiteral elements: String...) {
+        self.storage = elements.map { .string($0) }
+    }
+
+    /// Create an Arguments object using the given array
+    public init(_ array: [String]) {
+        self.storage = array.map { .string($0) }
+    }
+}
+
+public protocol Runnable {
+    func config() -> Configuration
+}
+
+extension Runnable {
+    public func run(_ p: Platform, quiet: Bool) async throws {
+        let c = self.config()
+        let executable = switch c.executable.storage {
+        case let .executable(name):
+            name
+        case let .path(p):
+            p.string
+        }
+        let args = c.arguments.storage.map(\.description)
+        var env: [String: String] = ProcessInfo.processInfo.environment
+        switch c.environment.config {
+        case let .inherit(newValue):
+            for (key, value) in newValue {
+                env[key] = value
+            }
+        case let .custom(newValue):
+            env = newValue
+        }
+        try await p.runProgram([executable] + args, quiet: quiet, env: env)
+    }
+}
+
+public protocol Output {
+    func config() -> Configuration
+}
+
+// TODO: look into making this something that can be Decodable (i.e. streamable)
+extension Output {
+    public func output(_ p: Platform) async throws -> String? {
+        let c = self.config()
+        let executable = switch c.executable.storage {
+        case let .executable(name):
+            name
+        case let .path(p):
+            p.string
+        }
+        let args = c.arguments.storage.map(\.description)
+        var env: [String: String] = ProcessInfo.processInfo.environment
+        switch c.environment.config {
+        case let .inherit(newValue):
+            for (key, value) in newValue {
+                env[key] = value
+            }
+        case let .custom(newValue):
+            env = newValue
+        }
+        return try await p.runProgramOutput(executable, args, env: env)
+    }
+}

Tests/SwiftlyTests/CommandLineTests.swift

@@ -0,0 +1,31 @@
+@testable import Swiftly
+@testable import SwiftlyCore
+import SystemPackage
+import Testing
+
+public typealias sys = SystemCommand
+
+@Suite
+public struct CommandLineTests {
+    @Test func testDsclModel() {
+        var config = sys.dscl(datasource: ".").read(path: .init("/Users/swiftly"), keys: "UserShell").config()
+        #expect(config.executable == .name("dscl"))
+        #expect(config.arguments.storage.map(\.description) == [".", "-read", "/Users/swiftly", "UserShell"])
+
+        config = sys.dscl(datasource: ".").read(path: .init("/Users/swiftly"), keys: "UserShell", "Picture").config()
+        #expect(config.executable == .name("dscl"))
+        #expect(config.arguments.storage.map(\.description) == [".", "-read", "/Users/swiftly", "UserShell", "Picture"])
+    }
+
+    @Test(
+        .tags(.medium),
+        .enabled {
+            try await sys.DsclCommand.defaultExecutable.exists()
+        }
+    )
+    func testDscl() async throws {
+        let properties = try await sys.dscl(datasource: ".").read(path: fs.home, keys: "UserShell").properties(Swiftly.currentPlatform)
+        #expect(properties.count == 1) // Only one shell for the current user
+        #expect(properties[0].key == "UserShell") // The one property key should be the one that is requested
+    }
+}

Tests/SwiftlyTests/HTTPClientTests.swift

@@ -7,7 +7,7 @@ import SystemPackage
 import Testing
 
 @Suite(.serialized) struct HTTPClientTests {
-    @Test func getSwiftOrgGPGKeys() async throws {
+    @Test(.tags(.large)) func getSwiftOrgGPGKeys() async throws {
         let tmpFile = fs.mktemp()
         try await fs.create(file: tmpFile, contents: nil)
 
@@ -24,7 +24,7 @@ import Testing
         }
     }
 
-    @Test func getSwiftToolchain() async throws {
+    @Test(.tags(.large)) func getSwiftToolchain() async throws {
         let tmpFile = fs.mktemp()
         try await fs.create(file: tmpFile, contents: nil)
         let tmpFileSignature = fs.mktemp(ext: ".sig")
@@ -53,7 +53,7 @@ import Testing
         }
     }
 
-    @Test func getSwiftlyRelease() async throws {
+    @Test(.tags(.large)) func getSwiftlyRelease() async throws {
         let tmpFile = fs.mktemp()
         try await fs.create(file: tmpFile, contents: nil)
         let tmpFileSignature = fs.mktemp(ext: ".sig")
@@ -82,7 +82,7 @@ import Testing
         }
     }
 
-    @Test func getSwiftlyReleaseMetadataFromSwiftOrg() async throws {
+    @Test(.tags(.large)) func getSwiftlyReleaseMetadataFromSwiftOrg() async throws {
         let httpClient = SwiftlyHTTPClient(httpRequestExecutor: HTTPRequestExecutorImpl())
         do {
             let currentRelease = try await httpClient.getCurrentSwiftlyRelease()
@@ -94,6 +94,7 @@ import Testing
     }
 
     @Test(
+        .tags(.large),
         arguments:
         [PlatformDefinition.macOS, .ubuntu2404, .ubuntu2204, .rhel9, .fedora39, .amazonlinux2, .debian12],
         [SwiftlyWebsiteAPI.Components.Schemas.Architecture.x8664, .aarch64]