Generate command models from argument parser info JSON files (#345)

Instead of hand-crafting the helper functions, options, and structs for each command,
describe the command structure using an argument parser compatible JSON format
for each command. Introduce a generator plugin that generates all of the Swift types in
a manner that is similar to the way that they were hand-written before.

There are certain limitations in the JSON schema for the Swift argument parser that
prevent the generator from knowing certain important details. For example, there is
no information on which arguments contain file paths, which is important information
for type safety in this context. For this information, and more potentially in the future
introduce a "-ext.json" file with an extension schema to hold the extra information.

Author: cmcgee1024

Package.swift

@@ -68,7 +68,8 @@ let package = Package(
                 .product(name: "OpenAPIAsyncHTTPClient", package: "swift-openapi-async-http-client"),
                 .product(name: "SystemPackage", package: "swift-system"),
             ],
-            swiftSettings: swiftSettings
+            swiftSettings: swiftSettings,
+            plugins: ["GenerateCommandModels"]
         ),
         .target(
             name: "SwiftlyDownloadAPI",
@@ -107,13 +108,28 @@ let package = Package(
             ),
             dependencies: ["generate-docs-reference"]
         ),
+        .plugin(
+            name: "GenerateCommandModels",
+            capability: .buildTool(),
+            dependencies: [
+                "generate-command-models",
+            ]
+        ),
         .executableTarget(
             name: "generate-docs-reference",
             dependencies: [
                 .product(name: "ArgumentParser", package: "swift-argument-parser"),
             ],
             path: "Tools/generate-docs-reference"
         ),
+        .executableTarget(
+            name: "generate-command-models",
+            dependencies: [
+                .product(name: "ArgumentParser", package: "swift-argument-parser"),
+                .product(name: "SystemPackage", package: "swift-system"),
+            ],
+            path: "Tools/generate-command-models"
+        ),
         .executableTarget(
             name: "build-swiftly-release",
             dependencies: [

Plugins/GenerateCommandModels/plugin.swift

@@ -0,0 +1,24 @@
+import PackagePlugin
+
+@main
+struct GenerateCommandModelsPlugin: BuildToolPlugin {
+    func createBuildCommands(context: PluginContext, target: Target) throws -> [Command] {
+        guard let target = target as? SourceModuleTarget else { return [] }
+
+        let jsonSources = target.sourceFiles.map(\.path).filter { $0.extension == "json" }
+
+        guard jsonSources.count > 0 else { return [] }
+
+        let outputPath = context.pluginWorkDirectory.appending("Commands.swift")
+
+        return [
+            .buildCommand(
+                displayName: "Generating Command Models from dumped JSON help",
+                executable: try context.tool(named: "generate-command-models").path,
+                arguments: ["--output-file", outputPath] + jsonSources,
+                inputFiles: jsonSources,
+                outputFiles: [outputPath]
+            ),
+        ]
+    }
+}

Sources/LinuxPlatform/Linux.swift

@@ -277,9 +277,9 @@ public struct Linux: Platform {
                 if let mockedHomeDir = ctx.mockedHomeDir {
                     var env = ProcessInfo.processInfo.environment
                     env["GNUPGHOME"] = (mockedHomeDir / ".gnupg").string
-                    try await sys.gpg()._import(keys: tmpFile).run(self, env: env, quiet: true)
+                    try await sys.gpg()._import(key: tmpFile).run(self, env: env, quiet: true)
                 } else {
-                    try await sys.gpg()._import(keys: tmpFile).run(self, quiet: true)
+                    try await sys.gpg()._import(key: tmpFile).run(self, quiet: true)
                 }
             }
         }
@@ -418,9 +418,9 @@ public struct Linux: Platform {
                 if let mockedHomeDir = ctx.mockedHomeDir {
                     var env = ProcessInfo.processInfo.environment
                     env["GNUPGHOME"] = (mockedHomeDir / ".gnupg").string
-                    try await sys.gpg().verify(detachedSignature: sigFile, signedData: archive).run(self, env: env, quiet: false)
+                    try await sys.gpg().verify(detached_signature: sigFile, signed_data: archive).run(self, env: env, quiet: false)
                 } else {
-                    try await sys.gpg().verify(detachedSignature: sigFile, signedData: archive).run(self, quiet: !verbose)
+                    try await sys.gpg().verify(detached_signature: sigFile, signed_data: archive).run(self, quiet: !verbose)
                 }
             } catch {
                 throw SwiftlyError(message: "Signature verification failed: \(error).")
@@ -447,9 +447,9 @@ public struct Linux: Platform {
                 if let mockedHomeDir = ctx.mockedHomeDir {
                     var env = ProcessInfo.processInfo.environment
                     env["GNUPGHOME"] = (mockedHomeDir / ".gnupg").string
-                    try await sys.gpg().verify(detachedSignature: sigFile, signedData: archive).run(self, env: env, quiet: false)
+                    try await sys.gpg().verify(detached_signature: sigFile, signed_data: archive).run(self, env: env, quiet: false)
                 } else {
-                    try await sys.gpg().verify(detachedSignature: sigFile, signedData: archive).run(self, quiet: !verbose)
+                    try await sys.gpg().verify(detached_signature: sigFile, signed_data: archive).run(self, quiet: !verbose)
                 }
             } catch {
                 throw SwiftlyError(message: "Signature verification failed: \(error).")
@@ -603,7 +603,7 @@ public struct Linux: Platform {
 
     public func getShell() async throws -> String {
         let userName = ProcessInfo.processInfo.userName
-        if let entry = try await sys.getent(database: "passwd", keys: userName).entries(self).first {
+        if let entry = try await sys.getent(database: "passwd", key: userName).entries(self).first {
             if let shell = entry.last { return shell }
         }
 

Sources/MacOSPlatform/MacOS.swift

@@ -71,7 +71,7 @@ public struct MacOS: Platform {
             // If the toolchains go into the default user location then we use the installer to install them
             await ctx.print("Installing package in user home directory...")
 
-            try await sys.installer(.verbose, pkg: tmpFile, target: "CurrentUserHomeDirectory").run(self, quiet: !verbose)
+            try await sys.installer(.verbose, .pkg(tmpFile), .target("CurrentUserHomeDirectory")).run(self, quiet: !verbose)
         } else {
             // Otherwise, we extract the pkg into the requested toolchains directory.
             await ctx.print("Expanding pkg...")
@@ -84,7 +84,7 @@ public struct MacOS: Platform {
 
             await ctx.print("Checking package signature...")
             do {
-                try await sys.pkgutil().checkSignature(pkgPath: tmpFile).run(self, quiet: !verbose)
+                try await sys.pkgutil().checksignature(pkg_path: tmpFile).run(self, quiet: !verbose)
             } catch {
                 // If this is not a test that uses mocked toolchains then we must throw this error and abort installation
                 guard ctx.mockedHomeDir != nil else {
@@ -94,7 +94,7 @@ public struct MacOS: Platform {
                 // We permit the signature verification to fail during testing
                 await ctx.print("Signature verification failed, which is allowable during testing with mocked toolchains")
             }
-            try await sys.pkgutil(.verbose).expand(pkgPath: tmpFile, dirPath: tmpDir).run(self, quiet: !verbose)
+            try await sys.pkgutil(.verbose).expand(pkg_path: tmpFile, dir_path: tmpDir).run(self, quiet: !verbose)
 
             // There's a slight difference in the location of the special Payload file between official swift packages
             // and the ones that are mocked here in the test framework.
@@ -118,18 +118,18 @@ public struct MacOS: Platform {
         if ctx.mockedHomeDir == nil {
             await ctx.print("Extracting the swiftly package...")
             try await sys.installer(
-                pkg: archive,
-                target: "CurrentUserHomeDirectory"
+                .pkg(archive),
+                .target("CurrentUserHomeDirectory")
             )
-            try? await sys.pkgutil(.volume(userHomeDir)).forget(packageId: "org.swift.swiftly").run(self)
+            try? await sys.pkgutil(.volume(userHomeDir)).forget(pkg_id: "org.swift.swiftly").run(self)
         } else {
             let installDir = userHomeDir / ".swiftly"
             try await fs.mkdir(.parents, atPath: installDir)
 
             // In the case of a mock for testing purposes we won't use the installer, perferring a manual process because
             //  the installer will not install to an arbitrary path, only a volume or user home directory.
             let tmpDir = fs.mktemp()
-            try await sys.pkgutil().expand(pkgPath: archive, dirPath: tmpDir).run(self)
+            try await sys.pkgutil().expand(pkg_path: archive, dir_path: tmpDir).run(self)
 
             // There's a slight difference in the location of the special Payload file between official swift packages
             // and the ones that are mocked here in the test framework.
@@ -164,7 +164,7 @@ public struct MacOS: Platform {
 
         try await fs.remove(atPath: toolchainDir)
 
-        try? await sys.pkgutil(.volume(fs.home)).forget(packageId: pkgInfo.CFBundleIdentifier).run(self, quiet: !verbose)
+        try? await sys.pkgutil(.volume(fs.home)).forget(pkg_id: pkgInfo.CFBundleIdentifier).run(self, quiet: !verbose)
     }
 
     public func getExecutableName() -> String {
@@ -193,7 +193,7 @@ public struct MacOS: Platform {
     }
 
     public func getShell() async throws -> String {
-        for (key, value) in try await sys.dscl(datasource: ".").read(path: fs.home, keys: "UserShell").properties(self) {
+        for (key, value) in try await sys.dscl(datasource: ".").read(path: fs.home, key: ["UserShell"]).properties(self) {
             return value
         }
 

Sources/SwiftlyCore/Commands+Runnable+Output.swift

@@ -0,0 +1,67 @@
+import Foundation
+import SystemPackage
+
+extension SystemCommand.dsclCommand.readCommand: Output {
+    public func properties(_ p: Platform) async throws -> [(key: String, value: String)] {
+        let output = try await self.output(p)
+        guard let output else { return [] }
+
+        var props: [(key: String, value: String)] = []
+        for line in output.components(separatedBy: "\n") {
+            if case let comps = line.components(separatedBy: ": "), comps.count == 2 {
+                props.append((key: comps[0], value: comps[1]))
+            }
+        }
+        return props
+    }
+}
+
+extension SystemCommand.lipoCommand.createCommand: Runnable {}
+
+extension SystemCommand.pkgbuildCommand: Runnable {}
+
+extension SystemCommand.getentCommand: Output {
+    public func entries(_ platform: Platform) async throws -> [[String]] {
+        let output = try await output(platform)
+        guard let output else { return [] }
+
+        var entries: [[String]] = []
+        for line in output.components(separatedBy: "\n") {
+            entries.append(line.components(separatedBy: ":"))
+        }
+        return entries
+    }
+}
+
+extension SystemCommand.gitCommand.logCommand: Output {}
+extension SystemCommand.gitCommand.diffindexCommand: Runnable {}
+extension SystemCommand.gitCommand.initCommand: Runnable {}
+extension SystemCommand.gitCommand.commitCommand: Runnable {}
+
+extension SystemCommand.tarCommand.createCommand: Runnable {}
+extension SystemCommand.tarCommand.extractCommand: Runnable {}
+
+extension SystemCommand.swiftCommand.packageCommand.resetCommand: Runnable {}
+extension SystemCommand.swiftCommand.packageCommand.cleanCommand: Runnable {}
+extension SystemCommand.swiftCommand.packageCommand.initCommand: Runnable {}
+extension SystemCommand.swiftCommand.sdkCommand.installCommand: Runnable {}
+extension SystemCommand.swiftCommand.sdkCommand.removeCommand: Runnable {}
+extension SystemCommand.swiftCommand.buildCommand: Runnable {}
+
+extension SystemCommand.makeCommand: Runnable {}
+extension SystemCommand.makeCommand.installCommand: Runnable {}
+
+extension SystemCommand.stripCommand: Runnable {}
+
+extension SystemCommand.sha256sumCommand: Output {}
+
+extension SystemCommand.productbuildCommand: Runnable {}
+
+extension SystemCommand.gpgCommand.importCommand: Runnable {}
+extension SystemCommand.gpgCommand.verifyCommand: Runnable {}
+
+extension SystemCommand.pkgutilCommand.checksignatureCommand: Runnable {}
+extension SystemCommand.pkgutilCommand.expandCommand: Runnable {}
+extension SystemCommand.pkgutilCommand.forgetCommand: Runnable {}
+
+extension SystemCommand.installerCommand: Runnable {}