Merge branch 'main' of github.com:cmcgee1024/swiftly into proxies-design

Author: cmcgee1024

.github/workflows/pull_request.yml

@@ -0,0 +1,52 @@
+name: Pull request
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+
+jobs:
+  soundness:
+    name: Soundness
+    uses: swiftlang/github-workflows/.github/workflows/soundness.yml@main
+    with:
+      license_header_check_enabled: false
+      license_header_check_project_name: "Swift.org"
+      api_breakage_check_enabled: false
+      docs_check_enabled: false
+      format_check_enabled: false
+      shell_check_enabled: false
+      unacceptable_language_check_enabled: true
+
+  tests:
+    name: Test
+    uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main
+    with:
+      # linux_os_versions: "[\"jammy\", \"noble\", \"focal\", \"amazonlinux2\", \"rhel-ubi9\", \"bookworm\", \"fedora39\"]"
+      # Not working: noble (compile error in TSC FileSystem), bookworm (missing memory.h), fedora39 (missing memory.h)
+      linux_os_versions: "[\"jammy\", \"focal\", \"rhel-ubi9\"]"
+      # We only care about the current stable release, because that's where we make our swiftly releases
+      linux_exclude_swift_versions: "[{\"swift_version\": \"nightly-main\"},{\"swift_version\": \"nightly-6.0\"},{\"swift_version\": \"5.8\"},{\"swift_version\": \"5.9\"},{\"swift_version\": \"5.10\"}]"
+      linux_pre_build_command: ((apt-get update && apt-get -y install curl make) || ((curl --help || yum -y install curl) && yum -y install make)) && ./scripts/install-libarchive.sh
+      enable_windows_checks: false
+
+  releasebuild:
+    name: Release Build
+    uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main
+    with:
+      linux_os_versions: "[\"rhel-ubi9\"]"
+      # We only care about the current stable release, because that's where we make our swiftly releases
+      linux_exclude_swift_versions: "[{\"swift_version\": \"nightly-main\"},{\"swift_version\": \"nightly-6.0\"},{\"swift_version\": \"5.8\"},{\"swift_version\": \"5.9\"},{\"swift_version\": \"5.10\"}]"
+      linux_pre_build_command: echo ""
+      linux_build_command: swift run build-swiftly-release --skip 0.4.0
+      enable_windows_checks: false
+
+  formatcheck:
+    name: Format Check
+    uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main
+    with:
+      # We only need to run this with one swift release and on one of the linux distributions
+      linux_os_versions: "[\"jammy\"]"
+      linux_exclude_swift_versions: "[{\"swift_version\": \"nightly-main\"},{\"swift_version\": \"nightly-6.0\"},{\"swift_version\": \"5.8\"},{\"swift_version\": \"5.9\"},{\"swift_version\": \"5.10\"}]"
+      linux_pre_build_command: echo ""
+      linux_build_command: swift run swiftformat --lint --dryrun .
+      enable_windows_checks: false

NOTICE.txt

@@ -56,6 +56,6 @@ following third party libraries:
 * libarchive - Copyright (c) 2003-2018 Tim Kientzle. All rights reserved.
 
   * LICENSE ("New BSD license"):
-    * https://raw.githubusercontent.com/libarchive/libarchive/master/COPYING
+    * https://raw.githubusercontent.com/libarchive/libarchive/release/COPYING
   * HOMEPAGE:
     * http://www.libarchive.org/

RELEASING.md

@@ -13,7 +13,7 @@ Swift has a tool for producing final product packages, suitable for distribution
 5. Create a tag on that commit with the format "x.y.z". Do not omit "z", even if its value is 0.
 
 6. Build the executables for the release by running `swift run build-swiftly-release <version>` from the root of the swiftly repository
-  * Build on a Apple silicon macOS machine to produce a universal package for x86_64 and arm64
+  * Build on an Apple macOS machine to produce a universal package for x86_64 and arm64 (add the --cert option to provide a signing certificate file for the .pkg)
   * Build on an Amazon Linux 2 image for x86_64
   * Build on an Amazon Linux 2 image for arm64
 

Sources/LinuxPlatform/Linux.swift

@@ -162,11 +162,11 @@ public struct Linux: Platform {
 
         let manager: String? = switch platformName {
         case "ubuntu1804":
-            "apt"
+            "apt-get"
         case "ubuntu2004":
-            "apt"
+            "apt-get"
         case "ubuntu2204":
-            "apt"
+            "apt-get"
         case "amazonlinux2":
             "yum"
         case "ubi9":
@@ -214,21 +214,38 @@ public struct Linux: Platform {
             return nil
         }
 
-        let missingPackages = packages.filter { !self.isSystemPackageInstalled(manager, $0) }.joined(separator: " ")
+        var missingPackages: [String] = []
+
+        for pkg in packages {
+            if case let pkgInstalled = await self.isSystemPackageInstalled(manager, pkg), !pkgInstalled {
+                missingPackages.append(pkg)
+            }
+        }
 
         guard !missingPackages.isEmpty else {
             return nil
         }
 
-        return "\(manager) -y install \(missingPackages)"
+        return "\(manager) -y install \(missingPackages.joined(separator: " "))"
     }
 
-    public func isSystemPackageInstalled(_ manager: String?, _ package: String) -> Bool {
+    public func isSystemPackageInstalled(_ manager: String?, _ package: String) async -> Bool {
         do {
             switch manager {
-            case "apt":
-                try self.runProgram("dpkg", "-l", package, quiet: true)
-                return true
+            case "apt-get":
+                if let pkgList = try await self.runProgramOutput("dpkg", "-l", package) {
+                    // The package might be listed but not in an installed non-error state.
+                    //
+                    // Look for something like this:
+                    //
+                    //   Desired=Unknown/Install/Remove/Purge/Hold
+                    //   | Status=Not/Inst/Conf-files/Unpacked/halF-conf/Half-inst/trig-aWait/Trig-pend
+                    //   |/ Err?=(none)/Reinst-required (Status,Err: uppercase=bad)
+                    //   ||/
+                    //   ii  pkgfoo         1.0.0ubuntu12        My description goes here....
+                    return pkgList.contains("\nii ")
+                }
+                return false
             case "yum":
                 try self.runProgram("yum", "list", "installed", package, quiet: true)
                 return true

Tools/build-swiftly-release/BuildSwiftlyRelease.swift

@@ -126,7 +126,7 @@ public func getShell() async throws -> String {
 }
 #endif
 
-public func isAmazonLinux2() -> Bool {
+public func isRHEL9() -> Bool {
     let osReleaseFiles = ["/etc/os-release", "/usr/lib/os-release"]
     var releaseFile: String?
     for file in osReleaseFiles {
@@ -165,7 +165,7 @@ public func isAmazonLinux2() -> Bool {
         return false
     }
 
-    guard let versionID = versionID, versionID == "2", (id + idlike).contains("amzn") else {
+    guard let versionID, versionID.hasPrefix("9"), (id + idlike).contains("rhel") else {
         return false
     }
 
@@ -182,6 +182,14 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
     @Flag(name: .long, help: "Skip the git repo checks and proceed.")
     var skip: Bool = false
 
+#if os(macOS)
+    @Option(help: "Installation certificate to use when building the macOS package")
+    var cert: String?
+
+    @Option(help: "Package identifier of macOS package")
+    var identifier: String = "org.swift.swiftly"
+#endif
+
     @Argument(help: "Version of swiftly to build the release.")
     var version: String
 
@@ -191,7 +199,7 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
 #if os(Linux)
         try await self.buildLinuxRelease()
 #elseif os(macOS)
-        try await self.buildMacOSRelease()
+        try await self.buildMacOSRelease(cert: self.cert, identifier: self.identifier)
 #else
         #error("Unsupported OS")
 #endif
@@ -234,6 +242,10 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
     }
 
     func checkSwiftRequirement() async throws -> String {
+        guard !self.skip else {
+            return try await self.assertTool("swift", message: "Please install swift and make sure that it is added to your path.")
+        }
+
         guard let requiredSwiftVersion = try? self.findSwiftVersion() else {
             throw Error(message: "Unable to determine the required swift version for this version of swiftly. Please make sure that you `cd <swiftly_git_dir>` and there is a .swift-version file there.")
         }
@@ -275,7 +287,7 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
 
     func buildLinuxRelease() async throws {
         // Check system requirements
-        guard isAmazonLinux2() else {
+        guard isRHEL9() else {
             // TODO: see if docker can be used to spawn an Amazon Linux 2 container to continue the release building process
             throw Error(message: "Linux releases must be made from Amazon Linux 2 because it has the oldest version of glibc for maximum compatibility with other versions of Linux")
         }
@@ -286,13 +298,19 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
         let make = try await self.assertTool("make", message: "Please install make with `yum install make`")
         let git = try await self.assertTool("git", message: "Please install git with `yum install git`")
         let strip = try await self.assertTool("strip", message: "Please install strip with `yum install binutils`")
+        let sha256sum = try await self.assertTool("sha256sum", message: "Please install sha256sum with `yum install coreutils`")
 
         let swift = try await self.checkSwiftRequirement()
 
         try await self.checkGitRepoStatus(git)
 
-        // Build a specific version of libarchive
+        // Start with a fresh SwiftPM package
+        try runProgram(swift, "package", "reset")
+
+        // Build a specific version of libarchive with a check on the tarball's SHA256
         let libArchiveVersion = "3.7.4"
+        let libArchiveTarSha = "7875d49596286055b52439ed42f044bd8ad426aa4cc5aabd96bfe7abb971d5e8"
+
         let buildCheckoutsDir = FileManager.default.currentDirectoryPath + "/.build/checkouts"
         let libArchivePath = buildCheckoutsDir + "/libarchive-\(libArchiveVersion)"
         let pkgConfigPath = libArchivePath + "/pkgconfig"
@@ -302,6 +320,11 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
 
         try? FileManager.default.removeItem(atPath: libArchivePath)
         try runProgram(curl, "-o", "\(buildCheckoutsDir + "/libarchive-\(libArchiveVersion).tar.gz")", "--remote-name", "--location", "https://github.com/libarchive/libarchive/releases/download/v\(libArchiveVersion)/libarchive-\(libArchiveVersion).tar.gz")
+        let libArchiveTarShaActual = try await runProgramOutput(sha256sum, "\(buildCheckoutsDir)/libarchive-\(libArchiveVersion).tar.gz")
+        guard let libArchiveTarShaActual, libArchiveTarShaActual.starts(with: libArchiveTarSha) else {
+            let shaActual = libArchiveTarShaActual ?? "none"
+            throw Error(message: "The libarchive tar.gz file sha256sum is \(shaActual), but expected \(libArchiveTarSha)")
+        }
         try runProgram(tar, "--directory=\(buildCheckoutsDir)", "-xzf", "\(buildCheckoutsDir)/libarchive-\(libArchiveVersion).tar.gz")
 
         let cwd = FileManager.default.currentDirectoryPath
@@ -338,8 +361,6 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
 
         FileManager.default.changeCurrentDirectoryPath(cwd)
 
-        try runProgram(swift, "package", "clean")
-
         // Statically link standard libraries for maximum portability of the swiftly binary
         try runProgram(swift, "build", "--product=swiftly", "--pkg-config-path=\(pkgConfigPath)/lib/pkgconfig", "--static-swift-stdlib", "--configuration=release")
 
@@ -361,7 +382,7 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
         print(releaseArchive)
     }
 
-    func buildMacOSRelease() async throws {
+    func buildMacOSRelease(cert: String?, identifier: String) async throws {
         // Check system requirements
         let git = try await self.assertTool("git", message: "Please install git with either `xcode-select --install` or `brew install git`")
 
@@ -389,17 +410,34 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
         try? FileManager.default.createDirectory(atPath: swiftlyLicenseDir, withIntermediateDirectories: true)
         try await self.collectLicenses(swiftlyLicenseDir)
 
-        try runProgram(
-            pkgbuild,
-            "--root",
-            swiftlyBinDir + "/..",
-            "--install-location",
-            "usr/local",
-            "--version",
-            self.version,
-            "--identifier",
-            "org.swift.swiftly",
-            ".build/release/swiftly-\(self.version).pkg"
-        )
+        if let cert {
+            try runProgram(
+                pkgbuild,
+                "--root",
+                swiftlyBinDir + "/..",
+                "--install-location",
+                "usr/local",
+                "--version",
+                self.version,
+                "--identifier",
+                identifier,
+                "--sign",
+                cert,
+                ".build/release/swiftly-\(self.version).pkg"
+            )
+        } else {
+            try runProgram(
+                pkgbuild,
+                "--root",
+                swiftlyBinDir + "/..",
+                "--install-location",
+                "usr/local",
+                "--version",
+                self.version,
+                "--identifier",
+                identifier,
+                ".build/release/swiftly-\(self.version).pkg"
+            )
+        }
     }
 }