gh-distribute-toolchain: Remove toolchain packaging code

Author: kateinoigakukun

tools/gh-distribute-toolchain

@@ -8,7 +8,6 @@
 import os
 import sys
 import subprocess
-import tarfile
 import asyncio
 import json
 from dataclasses import dataclass
@@ -21,11 +20,6 @@ USER_AGENT = "gh-distribute-toolchain by swiftwasm/swiftwasm-build"
 
 @dataclass
 class Secrets:
-    DARWIN_TOOLCHAIN_APPLICATION_CERT: str
-    DARWIN_TOOLCHAIN_INSTALLER_CERT: str
-    DARWIN_TOOLCHAIN_NOTARIZE_EMAIL: str
-    DARWIN_TOOLCHAIN_NOTARIZE_TEAM_ID: str
-    DARWIN_TOOLCHAIN_NOTARIZE_PASSWORD: str
     GITHUB_TOKEN: str
 
     @staticmethod
@@ -40,16 +34,6 @@ class Secrets:
             github_token = result.stdout.decode("utf-8").strip()
 
         return Secrets(
-            DARWIN_TOOLCHAIN_APPLICATION_CERT=Secrets.env_value(
-                "DARWIN_TOOLCHAIN_APPLICATION_CERT"),
-            DARWIN_TOOLCHAIN_INSTALLER_CERT=Secrets.env_value(
-                "DARWIN_TOOLCHAIN_INSTALLER_CERT"),
-            DARWIN_TOOLCHAIN_NOTARIZE_EMAIL=Secrets.env_value(
-                "DARWIN_TOOLCHAIN_NOTARIZE_EMAIL"),
-            DARWIN_TOOLCHAIN_NOTARIZE_TEAM_ID=Secrets.env_value(
-                "DARWIN_TOOLCHAIN_NOTARIZE_TEAM_ID"),
-            DARWIN_TOOLCHAIN_NOTARIZE_PASSWORD=Secrets.env_value(
-                "DARWIN_TOOLCHAIN_NOTARIZE_PASSWORD"),
             GITHUB_TOKEN=github_token,
         )
 
@@ -58,130 +42,6 @@ class Secrets:
         return os.environ.get(key)
 
 
-class DarwinToolchainPackaging:
-    def __init__(self, secrets: Secrets,
-                 dry_run: bool = False, verbose: bool = False):
-        self.secrets = secrets
-        self.dry_run = dry_run
-        self.verbose = verbose
-
-    async def package(self, toolchain_dir: str, tag_name: str, pkg_path: str,
-                swift_source_dir: str):
-        self.update_info_plist(toolchain_dir)
-        self.sign_darwin_toolchain(toolchain_dir)
-        await self.create_installer(toolchain_dir, pkg_path, tag_name, swift_source_dir)
-
-    def update_info_plist(self, toolchain_dir: str):
-        display_name = os.environ.get("DARWIN_TOOLCHAIN_DISPLAY_NAME")
-        info_plist = f"{toolchain_dir}/Info.plist"
-        if display_name:
-            subprocess.check_output([
-                "/usr/libexec/PlistBuddy",
-                "-c", f"Set DisplayName {display_name}", info_plist
-            ])
-
-        display_name_short = os.environ.get(
-            "DARWIN_TOOLCHAIN_DISPLAY_NAME_SHORT")
-        if display_name_short:
-            subprocess.check_output([
-                "/usr/libexec/PlistBuddy",
-                "-c", f"Set ShortDisplayName {display_name_short}", info_plist
-            ])
-
-    def sign_darwin_toolchain(self, toolchain_dir: str):
-        if self.secrets.DARWIN_TOOLCHAIN_APPLICATION_CERT is None:
-            raise Exception("Missing DARWIN_TOOLCHAIN_APPLICATION_CERT")
-
-        codesign_args = [
-            "/usr/bin/codesign",
-            "--force", "--verify", "--verbose", "--deep",
-            "--options", "runtime", "--timestamp",
-            "--sign", self.secrets.DARWIN_TOOLCHAIN_APPLICATION_CERT
-        ]
-
-        for root, dirs, files in os.walk(toolchain_dir):
-            for file in files:
-                path = os.path.join(root, file)
-                if not self.is_macho_binary(path):
-                    continue
-                self.subprocess_run(codesign_args + [path], check=True)
-
-        self.subprocess_run(codesign_args + [toolchain_dir], check=True)
-
-    async def create_installer(self, toolchain_dir: str, pkg_path: str,
-                               tag_name: str, swift_source_dir: str):
-        toolchain_name = tag_name
-        toolchain_installer_package = pkg_path
-        toolchain_install_location = (
-            f"/Library/Developer/Toolchains/{toolchain_name}.xctoolchain")
-        toolchain_version = subprocess.check_output([
-            "/usr/libexec/PlistBuddy",
-            "-c", "Print Version string",
-            f"{toolchain_dir}/Info.plist"
-        ]).decode("utf-8")
-        toolchain_bundle_identifier = subprocess.check_output([
-            "/usr/libexec/PlistBuddy",
-            "-c", "Print CFBundleIdentifier string",
-            f"{toolchain_dir}/Info.plist"
-        ]).decode("utf-8")
-
-        self.subprocess_run([
-            f"{swift_source_dir}/utils/toolchain-installer", toolchain_dir,
-            toolchain_bundle_identifier,
-            self.secrets.DARWIN_TOOLCHAIN_INSTALLER_CERT,
-            toolchain_installer_package,
-            toolchain_install_location,
-            toolchain_version,
-            f"{swift_source_dir}/utils/darwin-installer-scripts"
-        ], check=True)
-
-        await self.check_async_subprocess(
-            "xcrun", "notarytool", "submit",
-            toolchain_installer_package,
-            "--wait",
-            "--apple-id", self.secrets.DARWIN_TOOLCHAIN_NOTARIZE_EMAIL,
-            "--team-id", self.secrets.DARWIN_TOOLCHAIN_NOTARIZE_TEAM_ID,
-            "--password", self.secrets.DARWIN_TOOLCHAIN_NOTARIZE_PASSWORD)
-
-        self.subprocess_run(["xcrun", "stapler", "staple",
-                             toolchain_installer_package], check=True)
-
-    def subprocess_run(self, args, **kwargs):
-        """
-        Run a non-mutating subprocess and print the command if
-        verbose or dry_run is True.
-        """
-        if self.verbose or self.dry_run:
-            print(" ".join(args))
-        return subprocess.run(args, **kwargs)
-
-    async def check_async_subprocess(self, program, *args):
-        if self.verbose or self.dry_run:
-            print(f"[async] {program} {' '.join(args)}")
-        if self.dry_run:
-            return
-        proc = await asyncio.subprocess.create_subprocess_exec(program, *args)
-        retcode = await proc.wait()
-        if retcode != 0:
-            raise Exception(f"Failed to execute: {' '.join(args)}")
-
-    def is_macho_binary(self, file_path):
-        if not os.path.exists(file_path):
-            return False
-        magic_bytes = None
-        with open(file_path, mode="rb") as f:
-            magic_bytes = f.read(4)
-        macho_bytes = [
-            [0xca, 0xfe, 0xba, 0xbe],  # Mach-O Fat Binary
-            [0xcf, 0xfa, 0xed, 0xfe],  # Mach-O 64-bit executable
-            [0xce, 0xfa, 0xed, 0xfe],  # Mach-O 32-bit executable
-        ]
-        for b in macho_bytes:
-            if magic_bytes == bytes(b):
-                return True
-        return False
-
-
 class GitHub:
     def __init__(self, token: str, repo: str = "swiftwasm/swiftwasm-build"):
         self.token = token
@@ -348,13 +208,14 @@ class Distribution:
     async def run(self, options):
         downloads = []
         for artifact in self.toolchain_artifacts(options):
-            if options.only_swift_sdk:
-                if not artifact["name"].endswith("-artifactbundle"):
-                    print(f"Skipping {artifact['name']} because it's not an"
-                          " artifactbundle for --only-swift-sdk")
-                    continue
+            if not artifact["name"].endswith("-artifactbundle"):
+                print(f"Skipping {artifact['name']} because it's not an"
+                      " artifactbundle for --only-swift-sdk")
+                continue
+
             _, scheme, _ = derive_platform_suffix_and_scheme(
                 artifact["name"], options.scheme)
+
             if options.scheme != scheme:
                 print(f"Skipping {artifact['name']} because it's not scheme {options.scheme}")
                 # Skip unrelated artifact
@@ -388,22 +249,16 @@ class Distribution:
         # Create tag and release on GitHub
         if not tag_name:
             for artifact, artifact_path in downloaded_paths:
-                if artifact["name"].endswith("-installable"):
-                    artifact_path = self.unpack_toolchain(artifact_path)
-                    tag_name = self.guess_tag_name(artifact_path)
-                    break
-                elif artifact["name"].endswith("-artifactbundle"):
-                    platform_suffix, _, target_triple = derive_platform_suffix_and_scheme(
-                        artifact["name"], options.scheme)
-                    bundle_path = await self.unpack_artifactbundle(artifact_path)
-                    dirents = os.listdir(bundle_path)
-                    dirents.remove("info.json")
-                    sdk_artifact_id = dirents[0]
-                    if not platform_suffix:
-                        tag_name = "swift-wasm-" + sdk_artifact_id.removesuffix("-" + target_triple)
-                    else:
-                        tag_name = "swift-wasm-" + sdk_artifact_id.removesuffix("-wasm")
-                    break
+                if not artifact["name"].endswith("-artifactbundle"):
+                    continue
+                platform_suffix, _, target_triple = derive_platform_suffix_and_scheme(
+                    artifact["name"], options.scheme)
+                bundle_path = await self.unpack_artifactbundle(artifact_path)
+                dirents = os.listdir(bundle_path)
+                dirents.remove("info.json")
+                sdk_artifact_id = dirents[0]
+                tag_name = "swift-wasm-" + sdk_artifact_id.removesuffix("-" + target_triple)
+                break
 
         if not tag_name:
             raise Exception("Could not determine tag name")
@@ -425,10 +280,7 @@ class Distribution:
                 artifact, artifact_path = downloaded
                 platform_suffix, scheme, target_triple = derive_platform_suffix_and_scheme(
                     artifact["name"], options.scheme)
-                if not platform_suffix:
-                    artifact_format = SDKArtifactFormatV3(target_triple)
-                else:
-                    artifact_format = SDKArtifactFormatV2(platform_suffix)
+                artifact_format = SDKArtifactFormatV3(target_triple)
                 bundle_path = await self.unpack_artifactbundle(artifact_path)
                 package = await self.package_artifactbundle(
                     bundle_path, tag_name, artifact_format)
@@ -462,73 +314,6 @@ class Distribution:
         self.update_release_notes_with_checksums(
             release, swift_version, swiftwasm_build_version)
 
-        if options.only_swift_sdk:
-            return
-
-        # Package and upload toolchains
-        for artifact, artifact_path in downloaded_paths:
-            import shutil
-            if not artifact["name"].endswith("-installable"):
-                continue
-
-            platform_suffix, _, _ = derive_platform_suffix_and_scheme(
-                artifact["name"], options.scheme)
-            artifact_path = self.unpack_toolchain(artifact_path)
-            package, artifact_path = await self.package_toolchain(
-                artifact_path, tag_name, platform_suffix)
-            if options.dry_run:
-                print(f"Skip uploading actual artifact \"{package}\"")
-                continue
-            self.upload_to_release(package, release)
-            if options.optimize_disk_footprint:
-                print(f"Removing {artifact_path} to optimize disk footprint")
-                shutil.rmtree(artifact_path)
-
-    def guess_tag_name(self, path):
-        return os.path.basename(path)
-
-    async def package_toolchain(self, artifact_path: str, tag_name: str,
-                                platform_suffix: str):
-        print(f"Packaging {artifact_path}")
-        import shutil
-        dest_dir = os.path.dirname(artifact_path)
-        artifacts_path = os.path.dirname(dest_dir)
-
-        if os.path.basename(artifact_path) != tag_name:
-            # e.g.
-            # dest_dir:
-            # <run-id>/swift-wasm-DEVELOPMENT-SNAPSHOT-amazonlinux2_x86_64
-            # artifact_path:
-            # <dest-dir>/swift-wasm-DEVELOPMENT-SNAPSHOT-2023-6-25-a
-            # dest_path:
-            # <dest-dir>/swift-wasm-DEVELOPMENT-SNAPSHOT-2023-6-25-b
-            print(f"Re-package {artifact_path} as {tag_name}")
-            dest_path = os.path.join(dest_dir, tag_name)
-            shutil.rmtree(dest_path, ignore_errors=True)
-            shutil.move(artifact_path, dest_path)
-            artifact_path = dest_path
-
-        if platform_suffix.startswith("macos_"):
-            pkg_path = os.path.join(
-                artifacts_path, f"{tag_name}-{platform_suffix}.pkg")
-            if os.path.exists(pkg_path):
-                return [pkg_path, artifact_path]
-            swift_source_dir = os.path.join(self.checkout_dir, "swift")
-            await DarwinToolchainPackaging(
-                self.secrets, self.dry_run, self.verbose
-            ).package(artifact_path, tag_name, pkg_path, swift_source_dir)
-            return [pkg_path, artifact_path]
-        else:
-            tarball_path = os.path.join(
-                artifacts_path, f"{tag_name}-{platform_suffix}.tar.gz")
-            if os.path.exists(tarball_path):
-                return [tarball_path, artifact_path]
-            tar_args = ["tar", "cfz", tarball_path,
-                        "-C", os.path.dirname(artifact_path),
-                        os.path.basename(artifact_path)]
-            self.subprocess_run(tar_args, check=True)
-            return [tarball_path, artifact_path]
-
     async def package_artifactbundle(
         self, artifact_path: str, tag_name: str,
         artifact_format: SDKArtifactFormat
@@ -624,29 +409,6 @@ class Distribution:
             "unzip", "-q", bundlezip_path, "-d", os.path.dirname(bundle_path))
         return bundle_path
 
-    def unpack_toolchain(self, zip_tarball):
-        tarball_name, tarball_path = self.unzip_artifact(zip_tarball)
-
-        # Remove the .tar.gz extension
-        tarball_basename = os.path.splitext(
-            os.path.splitext(tarball_name)[0])[0]
-        tarball_output_dir = os.path.join(
-            os.path.dirname(tarball_path), tarball_basename)
-        if not os.path.exists(tarball_output_dir):
-            os.makedirs(tarball_output_dir, exist_ok=True)
-            print(f"Extracting {tarball_path}...")
-            with tarfile.open(tarball_path, "r:gz") as tar:
-                tar.extractall(tarball_output_dir)
-        else:
-            print(f"Tarball already extracted at {tarball_output_dir}")
-
-        dirents = os.listdir(tarball_output_dir)
-        if len(dirents) != 1:
-            raise Exception((
-                f"Unexpected number of files in {tarball_output_dir}:"
-                f" {len(dirents)} (expected 1)"))
-        return os.path.join(tarball_output_dir, dirents[0])
-
     async def download_artifact(self, artifact):
         if not os.path.exists(self.artifacts_dir):
             os.makedirs(self.artifacts_dir, exist_ok=True)
@@ -785,8 +547,7 @@ You can install Swift SDKs for WebAssembly using the following commands:
         artifacts = self.github.list_artifacts(self.run_id)
         for artifact in artifacts:
             artifact_name = artifact["name"]
-            should_yield = artifact_name.endswith("-artifactbundle") or \
-                artifact_name.endswith("-installable")
+            should_yield = artifact_name.endswith("-artifactbundle")
             if should_yield:
                 yield artifact
 
@@ -830,18 +591,12 @@ def derive_platform_suffix_and_scheme(
     e.g.
         "main-wasm32-unknown-wasi-artifactbundle", scheme="main"
           -> [None, "main", "wasm32-unknown-wasi"]
-        "macos_x86_64-main-artifactbundle", scheme="main"
-          -> ["macos_x86_64", "main", "wasm32-unknown-wasi"]
-        "ubuntu22.04_x86_64-installable", scheme="main"
-          -> ["ubuntu22.04_x86_64", "main", "wasm32-unknown-wasi"]
     """
 
     # v3 artifact name: <scheme>-<target-triple>-artifactbundle
-    # v2 artifact name: <platform-suffix>-<scheme>-(installable|artifactbundle)
-    # v1 artifact name: <platform-suffix>-installable
     # Note that <scheme> can contain '-'.
     name: str = artifact_name
-    artifact_suffixes = ("-installable", "-artifactbundle")
+    artifact_suffixes = ("-artifactbundle")
     if not name.endswith(artifact_suffixes):
         raise Exception((f"Unexpected artifact name {name}"
                          f", expected to have one of \"{artifact_suffixes}\""
@@ -857,14 +612,6 @@ def derive_platform_suffix_and_scheme(
         target_triple = rest[:-len("-artifactbundle")]
         return [None, target_scheme, target_triple]
 
-    components = name.split("-")
-    if len(components) >= 3:
-        scheme = "-".join(components[1:-1])
-        return [components[0], scheme, "wasm32-unknown-wasi"]
-    else:
-        # Assume legacy representation only used for the main scheme
-        return [components[0], "main", "wasm32-unknown-wasi"]
-
 
 def latest_success_run_id(gh: GitHub, workflow_name: str, branch: str, scheme: str):
     """
@@ -879,7 +626,7 @@ def latest_success_run_id(gh: GitHub, workflow_name: str, branch: str, scheme: s
         artifacts = gh.list_artifacts(run["id"])
         for artifact in artifacts:
             artifact_name = artifact["name"]
-            if not artifact_name.endswith("-installable") and not artifact_name.endswith("-artifactbundle"):
+            if not artifact_name.endswith("-artifactbundle"):
                 continue
             _, artifact_scheme, _ = derive_platform_suffix_and_scheme(artifact_name, scheme)
             if artifact_scheme == scheme: