feat: Multiarch wheel builds

Author: tmathern

.github/workflows/build-wheel.yml

@@ -121,7 +121,7 @@ jobs:
           # Verify wheel structure
           twine check dist/*
 
-      - name: Build macOS wheel (Universal)
+      - name: Build macOS wheel
         if: runner.os == 'macOS'
         run: |
           # Create necessary directories
@@ -138,13 +138,28 @@ jobs:
           # Download native artifacts
           python scripts/download_artifacts.py $C2PA_VERSION
 
-          # Build wheel
-          python setup.py bdist_wheel --plat-name macosx_10_9_universal2
+          # Determine platform name based on target architecture
+          if [ "${{ inputs.architecture }}" = "universal2" ]; then
+            PLATFORM_NAME="macosx_10_9_universal2"
+          elif [ "${{ inputs.architecture }}" = "arm64" ]; then
+            PLATFORM_NAME="macosx_11_0_arm64"
+          elif [ "${{ inputs.architecture }}" = "x86_64" ]; then
+            PLATFORM_NAME="macosx_10_9_x86_64"
+          else
+            echo "Unknown architecture: ${{ inputs.architecture }}"
+            exit 1
+          fi
+
+          echo "Building wheel for architecture: ${{ inputs.architecture }}"
+          echo "Using platform name: $PLATFORM_NAME"
+
+          # Build wheel with appropriate platform name
+          python setup.py bdist_wheel --plat-name $PLATFORM_NAME
 
           # Rename wheel to ensure unique filename
           cd dist
           for wheel in *.whl; do
-            mv "$wheel" "${wheel/macosx_10_9_universal2/macosx_10_9_universal2}"
+            mv "$wheel" "${wheel/$PLATFORM_NAME/$PLATFORM_NAME}"
           done
           cd ..
 

scripts/download_artifacts.py

@@ -40,11 +40,17 @@ def detect_arch():
     else:
         raise ValueError(f"Unsupported CPU architecture: {machine}")
 
-def get_platform_identifier():
-    """Get the full platform identifier (arch-os) for the current system,
-    matching the identifiers used by the Github publisher.
+def get_platform_identifier(target_arch=None):
+    """Get the full platform identifier (arch-os) for the current system or target.
+
+    Args:
+        target_arch: Optional target architecture. If provided, overrides auto-detection.
+                    For macOS: 'universal2', 'arm64', or 'x86_64'
+
     Returns one of:
-    - universal-apple-darwin (for Mac)
+    - universal-apple-darwin (for macOS universal)
+    - aarch64-apple-darwin (for macOS ARM64)
+    - x86_64-apple-darwin (for macOS x86_64)
     - x86_64-pc-windows-msvc (for Windows 64-bit)
     - x86_64-unknown-linux-gnu (for Linux x86_64)
     - aarch64-unknown-linux-gnu (for Linux ARM64)
@@ -53,11 +59,27 @@ def get_platform_identifier():
     machine = platform.machine().lower()
 
     if system == "darwin":
-        return "universal-apple-darwin"
+        if target_arch == "arm64":
+            return "aarch64-apple-darwin"
+        elif target_arch == "x86_64":
+            return "x86_64-apple-darwin"
+        elif target_arch == "universal2":
+            return "universal-apple-darwin"
+        else:
+            # Auto-detect: prefer specific architecture over universal
+            if machine == "arm64":
+                return "aarch64-apple-darwin"
+            elif machine == "x86_64":
+                return "x86_64-apple-darwin"
+            else:
+                return "universal-apple-darwin"
     elif system == "windows":
-        return "x86_64-pc-windows-msvc"
+        if target_arch == "arm64":
+            return "aarch64-pc-windows-msvc"
+        else:
+            return "x86_64-pc-windows-msvc"
     elif system == "linux":
-        if machine in ["arm64", "aarch64"]:
+        if target_arch == "aarch64" or machine in ["arm64", "aarch64"]:
             return "aarch64-unknown-linux-gnu"
         else:
             return "x86_64-unknown-linux-gnu"
@@ -87,19 +109,20 @@ def download_and_extract_libs(url, platform_name):
     response = requests.get(url, headers=headers)
     response.raise_for_status()
 
+    print(f"Downloaded zip file, extracting lib files...")
     with zipfile.ZipFile(io.BytesIO(response.content)) as zip_ref:
         # Extract only files inside the libs/ directory
+        extracted_count = 0
         for member in zip_ref.namelist():
-            print(f"  Processing zip member: {member}")
             if member.startswith("lib/") and not member.endswith("/"):
-                print(f"    Processing lib file from downloadedzip: {member}")
                 target_path = platform_dir / os.path.relpath(member, "lib")
-                print(f"      Moving file to target path: {target_path}")
                 target_path.parent.mkdir(parents=True, exist_ok=True)
                 with zip_ref.open(member) as source, open(target_path, "wb") as target:
                     target.write(source.read())
+                extracted_count += 1
+                print(f"  Extracted: {member} -> {target_path}")
 
-    print(f"Done downloading and extracting libraries for {platform_name}")
+    print(f"Done downloading and extracting {extracted_count} library files for {platform_name}")
 
 def copy_artifacts_to_root():
     """Copy the artifacts folder from scripts/artifacts to the root of the repository."""
@@ -108,56 +131,77 @@ def copy_artifacts_to_root():
         return
 
     print("Copying artifacts from scripts/artifacts to root...")
+    print("Contents of scripts/artifacts before copying:")
+    for item in sorted(SCRIPTS_ARTIFACTS_DIR.iterdir()):
+        print(f"  {item.name}")
+    
     if ROOT_ARTIFACTS_DIR.exists():
         shutil.rmtree(ROOT_ARTIFACTS_DIR)
     print(f"Copying from {SCRIPTS_ARTIFACTS_DIR} to {ROOT_ARTIFACTS_DIR}")
     shutil.copytree(SCRIPTS_ARTIFACTS_DIR, ROOT_ARTIFACTS_DIR)
     print("Done copying artifacts")
-    print("\nFolder content of artifacts directory:")
+    print("\nFolder content of root artifacts directory:")
     for item in sorted(ROOT_ARTIFACTS_DIR.iterdir()):
         print(f"  {item.name}")
 
 def main():
     if len(sys.argv) < 2:
-        print("Usage: python download_artifacts.py <release_tag>")
+        print("Usage: python download_artifacts.py <release_tag> [target_architecture]")
         print("Example: python download_artifacts.py c2pa-v0.49.5")
+        print("Example: python download_artifacts.py c2pa-v0.49.5 arm64")
         sys.exit(1)
 
     release_tag = sys.argv[1]
+    target_arch = sys.argv[2] if len(sys.argv) > 2 else None
+
     try:
+        # Clean up any existing artifacts before starting
+        print("Cleaning up existing artifacts...")
+        if SCRIPTS_ARTIFACTS_DIR.exists():
+            shutil.rmtree(SCRIPTS_ARTIFACTS_DIR)
         SCRIPTS_ARTIFACTS_DIR.mkdir(exist_ok=True)
         print(f"Fetching release information for tag {release_tag}...")
         release = get_release_by_tag(release_tag)
         print(f"Found release: {release['tag_name']} \n")
 
-        # Get the platform identifier for the current system
+        # Get the platform identifier for the target architecture
         env_platform = os.environ.get("C2PA_LIBS_PLATFORM")
         if env_platform:
             print(f"Using platform from environment variable C2PA_LIBS_PLATFORM: {env_platform}")
-        platform_id = env_platform or get_platform_identifier()
+            platform_id = env_platform
+        else:
+            platform_id = get_platform_identifier(target_arch)
+            print(f"Using target architecture: {target_arch or 'auto-detected'}")
+            print(f"Detected machine architecture: {platform.machine()}")
+            print(f"Detected system: {platform.system()}")
+
         print("Looking up releases for platform id: ", platform_id)
-        print("Environment variable set for lookup: ", env_platform)
-        platform_source = "environment variable" if env_platform else "auto-detection"
-        print(f"Target platform: {platform_id} (set through{platform_source})")
+        platform_source = "environment variable" if env_platform else "target architecture" if target_arch else "auto-detection"
+        print(f"Target platform: {platform_id} (set through {platform_source})")
 
         # Construct the expected asset name
         expected_asset_name = f"{release_tag}-{platform_id}.zip"
         print(f"Looking for asset: {expected_asset_name}")
 
         # Find the matching asset in the release
         matching_asset = None
+        print(f"Looking for asset: {expected_asset_name}")
+        print("Available assets in release:")
         for asset in release['assets']:
+            print(f"  - {asset['name']}")
             if asset['name'] == expected_asset_name:
                 matching_asset = asset
-                break
+                print(f"Matching asset: {matching_asset['name']}")
 
         if matching_asset:
-            print(f"Found matching asset: {matching_asset['name']}")
+            print(f"\nDownloading asset: {matching_asset['name']}")
             download_and_extract_libs(matching_asset['browser_download_url'], platform_id)
             print("\nArtifacts have been downloaded and extracted successfully!")
             copy_artifacts_to_root()
         else:
-            print(f"\nNo matching asset found: {expected_asset_name}")
+            print(f"\nNo matching asset found for platform: {platform_id}")
+            print(f"Expected asset name: {expected_asset_name}")
+            print("Please check if the asset exists in the release or if the platform identifier is correct.")
 
     except requests.exceptions.RequestException as e:
         print(f"Error: {e}")

setup.py

@@ -25,6 +25,8 @@ def get_version():
 # Based on what c2pa-rs repo publishes
 PLATFORM_FOLDERS = {
     'universal-apple-darwin': 'dylib',
+    'aarch64-apple-darwin': 'dylib',
+    'x86_64-apple-darwin': 'dylib',
     'x86_64-pc-windows-msvc': 'dll',
     'x86_64-unknown-linux-gnu': 'so',
     'aarch64-unknown-linux-gnu': 'so',
@@ -35,30 +37,42 @@ def get_version():
 PACKAGE_LIBS_DIR = Path('src/c2pa/libs')  # Where libraries will be copied for the wheel
 
 
-def get_platform_identifier() -> str:
-    """Get a platform identifier (arch-os) for the current system,
-    matching downloaded identifiers used by the Github publisher.
+def get_platform_identifier(target_arch=None) -> str:
+    """Get a platform identifier (arch-os) for the current system or target architecture.
 
     Args:
-        Only used on macOS systems.:
-            cpu_arch: Optional CPU architecture for macOS. If not provided, returns universal build.
+        target_arch: Optional target architecture. If provided, overrides auto-detection.
+                    For macOS: 'universal2', 'arm64', or 'x86_64'
+                    For Linux: 'aarch64' or 'x86_64'
+                    For Windows: 'x64' or 'arm64'
 
     Returns one of:
-    - universal-apple-darwin (for macOS)
+    - universal-apple-darwin (for macOS universal)
+    - aarch64-apple-darwin (for macOS ARM64)
+    - x86_64-apple-darwin (for macOS x86_64)
     - x86_64-pc-windows-msvc (for Windows 64-bit)
     - x86_64-unknown-linux-gnu (for Linux 64-bit)
     - aarch64-unknown-linux-gnu (for Linux ARM64)
     """
     system = platform.system().lower()
 
     if system == "darwin":
-        return "universal-apple-darwin"
+        if target_arch == "arm64":
+            return "aarch64-apple-darwin"
+        elif target_arch == "x86_64":
+            return "x86_64-apple-darwin"
+        else:
+            return "universal-apple-darwin"
     elif system == "windows":
-        return "x86_64-pc-windows-msvc"
+        if target_arch == "arm64":
+            return "aarch64-pc-windows-msvc"
+        else:
+            return "x86_64-pc-windows-msvc"
     elif system == "linux":
-        if platform.machine() == "aarch64":
+        if target_arch == "aarch64" or platform.machine() == "aarch64":
             return "aarch64-unknown-linux-gnu"
-        return "x86_64-unknown-linux-gnu"
+        else:
+            return "x86_64-unknown-linux-gnu"
     else:
         raise ValueError(f"Unsupported operating system: {system}")
 
@@ -144,60 +158,59 @@ def find_available_platforms():
 
 # For wheel building (both bdist_wheel and build)
 if 'bdist_wheel' in sys.argv or 'build' in sys.argv:
-    available_platforms = find_available_platforms()
-    if not available_platforms:
-        print("No platform-specific libraries found. Building wheel without platform-specific libraries.")
+    # Check if we're building for a specific architecture
+    target_arch = None
+    for i, arg in enumerate(sys.argv):
+        if arg == '--plat-name':
+            if i + 1 < len(sys.argv):
+                plat_name = sys.argv[i + 1]
+                if 'arm64' in plat_name:
+                    target_arch = 'arm64'
+                elif 'x86_64' in plat_name:
+                    target_arch = 'x86_64'
+                elif 'universal2' in plat_name:
+                    target_arch = 'universal2'
+                break
+
+    # Get the platform identifier for the target architecture
+    target_platform = get_platform_identifier(target_arch)
+    print(f"Building wheel for target platform: {target_platform}")
+
+    # Check if we have libraries for this platform
+    platform_dir = ARTIFACTS_DIR / target_platform
+    if not platform_dir.exists() or not any(platform_dir.iterdir()):
+        print(f"Warning: No libraries found for platform {target_platform}")
+        print("Available platforms:")
+        for platform_name in find_available_platforms():
+            print(f"  - {platform_name}")
+
+    # Copy libraries for the target platform
+    try:
+        copy_platform_libraries(target_platform, clean_first=True)
+
+        # Build the wheel
         setup(
             name=PACKAGE_NAME,
             version=VERSION,
             package_dir={"": "src"},
             packages=find_namespace_packages(where="src"),
             include_package_data=True,
             package_data={
-                "c2pa": ["libs/*"],  # Include all files in libs directory
+                "c2pa": ["libs/*"],
             },
             classifiers=[
                 "Programming Language :: Python :: 3",
-                get_platform_classifier(get_current_platform()),
+                get_platform_classifier(target_platform),
             ],
             python_requires=">=3.10",
             long_description=open("README.md").read(),
             long_description_content_type="text/markdown",
             license="MIT OR Apache-2.0",
         )
-        sys.exit(0)
-
-    print(f"Found libraries for platforms: {', '.join(available_platforms)}")
-
-    for platform_name in available_platforms:
-        print(f"\nBuilding wheel for {platform_name}...")
-        try:
-            # Copy libraries for this platform (cleaning first)
-            copy_platform_libraries(platform_name, clean_first=True)
-
-            # Build the wheel
-            setup(
-                name=PACKAGE_NAME,
-                version=VERSION,
-                package_dir={"": "src"},
-                packages=find_namespace_packages(where="src"),
-                include_package_data=True,
-                package_data={
-                    "c2pa": ["libs/*"],  # Include all files in libs directory
-                },
-                classifiers=[
-                    "Programming Language :: Python :: 3",
-                    get_platform_classifier(platform_name),
-                ],
-                python_requires=">=3.10",
-                long_description=open("README.md").read(),
-                long_description_content_type="text/markdown",
-                license="MIT OR Apache-2.0",
-            )
-        finally:
-            # Clean up by removing the package libs directory
-            if PACKAGE_LIBS_DIR.exists():
-                shutil.rmtree(PACKAGE_LIBS_DIR)
+    finally:
+        # Clean up
+        if PACKAGE_LIBS_DIR.exists():
+            shutil.rmtree(PACKAGE_LIBS_DIR)
     sys.exit(0)
 
 # For sdist and development installation