Merge pull request #56 from espressif/feat/manylinux

Added Docker image for older ARMv7 architecture support because of glibc issues

Author: jakub-kocka

.github/workflows/build-wheels-platforms.yml

@@ -28,6 +28,7 @@ jobs:
           - macOS ARM
           - Linux ARM64
           - Linux ARMv7
+          - Linux ARMv7 Legacy
         include:
           - os: Windows
             runner: windows-latest
@@ -47,11 +48,14 @@ jobs:
           - os: Linux ARMv7
             runner: ubuntu-latest
             arch: linux-armv7
+          - os: Linux ARMv7 Legacy
+            runner: ubuntu-latest
+            arch: linux-armv7legacy
         python-version: ['${{ needs.get-supported-versions.outputs.oldest_supported_python }}']
 
     steps:
       - name: Set up QEMU for ARMv7
-        if: matrix.os == 'Linux ARMv7'
+        if: matrix.os == 'Linux ARMv7' || matrix.os == 'Linux ARMv7 Legacy'
         uses: docker/setup-qemu-action@v3
         with:
           platforms: linux/arm/v7
@@ -79,20 +83,20 @@ jobs:
 
       - name: Setup Python
         # Skip setting python on ARM because of missing compatibility: https://github.com/actions/setup-python/issues/108
-        if: matrix.os != 'Linux ARMv7'
+        if: matrix.os != 'Linux ARMv7' && matrix.os != 'Linux ARMv7 Legacy'
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
 
       - name: Install build dependencies
-        if: matrix.os != 'Linux ARMv7'
+        if: matrix.os != 'Linux ARMv7' && matrix.os != 'Linux ARMv7 Legacy'
         run: |
           python -m pip install --upgrade pip
           python -m pip install -r build_requirements.txt
 
       - name: Get Tools versions
-        if: matrix.os != 'Linux ARMv7'
+        if: matrix.os != 'Linux ARMv7' && matrix.os != 'Linux ARMv7 Legacy'
         run: |
           python --version
           pip show pip setuptools
@@ -123,6 +127,7 @@ jobs:
             -e MIN_IDF_MINOR_VERSION=${{ needs.get-supported-versions.outputs.min_idf_minor_version }} \
             -e GH_TOKEN="${GH_TOKEN}" \
             -e PIP_NO_CACHE_DIR=1 \
+            -e LDFLAGS="-Wl,-z,max-page-size=0x1000" \
             python:${{ matrix.python-version }}-bookworm \
             bash -c "
               set -e
@@ -136,8 +141,34 @@ jobs:
               python build_wheels.py
             "
 
+      - name: Build wheels for IDF - ARMv7 Legacy (in Docker)
+        # Build on Bullseye (glibc 2.31) for compatibility with older systems
+        # Note: Buster has Python 3.7 which is too old for our dependencies
+        if: matrix.os == 'Linux ARMv7 Legacy'
+        run: |
+          docker run --rm --platform linux/arm/v7 \
+            -v $(pwd):/work \
+            -w /work \
+            -e MIN_IDF_MAJOR_VERSION=${{ needs.get-supported-versions.outputs.min_idf_major_version }} \
+            -e MIN_IDF_MINOR_VERSION=${{ needs.get-supported-versions.outputs.min_idf_minor_version }} \
+            -e GH_TOKEN="${GH_TOKEN}" \
+            -e PIP_NO_CACHE_DIR=1 \
+            -e LDFLAGS="-Wl,-z,max-page-size=0x1000" \
+            python:${{ matrix.python-version }}-bullseye \
+            bash -c "
+              set -e
+              python --version
+              # Install pip packages without cache to reduce memory usage
+              python -m pip install --no-cache-dir --upgrade pip
+              python -m pip install --no-cache-dir -r build_requirements.txt
+              bash os_dependencies/linux_arm.sh
+              # Source Rust environment after installation
+              . \$HOME/.cargo/env
+              python build_wheels.py
+            "
+
       - name: Build wheels for IDF - Linux/macOS
-        if: matrix.os != 'Windows' && matrix.os != 'Linux ARMv7'
+        if: matrix.os != 'Windows' && matrix.os != 'Linux ARMv7' && matrix.os != 'Linux ARMv7 Legacy'
         run: |
           # Set ARCHFLAGS for macOS to prevent universal2 wheels
           if [ "${{ matrix.os }}" = "macOS ARM" ]; then

.github/workflows/build-wheels-python-dependent.yml

@@ -28,6 +28,7 @@ jobs:
           - macOS ARM
           - Linux ARM64
           - Linux ARMv7
+          - Linux ARMv7 Legacy
         include:
           - os: Windows
             runner: windows-latest
@@ -47,6 +48,9 @@ jobs:
           - os: Linux ARMv7
             runner: ubuntu-latest
             arch: linux-armv7
+          - os: Linux ARMv7 Legacy
+            runner: ubuntu-latest
+            arch: linux-armv7legacy
         python-version: ${{ fromJson(inputs.supported_python_versions) }}
         exclude:
           # Exclude oldest supported Python since it's already built in the platform builds
@@ -62,10 +66,15 @@ jobs:
             os: Linux ARM64
           - python-version: ${{ inputs.oldest_supported_python }}
             os: Linux ARMv7
+          - python-version: ${{ inputs.oldest_supported_python }}
+            os: Linux ARMv7 Legacy
+          # Python 3.14 doesn't have bullseye images for ARM
+          - python-version: '3.14'
+            os: Linux ARMv7 Legacy
 
     steps:
       - name: Set up QEMU for ARMv7
-        if: matrix.os == 'Linux ARMv7'
+        if: matrix.os == 'Linux ARMv7' || matrix.os == 'Linux ARMv7 Legacy'
         uses: docker/setup-qemu-action@v3
         with:
           platforms: linux/arm/v7
@@ -75,19 +84,19 @@ jobs:
 
       - name: Setup Python
         # Skip setting python on ARMv7 (runs in Docker)
-        if: matrix.os != 'Linux ARMv7'
+        if: matrix.os != 'Linux ARMv7' && matrix.os != 'Linux ARMv7 Legacy'
         uses: actions/setup-python@v5
         with:
             python-version: ${{ matrix.python-version }}
 
       - name: Get Python version
-        if: matrix.os != 'Linux ARMv7'
+        if: matrix.os != 'Linux ARMv7' && matrix.os != 'Linux ARMv7 Legacy'
         run: |
             python --version
             python -m pip install --upgrade pip
 
       - name: Install dependencies
-        if: matrix.os != 'Linux ARMv7'
+        if: matrix.os != 'Linux ARMv7' && matrix.os != 'Linux ARMv7 Legacy'
         run: python -m pip install -r build_requirements.txt
 
       - name: Install additional OS dependencies - Ubuntu
@@ -125,6 +134,7 @@ jobs:
             -e PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 \
             -e GH_TOKEN="${GH_TOKEN}" \
             -e PIP_NO_CACHE_DIR=1 \
+            -e LDFLAGS="-Wl,-z,max-page-size=0x1000" \
             python:${{ matrix.python-version }}-bookworm \
             bash -c "
               set -e
@@ -138,8 +148,32 @@ jobs:
               python build_wheels_from_file.py dependent_requirements_${{ matrix.arch }}
             "
 
+      - name: Build Python dependent wheels - ARMv7 Legacy (in Docker)
+        # Build on Bullseye (glibc 2.31) for compatibility with older systems
+        if: matrix.os == 'Linux ARMv7 Legacy'
+        run: |
+          docker run --rm --platform linux/arm/v7 \
+            -v $(pwd):/work \
+            -w /work \
+            -e PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 \
+            -e GH_TOKEN="${GH_TOKEN}" \
+            -e PIP_NO_CACHE_DIR=1 \
+            -e LDFLAGS="-Wl,-z,max-page-size=0x1000" \
+            python:${{ matrix.python-version }}-bullseye \
+            bash -c "
+              set -e
+              python --version
+              # Install pip packages without cache to reduce memory usage
+              python -m pip install --no-cache-dir --upgrade pip
+              python -m pip install --no-cache-dir -r build_requirements.txt
+              bash os_dependencies/linux_arm.sh
+              # Source Rust environment after installation
+              . \$HOME/.cargo/env
+              python build_wheels_from_file.py dependent_requirements_${{ matrix.arch }}
+            "
+
       - name: Build Python dependent wheels - Linux/macOS
-        if: matrix.os != 'Windows' && matrix.os != 'Linux ARMv7'
+        if: matrix.os != 'Windows' && matrix.os != 'Linux ARMv7' && matrix.os != 'Linux ARMv7 Legacy'
         run: |
             # Set ARCHFLAGS for macOS to prevent universal2 wheels
             if [ "${{ matrix.os }}" = "macOS ARM" ]; then

.github/workflows/wheels-repair.yml

@@ -23,6 +23,7 @@ jobs:
           - Linux x86_64
           - Linux ARM64
           - Linux ARMv7
+          - Linux ARMv7 Legacy
         include:
           - platform: Windows
             runner: windows-latest
@@ -58,6 +59,15 @@ jobs:
             qemu_platform: arm
             docker_platform: linux/arm/v7
             arch: linux-armv7
+          - platform: Linux ARMv7 Legacy
+            runner: ubuntu-latest
+            tool: auditwheel
+            manylinux_platform: idf-python-wheels-armv7l-legacy
+            docker_image: ghcr.io/espressif/github-esp-dockerfiles/idf-python-wheels-armv7l-legacy:v1
+            setup_qemu: true
+            qemu_platform: arm
+            docker_platform: linux/arm/v7
+            arch: linux-armv7legacy
 
     steps:
       - name: Checkout repository
@@ -66,7 +76,9 @@ jobs:
       - name: Download wheel artifacts
         uses: actions/download-artifact@v4
         with:
-          pattern: wheels-download-directory-*
+          # Download only artifacts for this specific architecture to avoid mixing wheels
+          # (especially important for ARMv7 vs ARMv7 Legacy which produce same-named wheels)
+          pattern: wheels-download-directory-${{ matrix.arch }}-*
           path: ./downloaded_wheels
           merge-multiple: true
 
@@ -131,6 +143,7 @@ jobs:
             "
 
       - name: Repair Linux ARMv7 wheels in manylinux container
+        # Using --break-system-packages and --ignore-installed to avoid conflicts with system packages
         if: matrix.platform == 'Linux ARMv7'
         run: |
           docker run --rm \
@@ -141,7 +154,23 @@ jobs:
             bash -c "
               bash os_dependencies/linux_arm.sh
               python3 -m pip install --break-system-packages --upgrade pip
-              python3 -m pip install --break-system-packages -r build_requirements.txt
+              python3 -m pip install --break-system-packages --ignore-installed -r build_requirements.txt
+              python3 repair_wheels.py
+            "
+
+      - name: Repair Linux ARMv7 Legacy wheels in manylinux container
+        # Using --break-system-packages and --ignore-installed to avoid conflicts with system packages
+        if: matrix.platform == 'Linux ARMv7 Legacy'
+        run: |
+          docker run --rm \
+            --platform ${{ matrix.docker_platform }} \
+            -v $(pwd):/work \
+            -w /work \
+            ${{ matrix.docker_image }} \
+            bash -c "
+              bash os_dependencies/linux_arm.sh
+              python3 -m pip install --break-system-packages --upgrade pip
+              python3 -m pip install --break-system-packages --ignore-installed -r build_requirements.txt
               python3 repair_wheels.py
             "
 

.pre-commit-config.yaml

@@ -10,6 +10,7 @@ ci:
   autofix_prs: true
   autoupdate_commit_msg: 'ci: Bump pre-commit hooks'
   autoupdate_schedule: quarterly
+  skip: ['mypy']
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
@@ -23,20 +24,20 @@ repos:
       - id: check-yaml
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.13.3
+    rev: v0.14.10
     hooks:
       - id: ruff-check
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.18.2
+    rev: v1.18.2  # frozen: minimum Python version compatibility (newer versions require Python 3.9+)
     hooks:
     -   id: mypy
         additional_dependencies: ['types-all-latest']
 
   - repo: https://github.com/espressif/conventional-precommit-linter
-    rev: v1.10.0
+    rev: v1.11.0
     hooks:
       - id: conventional-precommit-linter
         stages: [commit-msg]

README.md

@@ -141,4 +141,9 @@ The main file is `build-wheels-platforms.yml` which is scheduled to run periodic
 > Python dependent wheels are wheels which depend on the [CPython’s Application Binary Interface (ABI)](https://docs.python.org/3/c-api/stable.html). These are checked based on the [wheel filename format](https://peps.python.org/pep-0491/#file-format) where the `abi tag` is checked for `cp`. Such wheels need to be build also for all supported Python versions, not only for the minimum Python version supported by [ESP-IDF].
 
 
+## Custom Docker images
+Docker files are in its own repository where there are build and published from. https://github.com/espressif/github-esp-dockerfiles
+- ARMv7 runner - [manylinux_armv7](https://github.com/espressif/github-esp-dockerfiles/pkgs/container/github-esp-dockerfiles%2Fidf-python-wheels-armv7l)
+- ARMv7 runner (older OSes) - [manylinux_armv7_legacy](https://github.com/espressif/github-esp-dockerfiles/pkgs/container/github-esp-dockerfiles%2Fidf-python-wheels-armv7l-legacy)
+
 [ESP-IDF]: https://github.com/espressif/esp-idf

_helper_functions.py

@@ -3,10 +3,55 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 #
+import platform
+import re
+
 from colorama import Fore
 from colorama import Style
 from packaging.requirements import Requirement
 
+# Packages that should be built from source on Linux to ensure correct library linking
+# These packages often have pre-built wheels on PyPI that link against different library versions
+# NOTE: This only applies to Linux (especially ARM) - Windows and macOS pre-built wheels work fine
+# NOTE: Do NOT add packages with Rust components (cryptography, pynacl, bcrypt) here
+# as they have complex build requirements and may not support all Python versions
+FORCE_SOURCE_BUILD_PACKAGES_LINUX = [
+    "cffi",
+    "pillow",
+    "pyyaml",
+    "brotli",
+    "greenlet",
+    "bitarray",
+]
+
+
+def get_no_binary_args(requirement_name: str) -> list:
+    """Get --no-binary arguments if this package should be built from source.
+
+    This only applies on Linux platforms where pre-built wheels may link against
+    different library versions. On Windows and macOS, pre-built wheels work correctly.
+
+    Args:
+        requirement_name: Package name or requirement string (e.g., "cffi" or "cffi>=1.0")
+
+    Returns:
+        List with --no-binary arguments if package should be built from source, empty list otherwise
+    """
+    # Only force source builds on Linux (where we have library version issues)
+    if platform.system() != "Linux":
+        return []
+
+    # Extract package name from requirement string (e.g., "cffi>=1.0" -> "cffi")
+    match = re.match(r"^([a-zA-Z0-9_-]+)", str(requirement_name).strip())
+    if not match:
+        return []
+    pkg_name = match.group(1).lower().replace("-", "_")
+
+    for pkg in FORCE_SOURCE_BUILD_PACKAGES_LINUX:
+        if pkg.lower().replace("-", "_") == pkg_name:
+            return ["--no-binary", match.group(1)]
+    return []
+
 
 def print_color(text: str, color: str = Fore.BLUE):
     """Print colored text specified by color argument based on colorama