Merge pull request #53 from espressif/fix/wheels_fixtures

Fixed wheels issues with Python 3.14 + improvements

Author: jakub-kocka

.github/workflows/build-wheels-defined.yml

@@ -226,7 +226,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ${{ fromJson(needs.get-supported-versions.outputs.supported_python) }}
-    container: python:${{ matrix.python-version }}-bullseye
+    container: python:${{ matrix.python-version }}-bookworm
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -265,7 +265,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ${{ fromJson(needs.get-supported-versions.outputs.supported_python) }}
-    container: python:${{ matrix.python-version }}-bullseye
+    container: python:${{ matrix.python-version }}-bookworm
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4

.github/workflows/build-wheels-platforms.yml

@@ -24,13 +24,13 @@ jobs:
         os: # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories
           - windows-latest
           - ubuntu-latest
-          - macos-13      # MacOS x86_64
+          - macos-15-intel     # MacOS x86_64
           - macos-latest  # MacOS arm64 (M1)
           - ubuntu-24.04-arm
           - linux-armv7-self-hosted
         include:
           - os: linux-armv7-self-hosted
-            CONTAINER: python:${{ needs.get-supported-versions.outputs.oldest_supported_python }}-bullseye
+            CONTAINER: python:${{ needs.get-supported-versions.outputs.oldest_supported_python }}-bookworm
         python-version: ['${{ needs.get-supported-versions.outputs.oldest_supported_python }}']
 
     # Use python container on ARM
@@ -66,23 +66,24 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
 
-      - name: Get Python version
+      - name: Install build dependencies
         run: |
-          python --version
-          python -m pip install --upgrade pip setuptools setuptools-scm
-          pip show pip setuptools setuptools-scm
+          python -m pip install --upgrade pip
+          python -m pip install -r build_requirements.txt
 
 
-      - name: Install build dependencies
-        run: python -m pip install -r build_requirements.txt
+      - name: Get Tools versions
+        run: |
+          python --version
+          pip show pip setuptools
 
 
       - name: Install additional OS dependencies - Ubuntu
         if: matrix.os == 'ubuntu-latest'
         run: os_dependencies/ubuntu.sh
 
       - name: Install additional OS dependencies - MacOS
-        if: matrix.os == 'macos-latest' || matrix.os == 'macos-13'
+        if: matrix.os == 'macos-latest' || matrix.os == 'macos-15-intel'
         run: os_dependencies/macos.sh
 
       - name: Install additional OS dependencies - Linux ARM
@@ -97,7 +98,7 @@ jobs:
       - name: Build wheels for IDF
         if: matrix.os != 'windows-latest'
         run: |
-          # Rust directory needs to be included for Linux ARM7
+          # Source Rust environment for self-hosted ARMv7 runner
           if [ "${{ matrix.os }}" = "linux-armv7-self-hosted" ]; then
             . $HOME/.cargo/env
           fi
@@ -106,8 +107,7 @@ jobs:
 
       - name: Build wheels for IDF - Windows
         if: matrix.os == 'windows-latest'
-        run: |
-          python build_wheels.py
+        run: python build_wheels.py
 
       - name: Upload artifacts of downloaded_wheels directory
         uses: actions/upload-artifact@v4

.github/workflows/build-wheels-python-dependent.yml

@@ -22,7 +22,7 @@ jobs:
         os:
           - windows-latest
           - ubuntu-latest
-          - macos-13      # MacOS x86_64
+          - macos-15-intel      # MacOS x86_64
           - macos-latest  # MacOS arm64 (M1)
           - ubuntu-24.04-arm
           - linux-armv7-self-hosted
@@ -34,16 +34,16 @@ jobs:
           - python-version: ${{ inputs.oldest_supported_python }}
             os: ubuntu-latest
           - python-version: ${{ inputs.oldest_supported_python }}
-            os: macos-13
+            os: macos-15-intel
           - python-version: ${{ inputs.oldest_supported_python }}
             os: macos-latest
           - python-version: ${{ inputs.oldest_supported_python }}
             os: ubuntu-24.04-arm
           - python-version: ${{ inputs.oldest_supported_python }}
             os: linux-armv7-self-hosted
 
-    # Use python container on ARM - dynamically constructed
-    container: ${{matrix.os == 'linux-armv7-self-hosted' && format('python:{0}-bullseye', matrix.python-version) || null }}
+    # Use python container on ARM - dynamically constructed with bookworm (Debian 12)
+    container: ${{matrix.os == 'linux-armv7-self-hosted' && format('python:{0}-bookworm', matrix.python-version) || null}}
 
     steps:
       - name: Checkout repository
@@ -71,7 +71,7 @@ jobs:
         run: os_dependencies/ubuntu.sh
 
       - name: Install additional OS dependencies - MacOS
-        if: matrix.os == 'macos-latest' || matrix.os == 'macos-13'
+        if: matrix.os == 'macos-latest' || matrix.os == 'macos-15-intel'
         run: os_dependencies/macos.sh
 
 
@@ -96,7 +96,7 @@ jobs:
       - name: Build Python dependent wheels for ${{ matrix.python-version }}
         if: matrix.os != 'windows-latest'
         run: |
-            # Rust directory needs to be included for Linux ARM7
+            # Source Rust environment for self-hosted ARMv7 runner
             if [ "${{ matrix.os }}" = "linux-armv7-self-hosted" ]; then
               export PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1
               . $HOME/.cargo/env

README.md

@@ -15,6 +15,7 @@ Supported architectures:
     - ARM64
 
 Supported Python versions:
+* 3.14
 * 3.13
 * 3.12
 * 3.11

build_requirements.txt

@@ -6,3 +6,15 @@ PyYAML~=6.0.1
 colorama~=0.4.6
 # ----- build process -----
 boto3~=1.34.4
+
+# setuptools and wheel are needed for Python < 3.10, otherwise resolution with importlib_metadata might be broken
+# https://github.com/espressif/esp-idf/commit/3bad4348d0597597e4079878aa5de1871403e0b2
+setuptools<75.8.1; python_version < "3.10"
+wheel<0.45.0; python_version < "3.10"
+
+# Common build dependencies for packages built with --no-build-isolation
+cffi>=1.15.0                # Required by packages with C extensions (e.g., xcffib, cryptography)
+setuptools-scm>=6.2,<9      # Required by packages using git-based versioning
+setuptools-rust>=1.5.0,<2   # Required by packages with Rust extensions (e.g., cryptography, bleak, rpds-py)
+cython>=0.29.0              # Required by packages using Cython (e.g., coverage, pyyaml speedups)
+poetry-core>=1.0.0          # Required as build backend for packages using Poetry

build_wheels.py

@@ -288,6 +288,8 @@ def build_wheels(requirements: set, local_links: bool = True) -> dict:
                         "https://pypi.org/simple/",
                         "--wheel-dir",
                         f"{dir}",
+                        "--no-cache-dir",
+                        "--no-build-isolation",
                         f"{argument}",
                         f"{arg_param}",
                     ],
@@ -314,6 +316,8 @@ def build_wheels(requirements: set, local_links: bool = True) -> dict:
                 "https://pypi.org/simple/",
                 "--wheel-dir",
                 f"{dir}",
+                "--no-cache-dir",
+                "--no-build-isolation",
             ],
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,

exclude_list.yaml

@@ -16,6 +16,11 @@
   platform: ['linux']
   python: '==3.8'
 
+# dbus-python can not be build with Python > 3.11 on MacOS
+- package_name: 'dbus-python'
+  platform: 'darwin'
+  python: '>3.11'
+
 # PyGObject is difficult to build from source on Windows due to MSYS2/pkg-config issues
 - package_name: 'pygobject'
   platform: ['win32']
@@ -25,11 +30,6 @@
 - package_name: 'pygobject'
   platform: ['linux']
 
-# dbus-python can not be build with Python > 3.11 on MacOS
-- package_name: 'dbus-python'
-  platform: 'darwin'
-  python: '>3.11'
-
 # gevent==1.5.0 can not be build with Python > 3.8
 - package_name: 'gevent'
   version: '==1.5.0'
@@ -43,15 +43,15 @@
 - package_name: 'Pillow'
   version: '==9.5.0'
 
-# some versions of greenlet are not supported by Python 3.13
+# some versions of greenlet are not supported by Python 3.13 and newer
 - package_name: 'greenlet'
   version: '<3.0'
-  python: '==3.13'
+  python: '>=3.13'
 
-# Python 3.13 does not support ruamel.yaml.clib 0.2.8 (0.2.12+ supports Python 3.13)
+# Python 3.13 and newer does not support ruamel.yaml.clib 0.2.8 (0.2.12+ supports Python 3.13 and newer)
 - package_name: 'ruamel.yaml.clib'
   version: '==0.2.8'
-  python: '==3.13'
+  python: '>=3.13'
 
 # tree-sitter versions are not supported by all Python versions
 # https://pypi.org/project/tree-sitter/
@@ -89,3 +89,27 @@
 - package_name: 'pytest-embedded-serial-esp'
   version: '>1.17.0'
   python: ['==3.8', '==3.9']
+
+# windows_curses does not support Python 3.14
+# https://pypi.org/project/windows-curses/
+# https://github.com/zephyrproject-rtos/windows-curses/issues/76
+- package_name: 'windows_curses'
+  platform: ['win32']
+  python: '>=3.14'
+
+# pydantic and some dependencies have issues with Python 3.14 and mainly MacOS
+- package_name: 'pydantic_core'
+  version: '<2.41.2'
+  python: '>=3.14'
+- package_name: 'rpds_py'
+  version: '<0.27.0'
+  python: '>=3.14'
+- package_name: 'pyobjc_core'
+  platform: ['darwin']
+  version: '==10.3.2'
+  python: '>=3.14'
+
+# mcp is not supported by Python older than 3.10
+# https://pypi.org/project/mcp/
+- package_name: 'mcp'
+  python: ['==3.8', '==3.9']

upload_wheels.py

@@ -25,17 +25,21 @@
 
 wheels_subdirs = os.listdir(WHEELS_DIR)
 
+
+def normalize(name):
+    return re.sub(r"[-_.]+", "-", name).lower()
+
+
 for subdir in wheels_subdirs:
     wheel_files = os.listdir(f"{WHEELS_DIR}{os.sep}{subdir}")
 
     for wheel in wheel_files:
-        pattern = re.compile(r"(\w*)-(\d+)")
+        pattern = re.compile(r"^(.+?)-(\d+)")
         match = pattern.search(wheel)
         if match:
             wheel_name = match.group(1)
 
-            wheel_name = wheel_name.lower()
-            wheel_name = wheel_name.replace("_", "-")
+            wheel_name = normalize(wheel_name)
 
             BUCKET.upload_file(f"{WHEELS_DIR}{os.sep}{subdir}{os.sep}{wheel}", f"pypi/{wheel_name}/{wheel}")
-            print(f"Uploaded {wheel}")
+            print(f"Uploaded {wheel_name}/{wheel}")