ci: For python wheel generation, use ccache

.github/workflows/wheel.yml

@@ -39,6 +39,12 @@ on:
   workflow_dispatch:
     # This allows manual triggering of the workflow from the web
 
+# Allow subsequent pushes to the same PR or REF to cancel any previous jobs.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+
 jobs:
   # Linux jobs run in Docker containers (manylinux), so the latest OS version
   # is OK. macOS and Windows jobs need to be locked to specific virtual
@@ -141,20 +147,50 @@ jobs:
         with:
           python-version: '3.9'
 
+      - name: ccache-restore
+        id: ccache-restore
+        uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ~/.ccache
+          key: wheel-${{runner.os}}-${{matrix.manylinux}}-${{matrix.python}}
+          restore-keys: wheel-${{runner.os}}-${{matrix.manylinux}}-${{matrix.python}}
+
       - name: Build wheels
         # Note: the version of cibuildwheel should be kept in sync with src/python/stubs/CMakeLists.txt
         uses: pypa/cibuildwheel@d4a2945fcc8d13f20a1b99d461b8e844d5fc6e23 # v2.21.1
         env:
           # pass GITHUB_ACTIONS through to the build container so that custom
           # processes can tell they are running in CI.
           CIBW_ENVIRONMENT_PASS_LINUX: GITHUB_ACTIONS
+          CIBW_BEFORE_ALL: "source src/build-scripts/build_ccache.bash && pwd && ext/dist/bin/ccache --max-size=200M && ext/dist/bin/ccache -sv && export CMAKE_C_COMPILER_LAUNCHER=/project/ext/dist/bin/ccache CMAKE_CXX_COMPILER_LAUNCHER=/project/ext/dist/bin/ccache"
+          CIBW_BEFORE_TEST: "ext/dist/bin/ccache -s"
           CIBW_BUILD: ${{ matrix.python }}
           CIBW_ARCHS: ${{ matrix.arch }}
           CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux }}
+          CIBW_ENVIRONMENT: >
+              CCACHE_DIR=/host//home/runner/.ccache
+              CCACHE_COMPRESSION=yes
+              CCACHE_PREBUILT=0
+              CMAKE_BUILD_PARALLEL_LEVEL=4
+              CTEST_PARALLEL_LEVEL=4
+              SKBUILD_CMAKE_ARGS="-DLINKSTATIC=1"
+              SKBUILD_CMAKE_BUILD_TYPE="MinSizeRel"
+              SKBUILD_BUILD_DIR=/project/build
+              CXXFLAGS="-Wno-error=stringop-overflow -Wno-pragmas"
+              WebP_BUILD_VERSION="1.5.0"
+          # FIXME: Getting build problems when using WebP 1.6.0, so hold it back
+          # CMAKE_GENERATOR = "Ninja"
+
+      - name: ccache-save
+        id: ccache-save
+        uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ~/.ccache
+          key: wheel-${{runner.os}}-${{matrix.manylinux}}-${{matrix.python}}
 
       - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
-          name: cibw-wheels-${{ matrix.python }}-${{ matrix.manylinux }}
+          name: cibw-wheels-${{matrix.manylinux}}-${{ matrix.python }}-${{ matrix.manylinux }}
           path: |
             ./wheelhouse/*.whl
 
@@ -173,12 +209,12 @@ jobs:
   # ---------------------------------------------------------------------------
 
   linux-arm:
-      name: Build wheels on Linux ARM
-      runs-on: ubuntu-24.04-arm
-      if: |
-        github.event_name != 'schedule' ||
-        github.repository == 'AcademySoftwareFoundation/OpenImageIO'
-      strategy:
+    name: Build wheels on Linux ARM
+    runs-on: ubuntu-24.04-arm
+    if: |
+      github.event_name != 'schedule' ||
+      github.repository == 'AcademySoftwareFoundation/OpenImageIO'
+    strategy:
         matrix:
           include:
             # -------------------------------------------------------------------
@@ -205,37 +241,66 @@ jobs:
               python: cp313-manylinux_aarch64
               arch: aarch64
 
-      steps:
-        - name: Checkout repo
-          uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
-        - name: Install Python
-          uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
-          with:
-            python-version: '3.9'
-
-        - name: Build wheels
-          uses: pypa/cibuildwheel@d4a2945fcc8d13f20a1b99d461b8e844d5fc6e23 # v2.21.1
-          env:
-            CIBW_BUILD: ${{ matrix.python }}
-            CIBW_ARCHS: ${{ matrix.arch }}
-            CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.manylinux }}
-
-        - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
-          with:
-            name: cibw-wheels-${{ matrix.python }}-${{ matrix.manylinux }}
-            path: |
-              ./wheelhouse/*.whl
+      - name: Install Python
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
+        with:
+          python-version: '3.9'
 
-        - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
-          with:
-            name: stubs-${{ matrix.python }}-${{ matrix.manylinux }}
-            path: |
-              ./wheelhouse/OpenImageIO/__init__.pyi
-          # if stub validation fails we want to upload the stubs for users to review.
-          # keep the python build in sync with the version specified in tool.cibuildwheel.overrides 
-          # section of pyproject.toml
-          if: always() && contains(matrix.python, 'cp311-manylinux')
+      - name: ccache-restore
+        id: ccache-restore
+        uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ~/.ccache
+          key: wheel-${{runner.os}}-${{matrix.python}}
+          restore-keys: wheel-${{runner.os}}-${{matrix.python}}
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@d4a2945fcc8d13f20a1b99d461b8e844d5fc6e23 # v2.21.1
+        env:
+          CIBW_ENVIRONMENT_PASS_LINUX: GITHUB_ACTIONS
+          CIBW_BEFORE_ALL: "source src/build-scripts/build_ccache.bash && pwd && /project/ext/dist/bin/ccache --max-size=200M && /project/ext/dist/bin/ccache -sv && export CMAKE_C_COMPILER_LAUNCHER=/project/ext/dist/bin/ccache CMAKE_CXX_COMPILER_LAUNCHER=/project/ext/dist/bin/ccache"
+          CIBW_BEFORE_TEST: "ext/dist/bin/ccache -s"
+          CIBW_BUILD: ${{ matrix.python }}
+          CIBW_ARCHS: ${{ matrix.arch }}
+          CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.manylinux }}
+          CIBW_ENVIRONMENT: >
+              CCACHE_DIR=/host//home/runner/.ccache
+              CCACHE_COMPRESSION=yes
+              CCACHE_PREBUILT=0
+              CMAKE_BUILD_PARALLEL_LEVEL=6
+              CTEST_PARALLEL_LEVEL=6
+              SKBUILD_CMAKE_ARGS="-DLINKSTATIC=1"
+              SKBUILD_CMAKE_BUILD_TYPE="MinSizeRel"
+              SKBUILD_BUILD_DIR=/project/build
+              CXXFLAGS="-Wno-error=stringop-overflow -Wno-pragmas"
+              WebP_BUILD_VERSION="1.5.0"
+
+      - name: ccache-save
+        id: ccache-save
+        uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ~/.ccache
+          key: wheel-${{runner.os}}-${{matrix.python}}
+
+      - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
+        with:
+          name: cibw-wheels-${{ matrix.python }}-${{ matrix.manylinux }}
+          path: |
+            ./wheelhouse/*.whl
+
+      - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
+        with:
+          name: stubs-${{ matrix.python }}-${{ matrix.manylinux }}
+          path: |
+            ./wheelhouse/OpenImageIO/__init__.pyi
+        # if stub validation fails we want to upload the stubs for users to review.
+        # keep the python build in sync with the version specified in tool.cibuildwheel.overrides 
+        # section of pyproject.toml
+        if: always() && contains(matrix.python, 'cp311-manylinux')
 
   # ---------------------------------------------------------------------------
   # macOS Wheels
@@ -278,6 +343,18 @@ jobs:
         with:
           python-version: '3.9'
 
+      - name: ccache-restore
+        id: ccache-restore
+        uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ~/.ccache
+          key: wheel-${{runner.os}}-${{matrix.python}}
+          restore-keys: wheel-${{runner.os}}-${{matrix.python}}
+
+      - name: Install build tools
+        run: |
+            brew install ninja ccache || true
+
       - name: Remove brew OpenEXR/Imath
         run: |
             brew uninstall --ignore-dependencies openexr imath || true
@@ -292,6 +369,18 @@ jobs:
           # TODO: Re-enable HEIF when we provide a build recipe that does 
           # not include GPL-licensed dynamic libraries.
           USE_Libheif: 'OFF'
+          CMAKE_BUILD_PARALLEL_LEVEL: 6
+          CTEST_PARALLEL_LEVEL: 6
+          SKBUILD_BUILD_DIR: "/Users/runner/work/OpenImageIO/OpenImageIO/build"
+          CCACHE_DIR: /Users/runner/.ccache
+          CCACHE_COMPRESSION: yes
+
+      - name: ccache-save
+        id: ccache-save
+        uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ~/.ccache
+          key: wheel-${{runner.os}}-${{matrix.python}}
 
       - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
@@ -340,13 +429,36 @@ jobs:
         with:
           python-version: '3.9'
 
+      - name: ccache-restore
+        id: ccache-restore
+        uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ~/.ccache
+          key: wheel-${{runner.os}}-${{matrix.python}}
+          restore-keys: wheel-${{runner.os}}-${{matrix.python}}
 
+      - name: Install build tools
+        run: |
+            brew install ninja ccache || true
+
       - name: Build wheels
         uses: pypa/cibuildwheel@d4a2945fcc8d13f20a1b99d461b8e844d5fc6e23 # v2.21.1
         env:
           CIBW_BUILD: ${{ matrix.python }}
           CIBW_ARCHS: ${{ matrix.arch }}
           CMAKE_GENERATOR: "Unix Makefiles"
+          CMAKE_BUILD_PARALLEL_LEVEL: 6
+          CTEST_PARALLEL_LEVEL: 6
+          SKBUILD_BUILD_DIR: "/Users/runner/work/OpenImageIO/OpenImageIO/build"
+          CCACHE_DIR: /Users/runner/.ccache
+          CCACHE_COMPRESSION: yes
+
+      - name: ccache-save
+        id: ccache-save
+        uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ~/.ccache
+          key: wheel-${{runner.os}}-${{matrix.python}}
 
       - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
@@ -400,6 +512,11 @@ jobs:
           CIBW_BUILD: ${{ matrix.python }}
           CIBW_ARCHS: ${{ matrix.arch }}
           CMAKE_POLICY_VERSION_MINIMUM: 3.5
+          CMAKE_BUILD_PARALLEL_LEVEL: 4
+          CTEST_PARALLEL_LEVEL: 4
+          SKBUILD_BUILD_DIR: "$HOME/OpenImageIO/OpenImageIO/build"
+          CCACHE_DIR: ~/.ccache
+          CCACHE_COMPRESSION: yes
 
       - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:

pyproject.toml

@@ -1,3 +1,6 @@
+# See docs at https://cibuildwheel.pypa.io/en/stable/options
+# for description of all the options in this file.
+
 [project]
 name = "OpenImageIO"
 # The build backend ascertains the version from the CMakeLists.txt file.

src/build-scripts/build_ccache.bash

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+
+# Utility script to download or build ccacheccache
+#
+# Copyright Contributors to the OpenImageIO project.
+# SPDX-License-Identifier: Apache-2.0
+# https://github.com/AcademySoftwareFoundation/OpenImageIO
+
+# Exit the whole script if any command fails.
+set -ex
+
+echo "Building ccache"
+uname
+ARCH=`uname -m`
+echo "HOME=$HOME"
+echo "PWD=$PWD"
+echo "ARCH=$ARCH"
+
+CCACHE_PREBULT=${CCACHE_PREBULT:=1}
+
+# Repo and branch/tag/commit of ccache to download if we don't have it yet
+CCACHE_REPO=${CCACHE_REPO:=https://github.com/ccache/ccache}
+CCACHE_VERSION=${CCACHE_VERSION:=4.12}
+CCACHE_TAG=${CCACHE_TAG:=v${CCACHE_VERSION}}
+LOCAL_DEPS_DIR=${LOCAL_DEPS_DIR:=${PWD}/ext}
+# Where to put ccache repo source (default to the ext area)
+CCACHE_SRC_DIR=${CCACHE_SRC_DIR:=${LOCAL_DEPS_DIR}/ccache}
+# Temp build area (default to a build/ subdir under source)
+CCACHE_BUILD_DIR=${CCACHE_BUILD_DIR:=${CCACHE_SRC_DIR}/build}
+# Install area for ccache (default to ext/dist)
+CCACHE_INSTALL_DIR=${CCACHE_INSTALL_DIR:=${PWD}/ext/dist}
+CCACHE_CONFIG_OPTS=${CCACHE_CONFIG_OPTS:=}
+
+
+# if [[ `uname` == "Linux" && `uname -m` == "x86_64" ]] ; then
+if [[ `uname` == "Linux" ]] ; then
+    mkdir -p ${CCACHE_SRC_DIR}
+    pushd ${CCACHE_SRC_DIR}
+
+    if [[ "$CCACHE_PREBUILT" != "0" ]] ; then
+        #
+        # Try to download -- had trouble with this on runners
+        #
+        CCACHE_DESCRIPTOR="ccache-${CCACHE_VERSION}-linux-x86_64"
+        curl --location "${CCACHE_REPO}/releases/download/${CCACHE_TAG}/${CCACHE_DESCRIPTOR}.tar.xz" -o ccache.tar.xz
+        tar xJvf ccache.tar.xz
+        mkdir -p ${CCACHE_INSTALL_DIR}/bin
+        cp ${CCACHE_SRC_DIR}/${CCACHE_DESCRIPTOR}/ccache ${CCACHE_INSTALL_DIR}/bin
+    else
+        # Clone ccache project from GitHub and build
+        if [[ ! -e ${CCACHE_SRC_DIR}/.git ]] ; then
+            echo "git clone ${CCACHE_REPO} ${CCACHE_SRC_DIR}"
+            git clone ${CCACHE_REPO} ${CCACHE_SRC_DIR}
+        fi
+
+        echo "git checkout ${CCACHE_TAG} --force"
+        git checkout ${CCACHE_TAG} --force
+
+        if [[ -z $DEP_DOWNLOAD_ONLY ]]; then
+            time cmake -S . -B ${CCACHE_BUILD_DIR} -DCMAKE_BUILD_TYPE=Release \
+                       -DCMAKE_INSTALL_PREFIX=${CCACHE_INSTALL_DIR} \
+                       -DENABLE_TESTING=OFF -DENABLE_DOCUMENTATION=OFF \
+                       ${CCACHE_CONFIG_OPTS}
+            time cmake --build ${CCACHE_BUILD_DIR} --config Release --target install
+        fi
+    fi
+
+    popd
+    ls ${CCACHE_INSTALL_DIR}
+    ls ${CCACHE_INSTALL_DIR}/bin
+    echo "CCACHE_INSTALL_DIR=$CCACHE_INSTALL_DIR"
+    echo "CCACHE_DIR=$CCACHE_DIR"
+    mkdir -p $CCACHE_DIR
+    ls $CCACHE_DIR || true
+    export PATH=${CCACHE_INSTALL_DIR}/bin:$PATH
+    ${CCACHE_INSTALL_DIR}/bin/ccache -sv
+fi