diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 377f74d363..f464007721 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -39,6 +39,12 @@ on: workflow_dispatch: # This allows manual triggering of the workflow from the web +# Allow subsequent pushes to the same PR or REF to cancel any previous jobs. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + + jobs: # Linux jobs run in Docker containers (manylinux), so the latest OS version # is OK. macOS and Windows jobs need to be locked to specific virtual @@ -141,6 +147,14 @@ jobs: with: python-version: '3.9' + - name: ccache-restore + id: ccache-restore + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ~/.ccache + key: wheel-${{runner.os}}-${{matrix.manylinux}}-${{matrix.python}} + restore-keys: wheel-${{runner.os}}-${{matrix.manylinux}}-${{matrix.python}} + - name: Build wheels # Note: the version of cibuildwheel should be kept in sync with src/python/stubs/CMakeLists.txt uses: pypa/cibuildwheel@d4a2945fcc8d13f20a1b99d461b8e844d5fc6e23 # v2.21.1 @@ -148,13 +162,35 @@ jobs: # pass GITHUB_ACTIONS through to the build container so that custom # processes can tell they are running in CI. CIBW_ENVIRONMENT_PASS_LINUX: GITHUB_ACTIONS + CIBW_BEFORE_ALL: "source src/build-scripts/build_ccache.bash && pwd && ext/dist/bin/ccache --max-size=200M && ext/dist/bin/ccache -sv && export CMAKE_C_COMPILER_LAUNCHER=/project/ext/dist/bin/ccache CMAKE_CXX_COMPILER_LAUNCHER=/project/ext/dist/bin/ccache" + CIBW_BEFORE_TEST: "ext/dist/bin/ccache -s" CIBW_BUILD: ${{ matrix.python }} CIBW_ARCHS: ${{ matrix.arch }} CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux }} + CIBW_ENVIRONMENT: > + CCACHE_DIR=/host//home/runner/.ccache + CCACHE_COMPRESSION=yes + CCACHE_PREBUILT=0 + CMAKE_BUILD_PARALLEL_LEVEL=4 + CTEST_PARALLEL_LEVEL=4 + SKBUILD_CMAKE_ARGS="-DLINKSTATIC=1" + SKBUILD_CMAKE_BUILD_TYPE="MinSizeRel" + SKBUILD_BUILD_DIR=/project/build + CXXFLAGS="-Wno-error=stringop-overflow -Wno-pragmas" + WebP_BUILD_VERSION="1.5.0" + # FIXME: Getting build problems when using WebP 1.6.0, so hold it back + # CMAKE_GENERATOR = "Ninja" + + - name: ccache-save + id: ccache-save + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ~/.ccache + key: wheel-${{runner.os}}-${{matrix.manylinux}}-${{matrix.python}} - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 with: - name: cibw-wheels-${{ matrix.python }}-${{ matrix.manylinux }} + name: cibw-wheels-${{matrix.manylinux}}-${{ matrix.python }}-${{ matrix.manylinux }} path: | ./wheelhouse/*.whl @@ -173,12 +209,12 @@ jobs: # --------------------------------------------------------------------------- linux-arm: - name: Build wheels on Linux ARM - runs-on: ubuntu-24.04-arm - if: | - github.event_name != 'schedule' || - github.repository == 'AcademySoftwareFoundation/OpenImageIO' - strategy: + name: Build wheels on Linux ARM + runs-on: ubuntu-24.04-arm + if: | + github.event_name != 'schedule' || + github.repository == 'AcademySoftwareFoundation/OpenImageIO' + strategy: matrix: include: # ------------------------------------------------------------------- @@ -205,37 +241,66 @@ jobs: python: cp313-manylinux_aarch64 arch: aarch64 - steps: - - name: Checkout repo - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + steps: + - name: Checkout repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Install Python - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 - with: - python-version: '3.9' - - - name: Build wheels - uses: pypa/cibuildwheel@d4a2945fcc8d13f20a1b99d461b8e844d5fc6e23 # v2.21.1 - env: - CIBW_BUILD: ${{ matrix.python }} - CIBW_ARCHS: ${{ matrix.arch }} - CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.manylinux }} - - - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 - with: - name: cibw-wheels-${{ matrix.python }}-${{ matrix.manylinux }} - path: | - ./wheelhouse/*.whl + - name: Install Python + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version: '3.9' - - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 - with: - name: stubs-${{ matrix.python }}-${{ matrix.manylinux }} - path: | - ./wheelhouse/OpenImageIO/__init__.pyi - # if stub validation fails we want to upload the stubs for users to review. - # keep the python build in sync with the version specified in tool.cibuildwheel.overrides - # section of pyproject.toml - if: always() && contains(matrix.python, 'cp311-manylinux') + - name: ccache-restore + id: ccache-restore + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ~/.ccache + key: wheel-${{runner.os}}-${{matrix.python}} + restore-keys: wheel-${{runner.os}}-${{matrix.python}} + + - name: Build wheels + uses: pypa/cibuildwheel@d4a2945fcc8d13f20a1b99d461b8e844d5fc6e23 # v2.21.1 + env: + CIBW_ENVIRONMENT_PASS_LINUX: GITHUB_ACTIONS + CIBW_BEFORE_ALL: "source src/build-scripts/build_ccache.bash && pwd && /project/ext/dist/bin/ccache --max-size=200M && /project/ext/dist/bin/ccache -sv && export CMAKE_C_COMPILER_LAUNCHER=/project/ext/dist/bin/ccache CMAKE_CXX_COMPILER_LAUNCHER=/project/ext/dist/bin/ccache" + CIBW_BEFORE_TEST: "ext/dist/bin/ccache -s" + CIBW_BUILD: ${{ matrix.python }} + CIBW_ARCHS: ${{ matrix.arch }} + CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.manylinux }} + CIBW_ENVIRONMENT: > + CCACHE_DIR=/host//home/runner/.ccache + CCACHE_COMPRESSION=yes + CCACHE_PREBUILT=0 + CMAKE_BUILD_PARALLEL_LEVEL=6 + CTEST_PARALLEL_LEVEL=6 + SKBUILD_CMAKE_ARGS="-DLINKSTATIC=1" + SKBUILD_CMAKE_BUILD_TYPE="MinSizeRel" + SKBUILD_BUILD_DIR=/project/build + CXXFLAGS="-Wno-error=stringop-overflow -Wno-pragmas" + WebP_BUILD_VERSION="1.5.0" + + - name: ccache-save + id: ccache-save + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ~/.ccache + key: wheel-${{runner.os}}-${{matrix.python}} + + - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 + with: + name: cibw-wheels-${{ matrix.python }}-${{ matrix.manylinux }} + path: | + ./wheelhouse/*.whl + + - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 + with: + name: stubs-${{ matrix.python }}-${{ matrix.manylinux }} + path: | + ./wheelhouse/OpenImageIO/__init__.pyi + # if stub validation fails we want to upload the stubs for users to review. + # keep the python build in sync with the version specified in tool.cibuildwheel.overrides + # section of pyproject.toml + if: always() && contains(matrix.python, 'cp311-manylinux') # --------------------------------------------------------------------------- # macOS Wheels @@ -278,6 +343,18 @@ jobs: with: python-version: '3.9' + - name: ccache-restore + id: ccache-restore + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ~/.ccache + key: wheel-${{runner.os}}-${{matrix.python}} + restore-keys: wheel-${{runner.os}}-${{matrix.python}} + + - name: Install build tools + run: | + brew install ninja ccache || true + - name: Remove brew OpenEXR/Imath run: | brew uninstall --ignore-dependencies openexr imath || true @@ -292,6 +369,18 @@ jobs: # TODO: Re-enable HEIF when we provide a build recipe that does # not include GPL-licensed dynamic libraries. USE_Libheif: 'OFF' + CMAKE_BUILD_PARALLEL_LEVEL: 6 + CTEST_PARALLEL_LEVEL: 6 + SKBUILD_BUILD_DIR: "/Users/runner/work/OpenImageIO/OpenImageIO/build" + CCACHE_DIR: /Users/runner/.ccache + CCACHE_COMPRESSION: yes + + - name: ccache-save + id: ccache-save + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ~/.ccache + key: wheel-${{runner.os}}-${{matrix.python}} - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 with: @@ -340,13 +429,36 @@ jobs: with: python-version: '3.9' + - name: ccache-restore + id: ccache-restore + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ~/.ccache + key: wheel-${{runner.os}}-${{matrix.python}} + restore-keys: wheel-${{runner.os}}-${{matrix.python}} + - name: Install build tools + run: | + brew install ninja ccache || true + - name: Build wheels uses: pypa/cibuildwheel@d4a2945fcc8d13f20a1b99d461b8e844d5fc6e23 # v2.21.1 env: CIBW_BUILD: ${{ matrix.python }} CIBW_ARCHS: ${{ matrix.arch }} CMAKE_GENERATOR: "Unix Makefiles" + CMAKE_BUILD_PARALLEL_LEVEL: 6 + CTEST_PARALLEL_LEVEL: 6 + SKBUILD_BUILD_DIR: "/Users/runner/work/OpenImageIO/OpenImageIO/build" + CCACHE_DIR: /Users/runner/.ccache + CCACHE_COMPRESSION: yes + + - name: ccache-save + id: ccache-save + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ~/.ccache + key: wheel-${{runner.os}}-${{matrix.python}} - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 with: @@ -400,6 +512,11 @@ jobs: CIBW_BUILD: ${{ matrix.python }} CIBW_ARCHS: ${{ matrix.arch }} CMAKE_POLICY_VERSION_MINIMUM: 3.5 + CMAKE_BUILD_PARALLEL_LEVEL: 4 + CTEST_PARALLEL_LEVEL: 4 + SKBUILD_BUILD_DIR: "$HOME/OpenImageIO/OpenImageIO/build" + CCACHE_DIR: ~/.ccache + CCACHE_COMPRESSION: yes - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 with: diff --git a/pyproject.toml b/pyproject.toml index 2c10db63ef..164eb706d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,6 @@ +# See docs at https://cibuildwheel.pypa.io/en/stable/options +# for description of all the options in this file. + [project] name = "OpenImageIO" # The build backend ascertains the version from the CMakeLists.txt file. diff --git a/src/build-scripts/build_ccache.bash b/src/build-scripts/build_ccache.bash new file mode 100755 index 0000000000..0290c599bb --- /dev/null +++ b/src/build-scripts/build_ccache.bash @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +# Utility script to download or build ccacheccache +# +# Copyright Contributors to the OpenImageIO project. +# SPDX-License-Identifier: Apache-2.0 +# https://github.com/AcademySoftwareFoundation/OpenImageIO + +# Exit the whole script if any command fails. +set -ex + +echo "Building ccache" +uname +ARCH=`uname -m` +echo "HOME=$HOME" +echo "PWD=$PWD" +echo "ARCH=$ARCH" + +CCACHE_PREBULT=${CCACHE_PREBULT:=1} + +# Repo and branch/tag/commit of ccache to download if we don't have it yet +CCACHE_REPO=${CCACHE_REPO:=https://github.com/ccache/ccache} +CCACHE_VERSION=${CCACHE_VERSION:=4.12} +CCACHE_TAG=${CCACHE_TAG:=v${CCACHE_VERSION}} +LOCAL_DEPS_DIR=${LOCAL_DEPS_DIR:=${PWD}/ext} +# Where to put ccache repo source (default to the ext area) +CCACHE_SRC_DIR=${CCACHE_SRC_DIR:=${LOCAL_DEPS_DIR}/ccache} +# Temp build area (default to a build/ subdir under source) +CCACHE_BUILD_DIR=${CCACHE_BUILD_DIR:=${CCACHE_SRC_DIR}/build} +# Install area for ccache (default to ext/dist) +CCACHE_INSTALL_DIR=${CCACHE_INSTALL_DIR:=${PWD}/ext/dist} +CCACHE_CONFIG_OPTS=${CCACHE_CONFIG_OPTS:=} + + +# if [[ `uname` == "Linux" && `uname -m` == "x86_64" ]] ; then +if [[ `uname` == "Linux" ]] ; then + mkdir -p ${CCACHE_SRC_DIR} + pushd ${CCACHE_SRC_DIR} + + if [[ "$CCACHE_PREBUILT" != "0" ]] ; then + # + # Try to download -- had trouble with this on runners + # + CCACHE_DESCRIPTOR="ccache-${CCACHE_VERSION}-linux-x86_64" + curl --location "${CCACHE_REPO}/releases/download/${CCACHE_TAG}/${CCACHE_DESCRIPTOR}.tar.xz" -o ccache.tar.xz + tar xJvf ccache.tar.xz + mkdir -p ${CCACHE_INSTALL_DIR}/bin + cp ${CCACHE_SRC_DIR}/${CCACHE_DESCRIPTOR}/ccache ${CCACHE_INSTALL_DIR}/bin + else + # Clone ccache project from GitHub and build + if [[ ! -e ${CCACHE_SRC_DIR}/.git ]] ; then + echo "git clone ${CCACHE_REPO} ${CCACHE_SRC_DIR}" + git clone ${CCACHE_REPO} ${CCACHE_SRC_DIR} + fi + + echo "git checkout ${CCACHE_TAG} --force" + git checkout ${CCACHE_TAG} --force + + if [[ -z $DEP_DOWNLOAD_ONLY ]]; then + time cmake -S . -B ${CCACHE_BUILD_DIR} -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=${CCACHE_INSTALL_DIR} \ + -DENABLE_TESTING=OFF -DENABLE_DOCUMENTATION=OFF \ + ${CCACHE_CONFIG_OPTS} + time cmake --build ${CCACHE_BUILD_DIR} --config Release --target install + fi + fi + + popd + ls ${CCACHE_INSTALL_DIR} + ls ${CCACHE_INSTALL_DIR}/bin + echo "CCACHE_INSTALL_DIR=$CCACHE_INSTALL_DIR" + echo "CCACHE_DIR=$CCACHE_DIR" + mkdir -p $CCACHE_DIR + ls $CCACHE_DIR || true + export PATH=${CCACHE_INSTALL_DIR}/bin:$PATH + ${CCACHE_INSTALL_DIR}/bin/ccache -sv +fi diff --git a/src/cmake/compiler.cmake b/src/cmake/compiler.cmake index 6dfad31395..1dc5995bac 100644 --- a/src/cmake/compiler.cmake +++ b/src/cmake/compiler.cmake @@ -243,18 +243,31 @@ endif () # logic here makes it work even if the user is unaware of ccache. If it's # not found on the system, it will simply be silently not used. option (USE_CCACHE "Use ccache if found" ON) -find_program (CCACHE_EXE ccache) -if (CCACHE_EXE AND USE_CCACHE) - if (CMAKE_COMPILER_IS_CLANG AND USE_QT AND (NOT DEFINED ENV{CCACHE_CPP2})) - message (STATUS "Ignoring ccache because clang + Qt + env CCACHE_CPP2 is not set") - else () - if (NOT ${CXX_COMPILER_LAUNCHER} MATCHES "ccache") - set (CXX_COMPILER_LAUNCHER ${CCACHE_EXE} ${CXX_COMPILER_LAUNCHER}) - endif () - if (NOT ${C_COMPILER_LAUNCHER} MATCHES "ccache") - set (C_COMPILER_LAUNCHER ${CCACHE_EXE} ${C_COMPILER_LAUNCHER}) +if (USE_CCACHE) + find_program (CCACHE_EXE ccache + PATHS "${PROJECT_SOURCE_DIR}/ext/dist/" + "${PROJECT_SOURCE_DIR}/ext/dist/bin") + if (CCACHE_EXE) + if (CMAKE_COMPILER_IS_CLANG AND USE_QT AND (NOT DEFINED ENV{CCACHE_CPP2})) + message (STATUS "Ignoring ccache because clang + Qt + env CCACHE_CPP2 is not set") + else () + message (STATUS "CMAKE_CXX_COMPILER_LAUNCHER: ${CMAKE_CXX_COMPILER_LAUNCHER}") + + if (NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache") + set (CMAKE_CXX_COMPILER_LAUNCHER ${CCACHE_EXE}) + message (STATUS "first if CMAKE_CXX_COMPILER_LAUNCHER: ${CMAKE_CXX_COMPILER_LAUNCHER}") + else () + message (STATUS "first else CMAKE_CXX_COMPILER_LAUNCHER: '${CMAKE_CXX_COMPILER_LAUNCHER}'") + endif () + if (NOT CMAKE_C_COMPILER_LAUNCHER MATCHES "ccache") + set (CMAKE_C_COMPILER_LAUNCHER ${CCACHE_EXE}) + endif () + message (STATUS "ccache enabled: ${CCACHE_EXE}") + message (STATUS "CCACHE_DIR env: $ENV{CCACHE_DIR}") + message (STATUS "CMAKE_CXX_COMPILER_LAUNCHER: ${CMAKE_CXX_COMPILER_LAUNCHER}") endif () - message (STATUS "ccache enabled: ${CCACHE_EXE}") + else () + message (STATUS "ccache not found") endif () endif () @@ -267,8 +280,8 @@ endif () # set `-j 1` or CMAKE_BUILD_PARALLEL_LEVEL to 1. option (TIME_COMMANDS "Time each compile and link command" OFF) if (TIME_COMMANDS) - set (CXX_COMPILER_LAUNCHER ${CMAKE_COMMAND} -E time ${CXX_COMPILER_LAUNCHER}) - set (C_COMPILER_LAUNCHER ${CMAKE_COMMAND} -E time ${C_COMPILER_LAUNCHER}) + set (CMAKE_CXX_COMPILER_LAUNCHER ${CMAKE_COMMAND} -E time ${CMAKE_CXX_COMPILER_LAUNCHER}) + set (CMAKE_C_COMPILER_LAUNCHER ${CMAKE_COMMAND} -E time ${CMAKE_C_COMPILER_LAUNCHER}) endif () diff --git a/src/cmake/dependency_utils.cmake b/src/cmake/dependency_utils.cmake index e1c084bc3e..a7bde64b72 100644 --- a/src/cmake/dependency_utils.cmake +++ b/src/cmake/dependency_utils.cmake @@ -110,7 +110,7 @@ function (print_package_notfound_report) list (REMOVE_DUPLICATES CFP_ALL_BUILD_DEPS_NOTFOUND) foreach (_pkg IN LISTS CFP_ALL_BUILD_DEPS_NOTFOUND) if (_pkg IN_LIST CFP_LOCALLY_BUILT_DEPS) - message (STATUS " ${_pkg} ${_${_pkg}_version_range} ${${_pkg}_NOT_FOUND_EXPLANATION} ${ColorMagenta}(${${_pkg}_VERSION} BUILT LOCALLY)${ColorReset}") + message (STATUS " ${_pkg} ${_${_pkg}_version_range} ${${_pkg}_NOT_FOUND_EXPLANATION} ${ColorMagenta}(${${_pkg}_VERSION} BUILT LOCALLY in ${${_pkg}_build_elapsed_time}s)${ColorReset}") else () message (STATUS " ${_pkg} ${_${_pkg}_version_range} ${${_pkg}_NOT_FOUND_EXPLANATION}") endif () @@ -594,9 +594,10 @@ macro (build_dependency_with_cmake pkgname) ${ARGN}) message (STATUS "Building local ${pkgname} ${_pkg_VERSION} from ${_pkg_GIT_REPOSITORY}") - + string (TIMESTAMP ${pkgname}_build_start_time "%s") + if(DEFINED ${pkgname}_CMAKELISTS_TEMPLATE_PATH AND ${pkgname}_CMAKELISTS_TEMPLATE_PATH) - message (STATUS "cmakelist template provided on: ${${pkgname}_CMAKELISTS_TEMPLATE_PATH}") + message (STATUS "cmakelist template provided on: ${${pkgname}_CMAKELISTS_TEMPLATE_PATH}") endif() set (${pkgname}_LOCAL_SOURCE_DIR "${${PROJECT_NAME}_LOCAL_DEPS_ROOT}/${pkgname}") @@ -685,6 +686,8 @@ macro (build_dependency_with_cmake pkgname) -DCMAKE_INSTALL_PREFIX=${${pkgname}_LOCAL_INSTALL_DIR} # Same build type as us -DCMAKE_BUILD_TYPE=${${PROJECT_NAME}_DEPENDENCY_BUILD_TYPE} + -DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER} + -DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER} # Shhhh -DCMAKE_MESSAGE_INDENT=" " -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF @@ -712,6 +715,9 @@ macro (build_dependency_with_cmake pkgname) set (${pkgname}_ROOT ${${pkgname}_LOCAL_INSTALL_DIR}) list (APPEND CMAKE_PREFIX_PATH ${${pkgname}_LOCAL_INSTALL_DIR}) endif () + string (TIMESTAMP ${pkgname}_build_end_time "%s") + math (EXPR ${pkgname}_build_elapsed_time "${${pkgname}_build_end_time} - ${${pkgname}_build_start_time}") + # message (STATUS "Build time of ${${pkgname}_build_elapsed_time}s") endmacro () @@ -748,4 +754,4 @@ macro (alias_library_if_not_exists newalias realtarget) if (NOT TARGET ${newalias} AND TARGET ${realtarget}) add_library(${newalias} ALIAS ${realtarget}) endif () -endmacro () \ No newline at end of file +endmacro ()