Skip to content

Add feature maps (fmaps) mode for sliding-kernel feature extraction #1015

Add feature maps (fmaps) mode for sliding-kernel feature extraction

Add feature maps (fmaps) mode for sliding-kernel feature extraction #1015

Workflow file for this run

name: Build Wheels
on:
workflow_dispatch:
inputs:
force_prereqs:
description: 'Force rebuild prereqs (ignore cache)'
type: boolean
default: false
pull_request:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true
permissions:
actions: write
contents: read
jobs:
# ============================================================
# Prereq jobs — one per OS, all start in parallel
# ============================================================
prereqs_ubuntu:
name: Build prereqs (ubuntu-22.04)
runs-on: ubuntu-22.04
continue-on-error: true
steps:
- uses: actions/checkout@v3
name: Check out
- name: Compute cache key
id: cache-key
run: echo "key=prereqs-ubuntu-22.04-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat', 'ci-utils/validate_prereqs.sh') }}" >> "$GITHUB_OUTPUT"
- name: Delete prereq cache (force rebuild)
if: inputs.force_prereqs == true
continue-on-error: true
env:
GH_TOKEN: ${{ github.token }}
run: |
gh actions-cache delete "${{ steps.cache-key.outputs.key }}" \
-R ${{ github.repository }} --confirm || true
- name: Restore cached prereqs
id: cache
uses: actions/cache/restore@v4
with:
path: prereqs.tar.gz
key: ${{ steps.cache-key.outputs.key }}
- name: Build prereqs
if: steps.cache.outputs.cache-hit != 'true'
run: |
docker run --rm \
-v "${{ github.workspace }}:/work" \
-w /work \
quay.io/pypa/manylinux_2_28_x86_64:latest \
bash -c "
dnf -y install llvm libevent-devel openssl-devel &&
bash ci-utils/install_prereq_linux.sh --build_arrow yes
"
- name: Validate prereqs
if: steps.cache.outputs.cache-hit != 'true'
shell: bash
run: bash ci-utils/validate_prereqs.sh
- name: Compress prereqs
if: steps.cache.outputs.cache-hit != 'true'
run: tar -czf prereqs.tar.gz local_install
- name: Save prereqs cache
if: steps.cache.outputs.cache-hit != 'true'
uses: actions/cache/save@v4
with:
path: prereqs.tar.gz
key: ${{ steps.cache-key.outputs.key }}
- name: Upload prereqs
uses: actions/upload-artifact@v4
with:
name: prereqs-ubuntu-22.04
path: prereqs.tar.gz
retention-days: 1
prereqs_ubuntu_arm:
name: Build prereqs (ubuntu-22.04-arm)
runs-on: ubuntu-22.04-arm
continue-on-error: true
steps:
- uses: actions/checkout@v3
name: Check out
- name: Compute cache key
id: cache-key
run: echo "key=prereqs-ubuntu-22.04-arm-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat', 'ci-utils/validate_prereqs.sh') }}" >> "$GITHUB_OUTPUT"
- name: Delete prereq cache (force rebuild)
if: inputs.force_prereqs == true
continue-on-error: true
env:
GH_TOKEN: ${{ github.token }}
run: |
gh actions-cache delete "${{ steps.cache-key.outputs.key }}" \
-R ${{ github.repository }} --confirm || true
- name: Restore cached prereqs
id: cache
uses: actions/cache/restore@v4
with:
path: prereqs.tar.gz
key: ${{ steps.cache-key.outputs.key }}
- name: Build prereqs
if: steps.cache.outputs.cache-hit != 'true'
run: |
docker run --rm \
-v "${{ github.workspace }}:/work" \
-w /work \
quay.io/pypa/manylinux_2_28_aarch64:latest \
bash -c "
dnf -y install llvm libevent-devel openssl-devel &&
bash ci-utils/install_prereq_linux.sh --build_arrow yes
"
- name: Validate prereqs
if: steps.cache.outputs.cache-hit != 'true'
shell: bash
run: bash ci-utils/validate_prereqs.sh
- name: Compress prereqs
if: steps.cache.outputs.cache-hit != 'true'
run: tar -czf prereqs.tar.gz local_install
- name: Save prereqs cache
if: steps.cache.outputs.cache-hit != 'true'
uses: actions/cache/save@v4
with:
path: prereqs.tar.gz
key: ${{ steps.cache-key.outputs.key }}
- name: Upload prereqs
uses: actions/upload-artifact@v4
with:
name: prereqs-ubuntu-22.04-arm
path: prereqs.tar.gz
retention-days: 1
prereqs_windows:
name: Build prereqs (windows-latest)
runs-on: windows-latest
continue-on-error: true
steps:
- uses: actions/checkout@v3
name: Check out
- name: Compute cache key
id: cache-key
shell: bash
run: echo "key=prereqs-windows-latest-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat', 'ci-utils/validate_prereqs.sh') }}" >> "$GITHUB_OUTPUT"
- name: Delete prereq cache (force rebuild)
if: inputs.force_prereqs == true
continue-on-error: true
shell: bash
env:
GH_TOKEN: ${{ github.token }}
run: |
gh actions-cache delete "${{ steps.cache-key.outputs.key }}" \
-R ${{ github.repository }} --confirm || true
- name: Restore cached prereqs
id: cache
uses: actions/cache/restore@v4
with:
path: prereqs.tar.gz
key: ${{ steps.cache-key.outputs.key }}
- uses: ilammy/msvc-dev-cmd@v1
if: steps.cache.outputs.cache-hit != 'true'
name: Add MSVS Path
- name: Build prereqs
if: steps.cache.outputs.cache-hit != 'true'
run: ci-utils\install_prereq_win.bat
- name: Validate prereqs
if: steps.cache.outputs.cache-hit != 'true'
shell: bash
run: bash ci-utils/validate_prereqs.sh
- name: Compress prereqs
if: steps.cache.outputs.cache-hit != 'true'
run: tar -czf prereqs.tar.gz local_install
- name: Save prereqs cache
if: steps.cache.outputs.cache-hit != 'true'
uses: actions/cache/save@v4
with:
path: prereqs.tar.gz
key: ${{ steps.cache-key.outputs.key }}
- name: Upload prereqs
uses: actions/upload-artifact@v4
with:
name: prereqs-windows-latest
path: prereqs.tar.gz
retention-days: 1
prereqs_macos_intel:
name: Build prereqs (macos-15-intel)
runs-on: macos-15-intel
continue-on-error: true
steps:
- uses: actions/checkout@v3
name: Check out
- name: Compute cache key
id: cache-key
run: echo "key=prereqs-macos-15-intel-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat', 'ci-utils/validate_prereqs.sh') }}" >> "$GITHUB_OUTPUT"
- name: Delete prereq cache (force rebuild)
if: inputs.force_prereqs == true
continue-on-error: true
env:
GH_TOKEN: ${{ github.token }}
run: |
gh actions-cache delete "${{ steps.cache-key.outputs.key }}" \
-R ${{ github.repository }} --confirm || true
- name: Restore cached prereqs
id: cache
uses: actions/cache/restore@v4
with:
path: prereqs.tar.gz
key: ${{ steps.cache-key.outputs.key }}
- name: Build prereqs
if: steps.cache.outputs.cache-hit != 'true'
env:
MACOSX_DEPLOYMENT_TARGET: "10.15"
run: bash ci-utils/install_prereq_linux.sh --build_arrow yes
- name: Validate prereqs
if: steps.cache.outputs.cache-hit != 'true'
shell: bash
run: bash ci-utils/validate_prereqs.sh
- name: Compress prereqs
if: steps.cache.outputs.cache-hit != 'true'
run: tar -czf prereqs.tar.gz local_install
- name: Save prereqs cache
if: steps.cache.outputs.cache-hit != 'true'
uses: actions/cache/save@v4
with:
path: prereqs.tar.gz
key: ${{ steps.cache-key.outputs.key }}
- name: Upload prereqs
uses: actions/upload-artifact@v4
with:
name: prereqs-macos-15-intel
path: prereqs.tar.gz
retention-days: 1
prereqs_macos_arm64:
name: Build prereqs (macos-14)
runs-on: macos-14
continue-on-error: true
steps:
- uses: actions/checkout@v3
name: Check out
- name: Compute cache key
id: cache-key
run: echo "key=prereqs-macos-14-${{ hashFiles('ci-utils/install_prereq_linux.sh', 'ci-utils/install_prereq_win.bat', 'ci-utils/validate_prereqs.sh') }}" >> "$GITHUB_OUTPUT"
- name: Delete prereq cache (force rebuild)
if: inputs.force_prereqs == true
continue-on-error: true
env:
GH_TOKEN: ${{ github.token }}
run: |
gh actions-cache delete "${{ steps.cache-key.outputs.key }}" \
-R ${{ github.repository }} --confirm || true
- name: Restore cached prereqs
id: cache
uses: actions/cache/restore@v4
with:
path: prereqs.tar.gz
key: ${{ steps.cache-key.outputs.key }}
- name: Build prereqs
if: steps.cache.outputs.cache-hit != 'true'
env:
MACOSX_DEPLOYMENT_TARGET: "11.0"
run: bash ci-utils/install_prereq_linux.sh --build_arrow yes
- name: Validate prereqs
if: steps.cache.outputs.cache-hit != 'true'
shell: bash
run: bash ci-utils/validate_prereqs.sh
- name: Compress prereqs
if: steps.cache.outputs.cache-hit != 'true'
run: tar -czf prereqs.tar.gz local_install
- name: Save prereqs cache
if: steps.cache.outputs.cache-hit != 'true'
uses: actions/cache/save@v4
with:
path: prereqs.tar.gz
key: ${{ steps.cache-key.outputs.key }}
- name: Upload prereqs
uses: actions/upload-artifact@v4
with:
name: prereqs-macos-14
path: prereqs.tar.gz
retention-days: 1
# ============================================================
# Wheel build jobs — each starts as soon as its prereq finishes
# ============================================================
wheels_ubuntu:
name: Build wheels (ubuntu-22.04 / ${{ matrix.cibw_build }})
runs-on: ubuntu-22.04
needs: prereqs_ubuntu
continue-on-error: true
strategy:
matrix:
cibw_build: ["cp310", "cp311", "cp312", "cp313"]
steps:
- uses: actions/checkout@v3
name: Check out
with:
submodules: recursive
- name: Download prereqs
uses: actions/download-artifact@v4
with:
name: prereqs-ubuntu-22.04
path: .
- name: Extract prereqs
run: tar -xzf prereqs.tar.gz
- uses: actions/setup-python@v4
name: Install Python
with:
python-version: '3.11'
- name: Install cibuildwheel
run: |
python -m pip install --upgrade pip
python -m pip install cibuildwheel==3.0.0 delvewheel wheel
- name: Building wheels
run: python -m cibuildwheel --output-dir dist
env:
CIBW_BUILD: ${{ matrix.cibw_build }}-*
CIBW_SKIP: "*musllinux*"
CIBW_BUILD_VERBOSITY: 3
CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux_2_28_x86_64:latest
CIBW_BEFORE_ALL_LINUX: >
dnf -y install llvm libevent-devel openssl-devel &&
mkdir -p /tmp/nyxus_bld &&
cp -a local_install /tmp/nyxus_bld
CIBW_ENVIRONMENT_LINUX: >
LD_LIBRARY_PATH="/tmp/nyxus_bld/local_install/lib:/tmp/nyxus_bld/local_install/lib64:$LD_LIBRARY_PATH"
NYXUS_DEP_DIR="/tmp/nyxus_bld/local_install"
CIBW_ARCHS: auto64
CIBW_TEST_REQUIRES: numpy pandas pyarrow pytest bfio
CIBW_TEST_COMMAND: pytest -vv {project}/tests/python
- name: Upload Artifact
uses: actions/upload-artifact@v4
with:
name: nyxus-wheels-ubuntu-22.04-auto64-${{ matrix.cibw_build }}
path: dist/*.whl
retention-days: 1
wheels_ubuntu_arm:
name: Build wheels (ubuntu-22.04-arm / ${{ matrix.cibw_build }})
runs-on: ubuntu-22.04-arm
needs: prereqs_ubuntu_arm
continue-on-error: true
strategy:
matrix:
cibw_build: ["cp310", "cp311", "cp312", "cp313"]
steps:
- uses: actions/checkout@v3
name: Check out
with:
submodules: recursive
- name: Download prereqs
uses: actions/download-artifact@v4
with:
name: prereqs-ubuntu-22.04-arm
path: .
- name: Extract prereqs
run: tar -xzf prereqs.tar.gz
- uses: actions/setup-python@v4
name: Install Python
with:
python-version: '3.11'
- name: Install cibuildwheel
run: |
python -m pip install --upgrade pip
python -m pip install cibuildwheel==3.0.0 delvewheel wheel
- name: Building wheels
run: python -m cibuildwheel --output-dir dist
env:
CIBW_BUILD: ${{ matrix.cibw_build }}-*
CIBW_SKIP: "*musllinux*"
CIBW_BUILD_VERBOSITY: 3
CIBW_MANYLINUX_AARCH64_IMAGE: quay.io/pypa/manylinux_2_28_aarch64:latest
CIBW_BEFORE_ALL_LINUX: >
dnf -y install llvm libevent-devel openssl-devel &&
mkdir -p /tmp/nyxus_bld &&
cp -a local_install /tmp/nyxus_bld
CIBW_ENVIRONMENT_LINUX: >
LD_LIBRARY_PATH="/tmp/nyxus_bld/local_install/lib:/tmp/nyxus_bld/local_install/lib64:$LD_LIBRARY_PATH"
NYXUS_DEP_DIR="/tmp/nyxus_bld/local_install"
CIBW_ARCHS: auto64
CIBW_TEST_REQUIRES: numpy pandas pyarrow pytest bfio
CIBW_TEST_COMMAND: pytest -vv {project}/tests/python
- name: Upload Artifact
uses: actions/upload-artifact@v4
with:
name: nyxus-wheels-ubuntu-22.04-arm-auto64-${{ matrix.cibw_build }}
path: dist/*.whl
retention-days: 1
wheels_windows:
name: Build wheels (windows-latest / ${{ matrix.cibw_build }})
runs-on: windows-latest
needs: prereqs_windows
continue-on-error: true
strategy:
matrix:
cibw_build: ["cp310", "cp311", "cp312", "cp313"]
steps:
- uses: actions/checkout@v3
name: Check out
with:
submodules: recursive
- name: Download prereqs
uses: actions/download-artifact@v4
with:
name: prereqs-windows-latest
path: .
- name: Extract prereqs
run: tar -xzf prereqs.tar.gz
- uses: ilammy/msvc-dev-cmd@v1
name: Add MSVS Path
- uses: actions/setup-python@v4
name: Install Python
with:
python-version: '3.11'
- name: Install cibuildwheel
run: |
python -m pip install --upgrade pip
python -m pip install cibuildwheel==3.0.0 delvewheel wheel
- name: Building wheels
run: python -m cibuildwheel --output-dir dist
env:
CIBW_BUILD: ${{ matrix.cibw_build }}-*
CIBW_BUILD_VERBOSITY: 3
CIBW_BEFORE_ALL_WINDOWS: xcopy /E /I /y local_install C:\TEMP\nyxus_bld\local_install
CIBW_ENVIRONMENT_WINDOWS: PATH="C:\\TEMP\\nyxus_bld\\local_install\\bin;$PATH" NYXUS_DEP_DIR="C:\\TEMP\\nyxus_bld\\local_install"
CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}"
CIBW_ARCHS: auto64
CIBW_TEST_REQUIRES: numpy pandas pyarrow pytest bfio
CIBW_TEST_COMMAND: pytest -vv {project}/tests/python
- name: Upload Artifact
uses: actions/upload-artifact@v4
with:
name: nyxus-wheels-windows-latest-auto64-${{ matrix.cibw_build }}
path: dist/*.whl
retention-days: 1
wheels_macos_intel:
name: Build wheels (macos-15-intel / ${{ matrix.cibw_build }})
runs-on: macos-15-intel
needs: prereqs_macos_intel
continue-on-error: true
env:
MACOSX_DEPLOYMENT_TARGET: "10.15"
strategy:
matrix:
cibw_build: ["cp310", "cp311", "cp312", "cp313"]
steps:
- uses: actions/checkout@v3
name: Check out
with:
submodules: recursive
- name: Download prereqs
uses: actions/download-artifact@v4
with:
name: prereqs-macos-15-intel
path: .
- name: Extract prereqs
run: tar -xzf prereqs.tar.gz
- uses: actions/setup-python@v4
name: Install Python
with:
python-version: '3.11'
- name: Install cibuildwheel
run: |
python -m pip install --upgrade pip
python -m pip install cibuildwheel==3.0.0 delvewheel wheel
- name: Building wheels
run: python -m cibuildwheel --output-dir dist
env:
CIBW_BUILD: ${{ matrix.cibw_build }}-*
CIBW_BUILD_VERBOSITY: 3
CIBW_BEFORE_ALL_MACOS: |
mkdir -p /tmp/nyxus_bld &&
cp -a local_install /tmp/nyxus_bld
CIBW_ENVIRONMENT_MACOS: >
MACOSX_DEPLOYMENT_TARGET=10.15
SDKROOT="$(xcrun --sdk macosx --show-sdk-path)"
CMAKE_OSX_SYSROOT="$SDKROOT"
CC="$(xcrun -f clang)"
CXX="$(xcrun -f clang++)"
CMAKE_C_COMPILER="$CC"
CMAKE_CXX_COMPILER="$CXX"
CXXFLAGS="-isysroot $SDKROOT -stdlib=libc++ -mmacosx-version-min=$MACOSX_DEPLOYMENT_TARGET"
LDFLAGS="-isysroot $SDKROOT -lc++ -lc++abi -mmacosx-version-min=$MACOSX_DEPLOYMENT_TARGET"
ARROW_HAVE_LIBATOMIC=OFF
REPAIR_LIBRARY_PATH="/tmp/nyxus_bld/local_install/lib:/tmp/nyxus_bld/local_install/lib64"
NYXUS_DEP_DIR="/tmp/nyxus_bld/local_install"
CIBW_REPAIR_WHEEL_COMMAND_MACOS: |
DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} &&
MACOSX_DEPLOYMENT_TARGET=10.15 DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel \
--require-archs {delocate_archs} \
--require-target-macos-version $MACOSX_DEPLOYMENT_TARGET \
-w {dest_dir} {wheel} \
-e libssl -e libzstd -e libcrypto -e libwebp -e libsharpyuv
CIBW_ARCHS: auto64
CIBW_TEST_REQUIRES: numpy pandas pyarrow pytest bfio
CIBW_TEST_COMMAND: pytest -vv {project}/tests/python
- name: Upload Artifact
uses: actions/upload-artifact@v4
with:
name: nyxus-wheels-macos-15-intel-auto64-${{ matrix.cibw_build }}
path: dist/*.whl
retention-days: 1
wheels_macos_arm64:
name: Build wheels (macos-14 / ${{ matrix.cibw_build }})
runs-on: macos-14
needs: prereqs_macos_arm64
continue-on-error: true
env:
MACOSX_DEPLOYMENT_TARGET: "11.0"
strategy:
matrix:
cibw_build: ["cp310", "cp311", "cp312", "cp313"]
steps:
- uses: actions/checkout@v3
name: Check out
- name: Download prereqs
uses: actions/download-artifact@v4
with:
name: prereqs-macos-14
path: .
- name: Extract prereqs
run: tar -xzf prereqs.tar.gz
- uses: actions/setup-python@v4
name: Install Python
with:
python-version: '3.11'
- name: Install cibuildwheel
run: |
python -m pip install --upgrade pip
python -m pip install cibuildwheel==3.0.0 delvewheel wheel
- name: Building wheels
run: python -m cibuildwheel --output-dir dist
env:
CIBW_BUILD: ${{ matrix.cibw_build }}-*
CIBW_BUILD_VERBOSITY: 3
CIBW_ARCHS_MACOS: arm64
CIBW_BEFORE_ALL_MACOS: |
mkdir -p /tmp/nyxus_bld && cp -a local_install /tmp/nyxus_bld || true
CIBW_ENVIRONMENT_MACOS: >
MACOSX_DEPLOYMENT_TARGET=11.0
SDKROOT="$(xcrun --sdk macosx --show-sdk-path)"
CMAKE_OSX_SYSROOT="$SDKROOT"
CC="clang" CXX="clang++"
CMAKE_C_COMPILER="$CC" CMAKE_CXX_COMPILER="$CXX"
CXXFLAGS="-isysroot $SDKROOT -stdlib=libc++"
LDFLAGS="-isysroot $SDKROOT -lc++ -lc++abi"
REPAIR_LIBRARY_PATH="/tmp/nyxus_bld/local_install/lib:/tmp/nyxus_bld/local_install/lib64"
NYXUS_DEP_DIR="/tmp/nyxus_bld/local_install"
CIBW_REPAIR_WHEEL_COMMAND_MACOS: |
DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} &&
MACOSX_DEPLOYMENT_TARGET=11.0 DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel \
--require-archs {delocate_archs} \
--require-target-macos-version $MACOSX_DEPLOYMENT_TARGET \
-w {dest_dir} {wheel} \
-e libssl -e libzstd -e libcrypto
CIBW_ARCHS: arm64
CIBW_TEST_REQUIRES: numpy pandas pyarrow pytest bfio
CIBW_TEST_COMMAND: pytest -vv {project}/tests/python
- name: Upload Artifact
uses: actions/upload-artifact@v4
with:
name: nyxus-wheels-macos-14-arm64-${{ matrix.cibw_build }}
path: dist/*.whl
retention-days: 1