Skip to content
CI/CD

generalize is_fileset_or_union utility to is_subclass_or_union #260

Sign in to view logs

generalize is_fileset_or_union utility to is_subclass_or_union

generalize is_fileset_or_union utility to is_subclass_or_union #260

Workflow file for this run

.github/workflows/ci-cd.yml at 756825b
# This workflows will upload a Python Package using Twine when a release is created
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
name: CI/CD
on:
release:
types: [published]
push:
branches:
- main
pull_request:
branches:
- main
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
env:
# Force tox and pytest to use color
FORCE_COLOR: true
jobs:
build:
name: Build & verify package
runs-on: ubuntu-latest
permissions:
attestations: write
id-token: write
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: hynek/build-and-inspect-python-package@v2
with:
attest-build-provenance-github: ${{ github.event_name != 'pull_request' }}
test:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
python-version: ['3.11', '3.12', '3.13']
dependencies: [latest, pre]
include:
# Test minimum dependencies on oldest supported Python
- os: ubuntu-latest
python-version: "3.11"
dependencies: min
fail-fast: false
env:
DEPENDS: ${{ matrix.dependencies }}
steps:
- name: Fetch repository
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Install the latest version of uv
uses: astral-sh/setup-uv@v6
- name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install tox
run: |
uv tool install tox --with=tox-uv --with=tox-gh-actions
- name: Show tox config
run: tox c
- name: Run tox
run: tox -v --exit-and-dump-after 1200
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v5
with:
fail_ci_if_error: true
token: ${{ secrets.CODECOV_TOKEN }}
test-singularity:
runs-on: ubuntu-22.04
strategy:
matrix:
python-version: ['3.11', '3.12', '3.13']
dependencies: [latest]
fail-fast: False
env:
DEPENDS: ${{ matrix.dependencies }}
steps:
- name: Set env
run: |
echo "RELEASE_VERSION=v3.7.1" >> $GITHUB_ENV
echo "NO_ET=TRUE" >> $GITHUB_ENV
- name: Setup Singularity
uses: actions/checkout@v4
with:
repository: hpcng/singularity
ref: 'v3.7.1'
path: 'singularity'
- name: Setup GO
uses: actions/setup-go@v5
with:
go-version: '^1.13'
- name: Install OS deps
run: |
sudo apt-get update
sudo apt-get install flawfinder squashfs-tools uuid-dev libuuid1 libffi-dev libssl-dev libssl1.1 \
libarchive-dev libgpgme11-dev libseccomp-dev wget gcc make pkg-config -y
- name: Build
run: |
cd singularity
./mconfig --without-suid -p /usr/local/
make -C ./builddir
sudo make -C ./builddir install
cd ..
- name: Echo singularity version
run: |
echo ${{ github.ref }}
singularity --version
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Install the latest version of uv
uses: astral-sh/setup-uv@v6
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install tox
run: |
uv tool install tox --with=tox-uv --with=tox-gh-actions
- name: Show tox config
run: tox c
- name: Run tox
# Run test files with singularity tests; re-add the overridable "-n auto"
run: |
tox -v --exit-and-dump-after 1200 -- -n auto \
pydra/environments/tests/test_singularity.py pydra/environments/tests/test_environments.py
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v5
with:
fail_ci_if_error: true
token: ${{ secrets.CODECOV_TOKEN }}
test-slurm:
strategy:
matrix:
python-version: [3.11.5]
fail-fast: false
runs-on: ubuntu-latest
env:
DOCKER_IMAGE: adi611/docker-centos7-slurm:23.02.1
steps:
- name: Disable etelemetry
run: echo "NO_ET=TRUE" >> $GITHUB_ENV
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Pull docker image
run: |
docker pull $DOCKER_IMAGE
# Have image running in the background
docker run `bash <(curl -s https://codecov.io/env)` -itd -h slurmctl --cap-add sys_admin -d --name slurm -v `pwd`:/pydra -e NO_ET=$NO_ET $DOCKER_IMAGE
- name: Display previous jobs with sacct
run: |
echo "Allowing ports/daemons time to start" && sleep 10
docker exec slurm bash -c "sacctmgr -i add account none,test Cluster=linux Description='none' Organization='none'"
docker exec slurm bash -c "sacct && sinfo && squeue" 2&> /dev/null
if [ $? -ne 0 ]; then
echo "Slurm docker image error"
exit 1
fi
- name: Setup Python
run: |
docker exec slurm bash -c "echo $NO_ET"
docker exec slurm bash -c "ls -la && echo list top level dir"
docker exec slurm bash -c "ls -la /pydra && echo list pydra dir"
docker exec slurm bash -c "CONFIGURE_OPTS=\"-with-openssl=/opt/openssl\" pyenv install -v ${{ matrix.python-version }}"
docker exec slurm bash -c "pyenv global ${{ matrix.python-version }}"
docker exec slurm bash -c "pip install --upgrade pip && pip install 'numpy<2' -e /pydra[test,psij] && python -c 'import pydra.engine; print(pydra.utils.__version__)'"
- name: Run pytest
run: |
docker exec slurm bash -c "cd /pydra; pytest pydra/workers/tests/test_worker.py --only-worker=slurm --color=yes"
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v5
with:
fail_ci_if_error: true
token: ${{ secrets.CODECOV_TOKEN }}
# test-sge:
# needs: ['build']
# strategy:
# matrix:
# python-version: [3.11.5]
# fail-fast: false
# runs-on: ubuntu-latest
# env:
# VERSION: 1.8.1
# steps:
# - name: Disable etelemetry
# run: echo "NO_ET=TRUE" >> $GITHUB_ENV
# - uses: actions/checkout@v4
# - name: Fetch tags
# run: git fetch --prune --unshallow
# - name: Set up Python ${{ matrix.python-version }}
# uses: actions/setup-python@v5
# with:
# python-version: ${{ matrix.python-version }}
# - name: Install SGE
# run: |
# mkdir -p /sge-build
# cd /sge-build
# wget -c https://github.com/sge-network/sge/archive/refs/tags/v${{ env.VERSION }}.tar.gz
# tar zxvf v${{ env.VERSION }}.tar.gz
# cd v${{ env.VERSION }}
# mkdir -p /opt/sge
# useradd -r -m -U -d /home/sgeadmin -s /bin/bash -c "Docker SGE Admin" sgeadmin
# usermod -a -G sudo sgeadmin
# sh scripts/bootstrap.sh && ./aimk -no-qmon -no-qtcsh && ./aimk -man
# echo Y | ./scripts/distinst -local -allall -libs -noexit
# cd /opt/sge
# touch bin/lx-amd64/{qmon,qtcsh}
# ln -s /opt/sge/default/common/settings.sh /etc/profile.d/sge_settings.sh
# echo . /etc/profile.d/sge_settings.sh >> /etc/bash.bashrc
# chmod a+x /root/boot-sge.sh
# apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# - name: Boot SGE
# run: |
# cd /opt/sge
# ./inst_sge -m -x -s -auto util/install_modules/inst_template.conf
# . /etc/profile.d/sge_settings.sh
# cd $HOME
# qconf -as `hostname`
# qconf -mattr queue shell_start_mode unix_behavior all.q
# update_conf() {
# TMPF=`mktemp`
# cat > $TMPF <<EOF
# #!/bin/sh
# sed -ri "$1" \$1
# EOF
# chmod a+x $TMPF
# EDITOR="$TMPF" qconf -m$2
# rm -f $TMPF
# }
# # allow root to qsub (yes, it's a security hole but it simplifies the container)
# update_conf "/min_/s/100/0/; s/posix_compliant/unix_behavior/" conf
# # shrink scheduling interval for faster response
# update_conf "/schedule_interval/s/[0-9]+:[0-9]+:[0-9]+/0:0:2/" sconf
# # finalize
# if [ -z "$1" ]; then
# exec /bin/bash
# else
# exec "$@"
# fi
# - name: Install Pydra
# run: |
# pip install --upgrade pip && pip install -e .[test] && python -c 'import pydra.engine; print(pydra.utils.__version__)'
# - name: Run pytest
# run: |
# pytest pydra/workers/tests/test_worker.py --import-mode=importlib --rootdir . --only-worker=sge --color=yes -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml
# - name: Upload coverage to Codecov
# uses: codecov/codecov-action@v5
# with:
# fail_ci_if_error: true
# token: ${{ secrets.CODECOV_TOKEN }}
# files: ./cov.xml
build-docs:
runs-on: ubuntu-latest
# Set up the environment so that it finds conda
defaults:
run:
shell: bash -l {0}
steps:
- name: Install Pandoc for NBSphinx and graphviz for workflow plotting (dot)
run: |
sudo apt-get update
sudo apt-get install -y pandoc graphviz
- name: Install Dependencies for virtual notifications in Adv.-Exec Tutorial
run: |
sudo apt update
sudo apt install -y xvfb libnotify-bin dbus-x11 xfce4-notifyd
- name: Start Virtual Display (for notifications)
run: |
Xvfb :99 &
export DISPLAY=:99
eval "$(dbus-launch --sh-syntax)"
echo "DISPLAY=:99" >> $GITHUB_ENV
echo "DBUS_SESSION_BUS_ADDRESS=$DBUS_SESSION_BUS_ADDRESS" >> $GITHUB_ENV
- name: Start Notification Daemon (for notifications)
run: |
xfce4-notifyd &
sleep 2 # Give it some time to start
- name: Send Notification (test notifications)
run: |
notify-send "GitHub Runner Notification" "This is a test notification from GitHub Actions"
- name: Debug Running Processes (for notifications)
run: |
ps aux | grep notify
ps aux | grep xfce4-notifyd
dbus-monitor --session &
sleep 3
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Install Minconda
uses: conda-incubator/setup-miniconda@v3
with:
auto-activate-base: true
activate-environment: ""
# We generate dummy aliases so they can be deleted by conda later without throwing an error
- name: Install MRtrix via Conda
run: |
alias shview='ls'
alias mrview='ls'
conda install -c conda-forge -c MRtrix3 mrtrix3 libstdcxx-ng
mrconvert --version
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.x'
- name: Install package
run: pip install .[doc]
- name: Install Python3 kernel
run: python -m ipykernel install --user
- name: Build docs
run: make -C docs html
- uses: actions/upload-artifact@v4
with:
name: docs
path: docs/build/html
deploy:
needs: [build, build-docs, test, test-singularity, test-slurm]
runs-on: ubuntu-latest
if: github.event_name == 'release'
permissions:
attestations: write
id-token: write
steps:
- name: Download dist
uses: actions/download-artifact@v4
with:
name: Packages
path: dist
- name: Upload to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
deploy-docs:
needs: [build-docs, deploy]
runs-on: ubuntu-latest
if: github.event_name == 'release'
steps:
- name: Download docs
uses: actions/download-artifact@v4
with:
name: docs
path: docs-build
- name: Deploy Docs to GitHub Pages
uses: peaceiris/actions-gh-pages@v4
with:
github_token: ${{ secrets.GHPAGES_DEPLOY_KEY }}
publish_dir: docs-build