Skip to content
CI/CD

Implementation of proposed syntax changes #50

Sign in to view logs

Implementation of proposed syntax changes

Implementation of proposed syntax changes #50

Workflow file for this run

.github/workflows/ci-cd.yml at 2383d1d
# This workflows will upload a Python Package using Twine when a release is created
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
name: CI/CD
on:
release:
types: [published]
push:
branches:
- master
pull_request:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-python@v5
with:
python-version: 3
- run: pip install --upgrade build twine
- run: python -m build
- run: twine check dist/*
- uses: actions/upload-artifact@v4
with:
name: dist
path: dist/
- name: Build archive
run: |
git clean -fxd
mkdir archive
git archive -o archive/pydra.zip HEAD
- uses: actions/upload-artifact@v4
with:
name: archive
path: archive/
test:
needs: ['build']
strategy:
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
python-version: ['3.11', '3.12', '3.13']
fail-fast: false
runs-on: ${{ matrix.os }}
steps:
- name: Fetch repository
uses: actions/checkout@v4
- name: Fetch tags
run: git fetch --prune --unshallow
- name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Update pip
run: python -m pip install --upgrade pip
- name: Install Pydra
run: pip install .[test]
- name: Print version
run: python -c "import pydra.engine; print(pydra.engine.__version__)"
- name: Disable etelemetry
run: echo "NO_ET=TRUE" >> $GITHUB_ENV
- name: Pytest
run: |
pytest -vs -n auto pydra --doctest-modules --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v2
with:
fail_ci_if_error: true
token: ${{ secrets.CODECOV_TOKEN }}
test-workers:
needs: ['build']
strategy:
matrix:
os: [ubuntu-latest]
python-version: ['3.11', '3.12', '3.13']
worker: ["psij-local", "dask"]
fail-fast: false
runs-on: ${{ matrix.os }}
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
repository: ${{ github.repository }}
- name: Fetch tags
run: git fetch --prune --unshallow
- name: Setup Python version ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install package
run: |
pip install -e ".[test,dask,psij]"
- name: Run tests for Dask
run: |
pytest -v pydra/engine/tests/test_submitter.py --only-worker=${{ matrix.worker }} --rootdir pydra --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v2
with:
fail_ci_if_error: true
token: ${{ secrets.CODECOV_TOKEN }}
test-singularity:
needs: ['build']
runs-on: ubuntu-22.04
strategy:
matrix:
python-version: ['3.11', '3.12', '3.13']
fail-fast: False
steps:
- name: Set env
run: |
echo "RELEASE_VERSION=v3.7.1" >> $GITHUB_ENV
echo "NO_ET=TRUE" >> $GITHUB_ENV
- name: Setup Singularity
uses: actions/checkout@v4
with:
repository: hpcng/singularity
ref: 'v3.7.1'
path: 'singularity'
- name: Setup GO
uses: actions/setup-go@v5
with:
go-version: '^1.13'
- name: Install OS deps
run: |
sudo apt-get update
sudo apt-get install flawfinder squashfs-tools uuid-dev libuuid1 libffi-dev libssl-dev libssl1.1 \
libarchive-dev libgpgme11-dev libseccomp-dev wget gcc make pkg-config -y
- name: Build
run: |
cd singularity
./mconfig --without-suid -p /usr/local/
make -C ./builddir
sudo make -C ./builddir install
cd ..
- name: Echo singularity version
run: |
echo ${{ github.ref }}
singularity --version
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Update build tools
run: python -m pip install --upgrade pip
- name: Checkout Pydra repo
uses: actions/checkout@v4
with:
repository: ${{ github.repository }}
- name: Fetch tags
run: git fetch --prune --unshallow
- name: Install pydra (test)
run: pip install -e ".[test]"
- name: Pytest
run: pytest -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml pydra/engine/tests/test_singularity.py pydra/engine/tests/test_environments.py --rootdir pydra
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v2
with:
fail_ci_if_error: true
token: ${{ secrets.CODECOV_TOKEN }}
test-slurm:
needs: ['build']
strategy:
matrix:
python-version: [3.11.5]
worker: ["slurm", "psij-slurm"]
fail-fast: false
runs-on: ubuntu-latest
env:
DOCKER_IMAGE: adi611/docker-centos7-slurm:23.02.1
steps:
- name: Disable etelemetry
run: echo "NO_ET=TRUE" >> $GITHUB_ENV
- uses: actions/checkout@v4
- name: Fetch tags
run: git fetch --prune --unshallow
- name: Pull docker image
run: |
docker pull $DOCKER_IMAGE
# Have image running in the background
docker run `bash <(curl -s https://codecov.io/env)` -itd -h slurmctl --cap-add sys_admin -d --name slurm -v `pwd`:/pydra -e NO_ET=$NO_ET $DOCKER_IMAGE
- name: Display previous jobs with sacct
run: |
echo "Allowing ports/daemons time to start" && sleep 10
docker exec slurm bash -c "sacctmgr -i add account none,test Cluster=linux Description='none' Organization='none'"
docker exec slurm bash -c "sacct && sinfo && squeue" 2&> /dev/null
if [ $? -ne 0 ]; then
echo "Slurm docker image error"
exit 1
fi
- name: Setup Python
run: |
docker exec slurm bash -c "echo $NO_ET"
docker exec slurm bash -c "ls -la && echo list top level dir"
docker exec slurm bash -c "ls -la /pydra && echo list pydra dir"
if [[ "${{ matrix.python-version }}" == "3.11.5" ]]; then
docker exec slurm bash -c "CONFIGURE_OPTS=\"-with-openssl=/opt/openssl\" pyenv install -v 3.11.5"
fi
docker exec slurm bash -c "pyenv global ${{ matrix.python-version }}"
docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test,psij] && python -c 'import pydra.engine; print(pydra.engine.__version__)'"
- name: Run pytest
run: |
docker exec slurm bash -c "pytest /pydra/pydra/engine/tests/test_submitter.py --rootdir /pydra/pydra --only-worker=${{ matrix.worker }} --color=yes -vs --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml"
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v2
with:
fail_ci_if_error: true
token: ${{ secrets.CODECOV_TOKEN }}
files: ./cov.xml
# test-sge:
# needs: ["build"]
# strategy:
# matrix:
# python-version: [3.11]
# fail-fast: false
# runs-on: ubuntu-latest
# steps:
# - name: Install system packages
# run: |
# export DEBIAN_FRONTEND=noninteractive
# sudo apt update
# sudo apt install -y apt-get install gridengine-master gridengine-client gridengine-exec gridengine-qmon cpp
# - name: Configuring SGE
# run: |
# # Get hostname
# HOSTNAME=$(hostname)
# # Set act_qmaster manually
# echo "$HOSTNAME" | sudo tee /var/lib/gridengine/default/common/act_qmaster
# # Add execution host (auto-generate config file)
# EXEC_HOST_FILE=/tmp/exec_host.conf
# cat <<EOF > "$EXEC_HOST_FILE"
# hostname $HOSTNAME
# load_scaling NONE
# complex_values NONE
# user_lists NONE
# xuser_lists NONE
# projects NONE
# department NONE
# use_start_proc FALSE
# queue_sort_method load
# EOF
# sudo qconf -Ae "$EXEC_HOST_FILE"
# # Add queue (auto-generate config file)
# QUEUE_FILE=/tmp/queue.conf
# cat <<EOF > "$QUEUE_FILE"
# qname test.q
# hostlist $HOSTNAME
# seq_no 0
# load_thresholds np_load_avg=1.75
# suspend_thresholds NONE
# nsuspend 1
# suspend_interval 00:05:00
# priority 0
# min_cpu_interval 00:05:00
# processors UNDEFINED
# qtype BATCH INTERACTIVE
# ckpt_list NONE
# pe_list make
# rerun FALSE
# slots 2
# tmpdir /tmp
# shell /bin/bash
# prolog NONE
# epilog NONE
# shell_start_mode posix_compliant
# starter_method NONE
# suspend_method NONE
# resume_method NONE
# terminate_method NONE
# notify 00:00:60
# owner_list NONE
# user_lists NONE
# xuser_lists NONE
# subordinate_list NONE
# complex_values NONE
# projects NONE
# xprojects NONE
# calendar NONE
# initial_state default
# s_rt INFINITY
# h_rt INFINITY
# s_cpu INFINITY
# h_cpu INFINITY
# s_fsize INFINITY
# h_fsize INFINITY
# s_data INFINITY
# h_data INFINITY
# s_stack INFINITY
# h_stack INFINITY
# s_core INFINITY
# h_core INFINITY
# s_rss INFINITY
# h_rss INFINITY
# EOF
# sudo qconf -Aq "$QUEUE_FILE"
# - name: Restart services
# run: |
# sudo service gridengine-master restart
# sudo service gridengine-exec restart
# - name: Confirm setup
# run: qhost -q
# - name: Checkout repository
# uses: actions/checkout@v4
# with:
# repository: ${{ github.repository }}
# - name: Fetch tags
# run: git fetch --prune --unshallow
# - name: Setup Python version ${{ matrix.python-version }}
# uses: actions/setup-python@v5
# with:
# python-version: ${{ matrix.python-version }}
# - name: Install package
# run: |
# pip install -e ".[test]"
# - name: Run tests for Dask
# run: |
# pytest -v pydra/engine/tests/test_submitter.py --only-worker=sge \
# --rootdir pydra --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml
# - name: Upload coverage to Codecov
# uses: codecov/codecov-action@v2
# with:
# fail_ci_if_error: true
# token: ${{ secrets.CODECOV_TOKEN }}
build-docs:
needs: ['build']
runs-on: ubuntu-latest
# Set up the environment so that it finds conda
defaults:
run:
shell: bash -l {0}
steps:
- name: Install Pandoc for NBSphinx
run: |
sudo apt-get update
sudo apt-get install -y pandoc
- name: Install Dependencies for virtual notifications in Adv.-Exec Tutorial
run: |
sudo apt update
sudo apt install -y xvfb libnotify-bin dbus-x11 xfce4-notifyd
- name: Start Virtual Display (for notifications)
run: |
Xvfb :99 &
export DISPLAY=:99
eval "$(dbus-launch --sh-syntax)"
echo "DISPLAY=:99" >> $GITHUB_ENV
echo "DBUS_SESSION_BUS_ADDRESS=$DBUS_SESSION_BUS_ADDRESS" >> $GITHUB_ENV
- name: Start Notification Daemon (for notifications)
run: |
xfce4-notifyd &
sleep 2 # Give it some time to start
- name: Send Notification (test notifications)
run: |
notify-send "GitHub Runner Notification" "This is a test notification from GitHub Actions"
- name: Debug Running Processes (for notifications)
run: |
ps aux | grep notify
ps aux | grep xfce4-notifyd
dbus-monitor --session &
sleep 3
- uses: actions/checkout@v4
- name: Fetch tags
run: git fetch --prune --unshallow
- name: Install Minconda
uses: conda-incubator/setup-miniconda@v3
with:
auto-activate-base: true
activate-environment: ""
- name: Install MRtrix via Conda
run: |
conda install -c mrtrix3 mrtrix3
mrconvert --version
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.x'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install build twine
- name: Install package
run: pip install .[doc]
- name: Install Python3 kernel
run: python -m ipykernel install --user
- name: Build docs
run: |
cd docs
make html
cd ..
- uses: actions/upload-artifact@v4
with:
name: docs
path: docs/build/html
deploy:
needs: [build-docs, test, test-workers, test-singularity, test-slurm]
runs-on: ubuntu-latest
steps:
- name: Download dist
uses: actions/download-artifact@v4
with:
name: dist
path: dist
- name: Check for PyPI token on tag
id: deployable
if: github.event_name == 'release'
env:
PYPI_API_TOKEN: "${{ secrets.PYPI_API_TOKEN }}"
run: if [ -n "$PYPI_API_TOKEN" ]; then echo "DEPLOY=true" >> $GITHUB_OUTPUT; fi
- name: Upload to PyPI
if: steps.deployable.outputs.DEPLOY
uses: pypa/gh-action-pypi-publish@release/v1
with:
user: __token__
password: ${{ secrets.PYPI_API_TOKEN }}
deploy-docs:
needs: [build-docs, deploy]
runs-on: ubuntu-latest
steps:
- name: Download docs
uses: actions/download-artifact@v4
with:
name: docs
path: docs-build
- name: Check for GHPAGES_DEPLOY_KEY token
id: deployable
if: github.event_name == 'release'
env:
GHPAGES_DEPLOY_KEY: "${{ secrets.GHPAGES_DEPLOY_KEY }}"
run: if [ -n "$GHPAGES_DEPLOY_KEY" ]; then echo "DEPLOY=true" >> $GITHUB_OUTPUT; fi
- name: Deploy Docs to GitHub Pages
if: steps.deployable.outputs.DEPLOY
uses: peaceiris/actions-gh-pages@v3
with:
github_token: ${{ secrets.GHPAGES_DEPLOY_KEY }}
publish_dir: docs-build