Skip to content

Commit 22a5c9f

Browse files
samuelgarciasamuelgarcia
authored
Merge pull request #1232 from JuliaSprenger/enh/CI_cache
Improve CI speed
2 parents 6a8c7d4 + e28480e commit 22a5c9f

File tree

3 files changed

+150
-10
lines changed

3 files changed

+150
-10
lines changed

‎.github/workflows/caches_cron_job.yml‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
name: Create caches for gin ecephys data and virtual env
2+
3+
on:
4+
workflow_dispatch: # Workflow can be triggered manually via GH actions webinterface
5+
push: # When something is pushed into master this checks if caches need to re-created
6+
branches:
7+
- master
8+
schedule:
9+
- cron: "0 12 * * *" # Daily at noon UTC
10+
11+
jobs:
12+
13+
create-conda-env-cache-if-missing:
14+
name: Caching conda env
15+
runs-on: "ubuntu-latest"
16+
strategy:
17+
fail-fast: true
18+
matrix:
19+
python-version: [ '3.9',]
20+
defaults:
21+
# by default run in bash mode (required for conda usage)
22+
run:
23+
shell: bash -l {0}
24+
steps:
25+
- uses: actions/checkout@v3
26+
27+
- name: Get current year-month
28+
id: date
29+
run: |
30+
echo "date=$(date +'%Y-%m')" >> $GITHUB_OUTPUT
31+
32+
- name: Get current dependencies hash
33+
id: dependencies
34+
run: |
35+
echo "hash=${{hashFiles('**/pyproject.toml', '**/environment_testing.yml')}}" >> $GITHUB_OUTPUT
36+
37+
- uses: actions/cache@v3
38+
# the cache for python package is reset:
39+
# * every month
40+
# * when package dependencies change
41+
id: cache-conda-env
42+
with:
43+
path: /usr/share/miniconda/envs/neo-test-env
44+
key: ${{ runner.os }}-conda-env-${{ steps.dependencies.outputs.hash }}-${{ steps.date.outputs.date }}
45+
46+
- name: Cache found?
47+
run: echo "Cache-hit == ${{steps.cache-conda-env.outputs.cache-hit == 'true'}}"
48+
49+
# activate environment if not restored from cache
50+
- uses: conda-incubator/setup-miniconda@v2
51+
if: steps.cache-venv.outputs.cache-hit != 'true'
52+
with:
53+
activate-environment: neo-test-env
54+
python-version: ${{ matrix.python-version }}
55+
56+
- name: Create the conda environment to be cached
57+
if: steps.cache-venv.outputs.cache-hit != 'true'
58+
# create conda env, configure git and install pip, neo and test dependencies from master
59+
# for PRs that change dependencies, this environment will be updated in the test workflow
60+
run: |
61+
conda env update neo-test-env --file environment_testing.yml
62+
git config --global user.email "neo_ci@fake_mail.com"
63+
git config --global user.name "neo CI"
64+
python -m pip install -U pip # Official recommended way
65+
pip install --upgrade -e .
66+
pip install .[test]
67+
68+
create-data-cache-if-missing:
69+
name: Caching data env
70+
runs-on: "ubuntu-latest"
71+
steps:
72+
73+
- name: Get current hash (SHA) of the ephy_testing_data repo
74+
id: ephy_testing_data
75+
run: |
76+
echo "dataset_hash=$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" >> $GITHUB_OUTPUT
77+
78+
- uses: actions/cache@v3
79+
# Loading cache of ephys_testing_dataset
80+
id: cache-datasets
81+
with:
82+
path: ~/ephy_testing_data
83+
key: ${{ runner.os }}-datasets-${{ steps.ephy_testing_data.outputs.dataset_hash }}
84+
85+
- name: Cache found?
86+
run: echo "Cache-hit == ${{steps.cache-datasets.outputs.cache-hit == 'true'}}"
87+
88+
- name: Installing datalad and git-annex
89+
if: steps.cache-datasets.outputs.cache-hit != 'true'
90+
run: |
91+
git config --global user.email "neo_ci@fake_mail.com"
92+
git config --global user.name "neo CI"
93+
python -m pip install -U pip # Official recommended way
94+
pip install datalad-installer
95+
datalad-installer --sudo ok git-annex --method datalad/packages
96+
pip install datalad
97+
git config --global filter.annex.process "git-annex filter-process" # recommended for efficiency
98+
99+
- name: Download dataset
100+
if: steps.cache-datasets.outputs.cache-hit != 'true'
101+
# Download repository and also fetch data
102+
run: |
103+
datalad install --recursive --get-data https://gin.g-node.org/NeuralEnsemble/ephy_testing_data
104+
105+
- name: Show size of the cache to assert data is downloaded
106+
run: |
107+
cd ~
108+
pwd
109+
du -hs ephy_testing_data
110+
cd ephy_testing_data
111+
pwd

‎.github/workflows/core-test.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,18 @@ on:
44
pull_request:
55
branches: [master]
66
types: [synchronize, opened, reopened, ready_for_review]
7+
paths:
8+
- 'neo/core/**'
9+
- 'pyproject.toml'
710

811
# run checks on any change of master, including merge of PRs
912
push:
1013
branches: [master]
1114

15+
concurrency: # Cancel previous workflows on the same pull request
16+
group: ${{ github.workflow }}-${{ github.ref }}
17+
cancel-in-progress: true
18+
1219
jobs:
1320
multi-os-python-numpy:
1421
runs-on: ${{ matrix.os }}

‎.github/workflows/io-test.yml‎

Lines changed: 32 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@ on:
99
push:
1010
branches: [master]
1111

12+
concurrency: # Cancel previous workflows on the same pull request
13+
group: ${{ github.workflow }}-${{ github.ref }}
14+
cancel-in-progress: true
1215

1316
jobs:
1417
build-and-test:
@@ -35,34 +38,44 @@ jobs:
3538

3639
- name: Get ephy_testing_data current head hash
3740
# the key depend on the last commit repo https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git
38-
id: ephy_testing_data_hash
41+
id: ephy_testing_data
3942
run: |
40-
echo "latest_hash=$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" >> $GITHUB_OUTPUT
43+
echo "dataset_hash=$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" >> $GITHUB_OUTPUT
4144
42-
- uses: actions/cache@v3
45+
- uses: actions/cache/restore@v3
4346
# Loading cache of ephys_testing_dataset
4447
id: cache-datasets
4548
with:
4649
path: ~/ephy_testing_data
47-
key: ${{ runner.os }}-datasets-${{ steps.ephy_testing_data_hash.outputs.latest_hash }}
50+
key: ${{ runner.os }}-datasets-${{ steps.ephy_testing_data.outputs.dataset_hash }}
51+
restore-keys: ${{ runner.os }}-datasets-
4852

4953
- uses: conda-incubator/setup-miniconda@v2
5054
with:
5155
activate-environment: neo-test-env
5256
python-version: ${{ matrix.python-version }}
53-
clean-patched-environment-file: false
5457

55-
- uses: actions/cache@v3
58+
- name: Get current dependencies hash
59+
id: dependencies
60+
run: |
61+
echo "hash=${{hashFiles('**/pyproject.toml', '**/environment_testing.yml')}}" >> $GITHUB_OUTPUT
62+
63+
- uses: actions/cache/restore@v3
5664
# the cache for python package is reset:
5765
# * every month
58-
# * when requirements/requirements_testing change
66+
# * when package dependencies change
5967
id: cache-conda-env
6068
with:
6169
path: /usr/share/miniconda/envs/neo-test-env
62-
key: ${{ runner.os }}-conda-env-${{ hashFiles('**/pyproject.toml') }}-${{ steps.date.outputs.date }}
70+
key: ${{ runner.os }}-conda-env-${{ steps.dependencies.outputs.hash }}-${{ steps.date.outputs.date }}
71+
# restore-keys match any key that starts with the restore-key
72+
restore-keys: |
73+
${{ runner.os }}-conda-env-${{ steps.dependencies.outputs.hash }}-
74+
${{ runner.os }}-conda-env-
6375
6476
- name: Install testing dependencies
65-
# testing environment is only installed if no cache was found
77+
# testing environment is only created from yml if no cache was found
78+
# restore-key hits should result in `cache-hit` == 'false'
6679
if: steps.cache-conda-env.outputs.cache-hit != 'true'
6780
run: |
6881
conda env update neo-test-env --file environment_testing.yml
@@ -72,11 +85,20 @@ jobs:
7285
git config --global user.email "neo_ci@fake_mail.com"
7386
git config --global user.name "neo CI"
7487
75-
- name: Install neo
88+
- name: Install neo including dependencies
89+
# installation with dependencies is only required if no cache was found
90+
# restore-key hits should result in `cache-hit` == 'false'
91+
if: steps.cache-conda-env.outputs.cache-hit != 'true'
7692
run: |
7793
pip install --upgrade -e .
7894
pip install .[test]
7995
96+
- name: Install neo without dependencies
97+
# only installing neo version to test as dependencies should be in cached conda env already
98+
if: steps.cache-conda-env.outputs.cache-hit == 'true'
99+
run: |
100+
pip install --no-dependencies -e .
101+
80102
- name: Test with pytest
81103
run: |
82104
# only neo.rawio and neo.io

0 commit comments

Comments
 (0)