|
| 1 | +name: Create caches for ephy_testing_data and conda env |
| 2 | + |
| 3 | +on: |
| 4 | + workflow_dispatch: # Workflow can be triggered manually via GH actions webinterface |
| 5 | + push: # When something is pushed into master this checks if caches need to re-created |
| 6 | + branches: |
| 7 | + - master |
| 8 | + schedule: |
| 9 | + - cron: "0 12 * * *" # Daily at noon UTC |
| 10 | + |
| 11 | +jobs: |
| 12 | + |
| 13 | + create-conda-env-cache-if-missing: |
| 14 | + name: Caching conda env |
| 15 | + runs-on: "ubuntu-latest" |
| 16 | + strategy: |
| 17 | + fail-fast: true |
| 18 | + defaults: |
| 19 | + # by default run in bash mode (required for conda usage) |
| 20 | + run: |
| 21 | + shell: bash -l {0} |
| 22 | + steps: |
| 23 | + - uses: actions/checkout@v3 |
| 24 | + |
| 25 | + - name: Get current year-month |
| 26 | + id: date |
| 27 | + run: | |
| 28 | + echo "date=$(date +'%Y-%m')" >> $GITHUB_OUTPUT |
| 29 | +
|
| 30 | + - name: Get current dependencies hash |
| 31 | + id: dependencies |
| 32 | + run: | |
| 33 | + echo "hash=${{hashFiles('**/pyproject.toml', '**/environment_testing.yml')}}" >> $GITHUB_OUTPUT |
| 34 | +
|
| 35 | + - uses: actions/cache@v3 |
| 36 | + # the cache for python package is reset: |
| 37 | + # * every month |
| 38 | + # * when package dependencies change |
| 39 | + id: cache-conda-env |
| 40 | + with: |
| 41 | + path: /usr/share/miniconda/envs/neo-test-env |
| 42 | + key: ${{ runner.os }}-conda-env-${{ steps.dependencies.outputs.hash }}-${{ steps.date.outputs.date }} |
| 43 | + |
| 44 | + - name: Cache found? |
| 45 | + run: echo "Cache-hit == ${{steps.cache-conda-env.outputs.cache-hit == 'true'}}" |
| 46 | + |
| 47 | + # activate environment if not restored from cache |
| 48 | + - uses: conda-incubator/setup-miniconda@v2 |
| 49 | + if: steps.cache-conda-env.outputs.cache-hit != 'true' |
| 50 | + with: |
| 51 | + activate-environment: neo-test-env |
| 52 | + python-version: 3.9 |
| 53 | + |
| 54 | + - name: Create the conda environment to be cached |
| 55 | + if: steps.cache-conda-env.outputs.cache-hit != 'true' |
| 56 | + # create conda env, configure git and install pip, neo and test dependencies from master |
| 57 | + # for PRs that change dependencies, this environment will be updated in the test workflow |
| 58 | + run: | |
| 59 | + conda env update neo-test-env --file environment_testing.yml |
| 60 | + git config --global user.email "neo_ci@fake_mail.com" |
| 61 | + git config --global user.name "neo CI" |
| 62 | + python -m pip install -U pip # Official recommended way |
| 63 | + pip install --upgrade -e . |
| 64 | + pip install .[test] |
| 65 | +
|
| 66 | + create-data-cache-if-missing: |
| 67 | + name: Caching data env |
| 68 | + runs-on: "ubuntu-latest" |
| 69 | + steps: |
| 70 | + |
| 71 | + - name: Get current hash (SHA) of the ephy_testing_data repo |
| 72 | + id: ephy_testing_data |
| 73 | + run: | |
| 74 | + echo "dataset_hash=$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" >> $GITHUB_OUTPUT |
| 75 | +
|
| 76 | + - uses: actions/cache@v3 |
| 77 | + # Loading cache of ephys_testing_dataset |
| 78 | + id: cache-datasets |
| 79 | + with: |
| 80 | + path: ~/ephy_testing_data |
| 81 | + key: ${{ runner.os }}-datasets-${{ steps.ephy_testing_data.outputs.dataset_hash }} |
| 82 | + |
| 83 | + - name: Cache found? |
| 84 | + run: echo "Cache-hit == ${{steps.cache-datasets.outputs.cache-hit == 'true'}}" |
| 85 | + |
| 86 | + - name: Installing datalad and git-annex |
| 87 | + if: steps.cache-datasets.outputs.cache-hit != 'true' |
| 88 | + run: | |
| 89 | + git config --global user.email "neo_ci@fake_mail.com" |
| 90 | + git config --global user.name "neo CI" |
| 91 | + python -m pip install -U pip # Official recommended way |
| 92 | + pip install datalad-installer |
| 93 | + datalad-installer --sudo ok git-annex --method datalad/packages |
| 94 | + pip install datalad |
| 95 | + git config --global filter.annex.process "git-annex filter-process" # recommended for efficiency |
| 96 | +
|
| 97 | + - name: Download dataset |
| 98 | + if: steps.cache-datasets.outputs.cache-hit != 'true' |
| 99 | + # Download repository and also fetch data |
| 100 | + run: | |
| 101 | + cd ~ |
| 102 | + datalad install --recursive --get-data https://gin.g-node.org/NeuralEnsemble/ephy_testing_data |
| 103 | +
|
| 104 | + - name: Show size of the cache to assert data is downloaded |
| 105 | + run: | |
| 106 | + cd ~ |
| 107 | + pwd |
| 108 | + du -hs ephy_testing_data |
| 109 | + cd ephy_testing_data |
| 110 | + pwd |
0 commit comments