Skip to content

Commit d959ef1

Browse files
microdev1microdev1
committed
use venv to cache python deps
1 parent 70a4efc commit d959ef1

File tree

12 files changed

+115
-66
lines changed

12 files changed

+115
-66
lines changed

‎.github/actions/external_deps/action.yml renamed to ‎.github/actions/deps/external/action.yml

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,18 @@ inputs:
1212
- riscv
1313
- none
1414

15+
apt:
16+
required: false
17+
default: true
18+
type: boolean
19+
20+
python:
21+
required: false
22+
default: true
23+
type: boolean
24+
1525
runs:
16-
using: "composite"
26+
using: composite
1727
steps:
1828
# aarch
1929
- name: Get aarch toolchain
@@ -74,8 +84,16 @@ runs:
7484
shell: bash
7585

7686
# common
87+
- name: Cache python dependencies
88+
if: inputs.python == 'true' && inputs.platform != 'espressif'
89+
uses: ./.github/actions/deps/python
90+
with:
91+
action: ${{ fromJSON('["restore", "cache"]')[github.job == 'scheduler'] }}
92+
- name: Install python dependencies
93+
if: inputs.python == 'true'
94+
run: pip install -r requirements-dev.txt
95+
shell: bash
7796
- name: Install dependencies
78-
run: |
79-
sudo apt-get install -y gettext
80-
pip install -r requirements-dev.txt
97+
if: inputs.apt == 'true'
98+
run: sudo apt-get install -y gettext
8199
shell: bash

‎.github/actions/port_deps/action.yml renamed to ‎.github/actions/deps/port/action.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ runs:
4343
uses: actions/cache@v3
4444
with:
4545
path: ${{ env.IDF_TOOLS_PATH }}
46-
key: tools-idf-${{ steps.idf-commit.outputs.commit }}-${{ runner.os }}-${{ env.pythonLocation }}
46+
key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-idf-${{ steps.idf-commit.outputs.commit }}
4747

4848
- name: Initialize IDF submodules
4949
if: inputs.platform == 'espressif'

‎.github/actions/deps/python/action.yml

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
name: Fetch python deps
2+
3+
inputs:
4+
action:
5+
description: The cache action to use
6+
required: false
7+
default: restore
8+
type: choice
9+
options:
10+
- cache
11+
- restore
12+
13+
runs:
14+
using: composite
15+
steps:
16+
- name: Cache python dependencies
17+
if: ${{ inputs.action == 'cache' }}
18+
uses: actions/cache@v3
19+
with:
20+
path: .cp_tools
21+
key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-cp-${{ hashFiles('requirements-dev.txt') }}
22+
23+
- name: Restore python dependencies
24+
if: ${{ inputs.action == 'restore' }}
25+
uses: actions/cache/restore@v3
26+
with:
27+
path: .cp_tools
28+
key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-cp-${{ hashFiles('requirements-dev.txt') }}
29+
30+
- name: Set up venv
31+
run: |
32+
python -m venv .cp_tools
33+
source .cp_tools/bin/activate
34+
echo >> $GITHUB_PATH "$PATH"
35+
shell: bash

‎.github/actions/fetch_submodules/action.yml renamed to ‎.github/actions/deps/submodules/action.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ inputs:
1212
default: '["extmod/ulab", "lib/", "tools/"]'
1313
type: string
1414

15-
cache:
15+
action:
1616
description: 'The cache action to use'
1717
required: false
1818
default: 'restore'
@@ -47,15 +47,15 @@ runs:
4747
shell: bash
4848

4949
- name: Cache submodules
50-
if: ${{ inputs.cache == 'cache' }}
50+
if: ${{ inputs.action == 'cache' }}
5151
uses: actions/cache@v3
5252
with:
5353
path: ".git/modules/\n${{ join(fromJSON(steps.create-submodule-status.outputs.submodules), '\n') }}"
5454
key: submodules-common-${{ hashFiles('submodule_status') }}
5555
enableCrossOsArchive: true
5656

5757
- name: Restore submodules
58-
if: ${{ inputs.cache == 'restore' }}
58+
if: ${{ inputs.action == 'restore' }}
5959
uses: actions/cache/restore@v3
6060
with:
6161
path: ".git/modules/\n${{ join(fromJSON(steps.create-submodule-status.outputs.submodules), '\n') }}"

‎.github/actions/mpy_cross/action.yml

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,36 @@
11
name: Set up mpy-cross
22

3+
inputs:
4+
download:
5+
required: false
6+
default: true
7+
type: boolean
8+
39
runs:
410
using: composite
511
steps:
612
- name: Download mpy-cross
713
id: download-mpy-cross
14+
if: inputs.download == 'true'
815
continue-on-error: true
916
uses: actions/download-artifact@v3
1017
with:
1118
name: mpy-cross
1219
path: mpy-cross
1320

1421
- name: Make mpy-cross executable
15-
if: steps.download-mpy-cross.outcome == 'success'
22+
if: inputs.download == 'true' && steps.download-mpy-cross.outcome == 'success'
1623
run: sudo chmod +x mpy-cross/mpy-cross
1724
shell: bash
1825

1926
- name: Build mpy-cross
20-
if: steps.download-mpy-cross.outcome == 'failure'
27+
if: inputs.download == 'false' || steps.download-mpy-cross.outcome == 'failure'
2128
run: make -C mpy-cross -j2
2229
shell: bash
2330

2431
- name: Upload mpy-cross
32+
if: inputs.download == 'false' || steps.download-mpy-cross.outcome == 'failure'
2533
continue-on-error: true
26-
if: steps.download-mpy-cross.outcome == 'failure'
2734
uses: actions/upload-artifact@v3
2835
with:
2936
name: mpy-cross

‎.github/workflows/build-boards.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,14 +35,14 @@ jobs:
3535
with:
3636
python-version: 3.x
3737
- name: Set up port
38-
uses: ./.github/actions/port_deps
38+
uses: ./.github/actions/deps/port
3939
with:
4040
platform: ${{ inputs.platform }}
4141
- name: Set up submodules
4242
id: set-up-submodules
43-
uses: ./.github/actions/fetch_submodules
43+
uses: ./.github/actions/deps/submodules
4444
- name: Set up external
45-
uses: ./.github/actions/external_deps
45+
uses: ./.github/actions/deps/external
4646
with:
4747
platform: ${{ inputs.platform }}
4848
- name: Set up mpy-cross
@@ -64,7 +64,7 @@ jobs:
6464
6565
- name: Set up build failure matcher
6666
run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
67-
- name: Build
67+
- name: Build board
6868
run: python3 -u build_release_files.py
6969
working-directory: tools
7070
env:

‎.github/workflows/build-mpy-cross.yml

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -27,43 +27,43 @@ jobs:
2727
with:
2828
submodules: false
2929
fetch-depth: 1
30-
3130
- name: Set up python
3231
uses: actions/setup-python@v4
3332
with:
3433
python-version: 3.x
35-
3634
- name: Set up submodules
37-
uses: ./.github/actions/fetch_submodules
35+
uses: ./.github/actions/deps/submodules
3836
with:
3937
target: mpy-cross
40-
41-
- name: Install dependencies
42-
run: |
43-
sudo apt-get update
44-
sudo apt-get install -y gettext
38+
- name: Set up external
39+
uses: ./.github/actions/deps/external
40+
with:
41+
python: false
4542

4643
- name: Install toolchain (aarch64)
4744
if: matrix.mpy-cross == 'static-aarch64'
4845
run: sudo apt-get install -y gcc-aarch64-linux-gnu
49-
5046
- name: Install toolchain (mingw)
5147
if: matrix.mpy-cross == 'static-mingw'
5248
run: sudo apt-get install -y mingw-w64
5349

5450
- name: Build mpy-cross.${{ matrix.mpy-cross }}
5551
run: make -C mpy-cross -j2 -f Makefile.${{ matrix.mpy-cross }}
5652

53+
- name: Set output
54+
run: |
55+
echo >> $GITHUB_ENV "EX=${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}"
56+
echo >> $GITHUB_ENV "OS=${{ env[format('OS_{0}', matrix.mpy-cross)] }}"
57+
5758
- name: Upload artifact
5859
uses: actions/upload-artifact@v3
5960
with:
60-
name: mpy-cross.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
61-
path: mpy-cross/mpy-cross.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
62-
61+
name: mpy-cross.${{ env.EX }}
62+
path: mpy-cross/mpy-cross.${{ env.EX }}
6363
- name: Upload to S3
6464
uses: ./.github/actions/upload_aws
6565
with:
66-
source: mpy-cross/mpy-cross.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
67-
destination: mpy-cross/${{ env[format('OS_{0}', matrix.mpy-cross)] }}/mpy-cross-${{ env.CP_VERSION }}.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
66+
source: mpy-cross/mpy-cross.${{ env.EX }}
67+
destination: mpy-cross/${{ env.OS }}/mpy-cross-${{ env.OS }}-${{ env.CP_VERSION }}.${{ env.EX }}
6868
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
6969
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

‎.github/workflows/build.yml

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,6 @@ jobs:
2727
boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }}
2828
boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
2929
cp-version: ${{ steps.set-up-submodules.outputs.version }}
30-
env:
31-
MICROPY_CPYTHON3: python3.8
32-
MICROPY_MICROPYTHON: ${{ github.workspace }}/ports/unix/micropython-coverage
3330
steps:
3431
- name: Dump GitHub context
3532
run: echo "$GITHUB_CONTEXT"
@@ -48,14 +45,12 @@ jobs:
4845
run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
4946
- name: Set up submodules
5047
id: set-up-submodules
51-
uses: ./.github/actions/fetch_submodules
48+
uses: ./.github/actions/deps/submodules
5249
with:
53-
cache: cache
50+
action: cache
5451
version: true
55-
- name: Install dependencies
56-
run: |
57-
sudo apt-get install -y gettext
58-
pip install requests requests-cache sh
52+
- name: Set up external
53+
uses: ./.github/actions/deps/external
5954
# Disabled: Needs to be updated
6055
# - name: Get last commit with checks
6156
# id: get-last-commit-with-checks
@@ -69,6 +64,8 @@ jobs:
6964
# EXCLUDE_COMMIT: ${{ github.event.after }}
7065
- name: Set up mpy-cross
7166
uses: ./.github/actions/mpy_cross
67+
with:
68+
download: false
7269
- name: Set head sha
7370
if: github.event_name == 'pull_request'
7471
run: echo "HEAD_SHA=$(git show -s --format=%s $GITHUB_SHA | grep -o -P "(?<=Merge ).*(?= into)")" >> $GITHUB_ENV
@@ -124,7 +121,7 @@ jobs:
124121
with:
125122
python-version: 3.x
126123
- name: Set up submodules
127-
uses: ./.github/actions/fetch_submodules
124+
uses: ./.github/actions/deps/submodules
128125
- name: Versions
129126
run: |
130127
gcc --version
@@ -179,7 +176,7 @@ jobs:
179176
with:
180177
python-version: 3.x
181178
- name: Set up submodules
182-
uses: ./.github/actions/fetch_submodules
179+
uses: ./.github/actions/deps/submodules
183180
- name: Install dependencies
184181
run: |
185182
sudo apt-get update

‎.github/workflows/create_website_pr.yml

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,15 @@ jobs:
2424
- name: Set up python
2525
uses: actions/setup-python@v4
2626
with:
27-
python-version: "3.x"
27+
python-version: 3.x
2828
- name: Set up submodules
29-
uses: ./.github/actions/fetch_submodules
29+
uses: ./.github/actions/deps/submodules
3030
with:
3131
version: true
32-
- name: Install dependencies
33-
run: pip install -r requirements-dev.txt
32+
- name: Set up external
33+
uses: ./.github/actions/deps/external
34+
with:
35+
apt: false
3436
- name: Versions
3537
run: |
3638
gcc --version

‎.github/workflows/ports_windows.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ jobs:
7777
fetch-depth: 1
7878

7979
- name: Set up submodules
80-
uses: ./.github/actions/fetch_submodules
80+
uses: ./.github/actions/deps/submodules
8181
with:
8282
version: true
8383

0 commit comments

Comments
 (0)