Skip to content

Commit 1d1fd5a

Browse files
nickfrasernickfraser
authored
ci (periodic): Added periodic tests for the entire suite of supported versions (#1368)
1 parent 216101e commit 1d1fd5a

File tree

8 files changed

+502
-3
lines changed

8 files changed

+502
-3
lines changed

‎.github/workflows/base_periodic.yml.template‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
name: &{name}
2+
3+
on:
4+
schedule:
5+
- cron: '37 19 * * 5' # 7:37am UTC every Friday
6+
7+
jobs:
8+
build:
9+
runs-on: ${{ matrix.platform }}
10+
strategy:
11+
&{strategy}
12+
matrix:
13+
&{matrix}
14+
&{exclude}
15+
16+
steps:
17+
18+
- name: Checkout repo
19+
uses: actions/checkout@v3
20+
21+
- name: Set up Python
22+
uses: actions/setup-python@v2
23+
with:
24+
python-version: ${{ matrix.python_version }}
25+
26+
- name: Install Nox dependencies
27+
shell: bash
28+
run: pip install -r requirements/requirements-nox.txt
29+
30+
- name: Install update
31+
shell: bash
32+
run: sudo apt-get update
33+
if: startsWith(runner.os, 'Linux') == true
34+
35+
- name: Install libsndfile and libgomp1 on Ubuntu
36+
shell: bash
37+
run: sudo apt-get install -y libsndfile-dev libgomp1
38+
if: startsWith(runner.os, 'Linux') == true
39+
40+
- name: Install libomp on macOS
41+
shell: bash
42+
run: brew install libomp
43+
if: startsWith(runner.os, 'macOS') == true
44+
45+
&{steps}

‎.github/workflows/gen_github_actions.py‎

Lines changed: 71 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@
55
from utils import generate_exclusion_list
66

77
TORCHVISION_VERSION_DICT = {
8+
'1.9.1': '0.10.1',
9+
'1.10.1': '0.11.2',
10+
'1.11.0': '0.12.0',
11+
'1.12.1': '0.13.1',
812
'1.13.1': '0.14.1',
913
'2.0.1': '0.15.2',
1014
'2.1.1': '0.16.1',
@@ -13,10 +17,13 @@
1317
'2.4.1': '0.19.1',
1418
'2.5.1': '0.20.1',
1519
'2.6.0': '0.21.0',
16-
'2.7.1': '0.22.1'}
20+
'2.7.1': '0.22.1',
21+
'2.8.0': '0.23.0',
22+
'2.9.0': '0.24.0'}
1723

1824
BASE_YML_TEMPLATE = 'base.yml.template'
1925
BASE_YML_REDUCED_TEMPLATE = 'base_reduced.yml.template'
26+
BASE_YML_PERIODIC_TEMPLATE = 'base_periodic.yml.template'
2027
PYTEST_YML = 'pytest.yml'
2128
EXAMPLES_PYTEST_YML = 'examples_pytest.yml'
2229
EXAMPLES_LLM_PYTEST_YML = 'examples_llm_pytest.yml'
@@ -59,6 +66,30 @@
5966
'jit_disabled',])])
6067

6168
# Data shared betwen Nox sessions and Github Actions, formatted as tuples
69+
ALL_SUPPORTED_PYTHON_VERSIONS = ('3.9', '3.10', '3.11', '3.12')
70+
71+
ALL_SUPPORTED_PYTORCH_VERSIONS = (
72+
'1.12.1',
73+
'1.13.1',
74+
'2.0.1',
75+
'2.1.1',
76+
'2.2.2',
77+
'2.3.1',
78+
'2.4.1',
79+
'2.5.1',
80+
'2.6.0',
81+
'2.7.1',
82+
'2.8.0',
83+
'2.9.0')
84+
ALL_SUPPORTED_EXCLUSION_LIST = generate_exclusion_list([
85+
[['python_version', [
86+
'3.9',]], ['pytorch_version', [
87+
'2.9.0',]]],
88+
[['python_version', [
89+
'3.11',]], ['pytorch_version', ['1.12.1', '1.13.1']]],
90+
[['python_version', [
91+
'3.12',]], ['pytorch_version', ['1.12.1', '1.13.1', '2.0.1', '2.1.1']]],])
92+
6293
PYTHON_VERSIONS = ('3.10', '3.11')
6394

6495
PYTORCH_VERSIONS = ('2.2.2', '2.3.1', '2.4.1', '2.5.1', '2.6.0', '2.7.1')
@@ -83,6 +114,10 @@
83114
MATRIX = od([('python_version', list(PYTHON_VERSIONS)), ('pytorch_version', list(PYTORCH_VERSIONS)),
84115
('platform', PLATFORM_LIST)])
85116

117+
MATRIX_ALL_SUPPORTED = od([('python_version', list(ALL_SUPPORTED_PYTHON_VERSIONS)),
118+
('pytorch_version', list(ALL_SUPPORTED_PYTORCH_VERSIONS)),
119+
('platform', PLATFORM_LIST)])
120+
86121
EXAMPLES_LLM_PYTEST_PYTORCH_VERSIONS = ('2.6.0', '2.7.1')
87122
EXAMPLES_LLM_PYTEST_MATRIX = od([('python_version', list(PYTHON_VERSIONS)),
88123
('pytorch_version', list(EXAMPLES_LLM_PYTEST_PYTORCH_VERSIONS)),
@@ -218,6 +253,13 @@ def gen_pytest_yml():
218253
PYTEST_STEP_LIST,
219254
STRATEGY)
220255
pytest.gen_yaml(BASE_YML_REDUCED_TEMPLATE, 'reduced_' + PYTEST_YML)
256+
pytest = Action(
257+
'Pytest',
258+
EXCLUDE_LIST + ALL_SUPPORTED_EXCLUSION_LIST,
259+
combine_od_list([MATRIX_ALL_SUPPORTED, PYTEST_MATRIX_EXTRA_REDUCED]),
260+
PYTEST_STEP_LIST,
261+
STRATEGY)
262+
pytest.gen_yaml(BASE_YML_PERIODIC_TEMPLATE, 'periodic_' + PYTEST_YML)
221263

222264

223265
def gen_examples_pytest_yml():
@@ -295,6 +337,13 @@ def gen_test_develop_install_yml():
295337
test_develop_install = Action(
296338
'Test develop install', EXCLUDE_LIST, MATRIX_REDUCED, TEST_INSTALL_DEV_STEP_LIST, STRATEGY)
297339
test_develop_install.gen_yaml(BASE_YML_REDUCED_TEMPLATE, 'reduced_' + DEVELOP_INSTALL_YML)
340+
test_develop_install = Action(
341+
'Test develop install',
342+
EXCLUDE_LIST + ALL_SUPPORTED_EXCLUSION_LIST,
343+
MATRIX_ALL_SUPPORTED,
344+
TEST_INSTALL_DEV_STEP_LIST,
345+
STRATEGY)
346+
test_develop_install.gen_yaml(BASE_YML_PERIODIC_TEMPLATE, 'periodic_' + DEVELOP_INSTALL_YML)
298347

299348

300349
def gen_test_brevitas_finn_integration():
@@ -325,6 +374,13 @@ def gen_test_brevitas_ort_integration():
325374
ORT_INTEGRATION_STEP_LIST,
326375
STRATEGY)
327376
test_ort_integration.gen_yaml(BASE_YML_REDUCED_TEMPLATE, 'reduced_' + ORT_INTEGRATION_YML)
377+
test_ort_integration = Action(
378+
'Test Brevitas-ORT integration',
379+
EXCLUDE_LIST + ALL_SUPPORTED_EXCLUSION_LIST,
380+
MATRIX_ALL_SUPPORTED,
381+
ORT_INTEGRATION_STEP_LIST,
382+
STRATEGY)
383+
test_ort_integration.gen_yaml(BASE_YML_PERIODIC_TEMPLATE, 'periodic_' + ORT_INTEGRATION_YML)
328384

329385

330386
def gen_test_brevitas_notebook():
@@ -342,6 +398,13 @@ def gen_test_brevitas_notebook():
342398
NOTEBOOK_STEP_LIST,
343399
STRATEGY)
344400
tests_brevitas_notebooks.gen_yaml(BASE_YML_REDUCED_TEMPLATE, 'reduced_' + NOTEBOOK_YML)
401+
tests_brevitas_notebooks = Action(
402+
'Test Notebook execution',
403+
EXCLUDE_LIST + ALL_SUPPORTED_EXCLUSION_LIST + NOTEBOOK_EXCLUDE_LIST,
404+
MATRIX_ALL_SUPPORTED,
405+
NOTEBOOK_STEP_LIST,
406+
STRATEGY)
407+
tests_brevitas_notebooks.gen_yaml(BASE_YML_PERIODIC_TEMPLATE, 'periodic_' + NOTEBOOK_YML)
345408

346409

347410
def gen_test_brevitas_end_to_end():
@@ -359,6 +422,13 @@ def gen_test_brevitas_end_to_end():
359422
ENDTOEND_STEP_LIST,
360423
STRATEGY)
361424
tests_brevitas_end_to_end.gen_yaml(BASE_YML_REDUCED_TEMPLATE, 'reduced_' + ENDTOEND_YML)
425+
tests_brevitas_end_to_end = Action(
426+
'Test End-to-end flows',
427+
EXCLUDE_LIST + ALL_SUPPORTED_EXCLUSION_LIST + END_TO_END_EXCLUDE_LIST,
428+
MATRIX_ALL_SUPPORTED,
429+
ENDTOEND_STEP_LIST,
430+
STRATEGY)
431+
tests_brevitas_end_to_end.gen_yaml(BASE_YML_PERIODIC_TEMPLATE, 'periodic_' + ENDTOEND_YML)
362432

363433

364434
if __name__ == '__main__':

‎.github/workflows/periodic_develop_install.yml‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
name: Test develop install
2+
3+
on:
4+
schedule:
5+
- cron: '37 19 * * 5' # 7:37am UTC every Friday
6+
7+
jobs:
8+
build:
9+
runs-on: ${{ matrix.platform }}
10+
strategy:
11+
fail-fast: false
12+
13+
14+
matrix:
15+
python_version: ['3.9', '3.10', '3.11', '3.12']
16+
pytorch_version: ['1.12.1', '1.13.1', '2.0.1', '2.1.1', '2.2.2', '2.3.1', '2.4.1', '2.5.1', '2.6.0', '2.7.1', '2.8.0', '2.9.0']
17+
platform: ['windows-latest', 'ubuntu-latest', 'macos-latest']
18+
19+
20+
exclude:
21+
- python_version: '3.9'
22+
pytorch_version: '2.9.0'
23+
24+
- python_version: '3.11'
25+
pytorch_version: '1.12.1'
26+
27+
- python_version: '3.11'
28+
pytorch_version: '1.13.1'
29+
30+
- python_version: '3.12'
31+
pytorch_version: '1.12.1'
32+
33+
- python_version: '3.12'
34+
pytorch_version: '1.13.1'
35+
36+
- python_version: '3.12'
37+
pytorch_version: '2.0.1'
38+
39+
- python_version: '3.12'
40+
pytorch_version: '2.1.1'
41+
42+
43+
44+
steps:
45+
46+
- name: Checkout repo
47+
uses: actions/checkout@v3
48+
49+
- name: Set up Python
50+
uses: actions/setup-python@v2
51+
with:
52+
python-version: ${{ matrix.python_version }}
53+
54+
- name: Install Nox dependencies
55+
shell: bash
56+
run: pip install -r requirements/requirements-nox.txt
57+
58+
- name: Install update
59+
shell: bash
60+
run: sudo apt-get update
61+
if: startsWith(runner.os, 'Linux') == true
62+
63+
- name: Install libsndfile and libgomp1 on Ubuntu
64+
shell: bash
65+
run: sudo apt-get install -y libsndfile-dev libgomp1
66+
if: startsWith(runner.os, 'Linux') == true
67+
68+
- name: Install libomp on macOS
69+
shell: bash
70+
run: brew install libomp
71+
if: startsWith(runner.os, 'macOS') == true
72+
73+
- name: Run Nox session for testing brevitas develop install and imports
74+
shell: bash
75+
run: nox -v -s tests_brevitas_install_dev-${{ matrix.python_version }}\(\pytorch_${{ matrix.pytorch_version }}\)
76+
77+
- name: Run Nox session for testing brevitas_examples develop install and imports
78+
shell: bash
79+
run: nox -v -s tests_brevitas_examples_install_dev-${{ matrix.python_version }}\(\pytorch_${{ matrix.pytorch_version }}\)

‎.github/workflows/periodic_end_to_end.yml‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
name: Test End-to-end flows
2+
3+
on:
4+
schedule:
5+
- cron: '37 19 * * 5' # 7:37am UTC every Friday
6+
7+
jobs:
8+
build:
9+
runs-on: ${{ matrix.platform }}
10+
strategy:
11+
fail-fast: false
12+
13+
14+
matrix:
15+
python_version: ['3.9', '3.10', '3.11', '3.12']
16+
pytorch_version: ['1.12.1', '1.13.1', '2.0.1', '2.1.1', '2.2.2', '2.3.1', '2.4.1', '2.5.1', '2.6.0', '2.7.1', '2.8.0', '2.9.0']
17+
platform: ['windows-latest', 'ubuntu-latest', 'macos-latest']
18+
19+
20+
exclude:
21+
- python_version: '3.9'
22+
pytorch_version: '2.9.0'
23+
24+
- python_version: '3.11'
25+
pytorch_version: '1.12.1'
26+
27+
- python_version: '3.11'
28+
pytorch_version: '1.13.1'
29+
30+
- python_version: '3.12'
31+
pytorch_version: '1.12.1'
32+
33+
- python_version: '3.12'
34+
pytorch_version: '1.13.1'
35+
36+
- python_version: '3.12'
37+
pytorch_version: '2.0.1'
38+
39+
- python_version: '3.12'
40+
pytorch_version: '2.1.1'
41+
42+
- platform: 'windows-latest'
43+
44+
45+
46+
steps:
47+
48+
- name: Checkout repo
49+
uses: actions/checkout@v3
50+
51+
- name: Set up Python
52+
uses: actions/setup-python@v2
53+
with:
54+
python-version: ${{ matrix.python_version }}
55+
56+
- name: Install Nox dependencies
57+
shell: bash
58+
run: pip install -r requirements/requirements-nox.txt
59+
60+
- name: Install update
61+
shell: bash
62+
run: sudo apt-get update
63+
if: startsWith(runner.os, 'Linux') == true
64+
65+
- name: Install libsndfile and libgomp1 on Ubuntu
66+
shell: bash
67+
run: sudo apt-get install -y libsndfile-dev libgomp1
68+
if: startsWith(runner.os, 'Linux') == true
69+
70+
- name: Install libomp on macOS
71+
shell: bash
72+
run: brew install libomp
73+
if: startsWith(runner.os, 'macOS') == true
74+
75+
- name: Run Nox session for end-to-end flows
76+
shell: bash
77+
run: nox -v -s tests_brevitas_end_to_end-${{ matrix.python_version }}\(\pytorch_${{ matrix.pytorch_version }}\)

0 commit comments

Comments
 (0)