Skip to content

Commit 3acc39e

Browse files
ahalahal
committed
ci: switch unit tests to one task per Python version
1 parent ae78c11 commit 3acc39e

File tree

18 files changed

+214
-204
lines changed

18 files changed

+214
-204
lines changed

‎pyproject.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ extend-exclude = """(\
88
xfail_strict = true
99

1010
[tool.coverage.run]
11-
parallel = true
1211
branch = true
1312
source = ["src/taskgraph/", "taskgraph", "src/taskgraph/run-task/"]
1413

‎requirements/dev.txt‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,27 @@ cfgv==3.4.0 \
1010
--hash=sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9 \
1111
--hash=sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560
1212
# via pre-commit
13+
distlib==0.3.8 \
14+
--hash=sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784 \
15+
--hash=sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64
16+
# via virtualenv
17+
filelock==3.14.0 \
18+
--hash=sha256:43339835842f110ca7ae60f1e1c160714c5a6afd15a2873419ab185334975c0f \
19+
--hash=sha256:6ea72da3be9b8c82afd3edcf99f2fffbb5076335a5ae4d03248bb5b6c3eae78a
20+
# via virtualenv
1321
identify==2.5.36 \
1422
--hash=sha256:37d93f380f4de590500d9dba7db359d0d3da95ffe7f9de1753faa159e71e7dfa \
1523
--hash=sha256:e5e00f54165f9047fbebeb4a560f9acfb8af4c88232be60a488e9b68d122745d
1624
# via pre-commit
25+
platformdirs==4.2.2 \
26+
--hash=sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee \
27+
--hash=sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3
28+
# via virtualenv
1729
pre-commit==3.5.0 \
1830
--hash=sha256:5804465c675b659b0862f07907f96295d490822a450c4c40e747d0b1c6ebcb32 \
1931
--hash=sha256:841dc9aef25daba9a0238cd27984041fa0467b4199fc4852e27950664919f660
2032
# via -r requirements/dev.in
33+
virtualenv==20.26.2 \
34+
--hash=sha256:82bf0f4eebbb78d36ddaee0283d43fe5736b53880b8a8cdcd37390a07ac3741c \
35+
--hash=sha256:a624db5e94f01ad993d476b9ee5346fdf7b9de43ccaee0e0197012dc838a0e9b
36+
# via pre-commit

‎requirements/test.in‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,4 @@ pyright
55
pytest
66
pytest-mock
77
responses
8-
tox
98
zstandard

‎requirements/test.txt‎

Lines changed: 4 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,11 @@
1-
# SHA1:8ce43f90c536f423c82aec18ea5b31f8f6fda858
1+
# SHA1:b7ef6de31770eabde8905b7448eb9bda82e7b669
22
#
33
# This file is autogenerated by pip-compile-multi
44
# To update, run:
55
#
66
# pip-compile-multi
77
#
88
-r base.txt
9-
cachetools==5.3.3 \
10-
--hash=sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945 \
11-
--hash=sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105
12-
# via tox
13-
colorama==0.4.6 \
14-
--hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \
15-
--hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6
16-
# via tox
179
coverage==7.5.3 \
1810
--hash=sha256:015eddc5ccd5364dcb902eaecf9515636806fa1e0d5bef5769d06d0f31b54523 \
1911
--hash=sha256:04aefca5190d1dc7a53a4c1a5a7f8568811306d7a8ee231c42fb69215571944f \
@@ -68,20 +60,10 @@ coverage==7.5.3 \
6860
--hash=sha256:fcf7d1d6f5da887ca04302db8e0e0cf56ce9a5e05f202720e49b3e8157ddb9a9 \
6961
--hash=sha256:fd27d8b49e574e50caa65196d908f80e4dff64d7e592d0c59788b45aad7e8b35
7062
# via -r requirements/test.in
71-
distlib==0.3.8 \
72-
--hash=sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784 \
73-
--hash=sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64
74-
# via virtualenv
7563
exceptiongroup==1.2.1 \
7664
--hash=sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad \
7765
--hash=sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16
7866
# via pytest
79-
filelock==3.14.0 \
80-
--hash=sha256:43339835842f110ca7ae60f1e1c160714c5a6afd15a2873419ab185334975c0f \
81-
--hash=sha256:6ea72da3be9b8c82afd3edcf99f2fffbb5076335a5ae4d03248bb5b6c3eae78a
82-
# via
83-
# tox
84-
# virtualenv
8567
iniconfig==2.0.0 \
8668
--hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \
8769
--hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374
@@ -97,26 +79,11 @@ nodeenv==1.9.1 \
9779
packaging==24.1 \
9880
--hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \
9981
--hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124
100-
# via
101-
# pyproject-api
102-
# pytest
103-
# tox
104-
platformdirs==4.2.2 \
105-
--hash=sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee \
106-
--hash=sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3
107-
# via
108-
# tox
109-
# virtualenv
82+
# via pytest
11083
pluggy==1.5.0 \
11184
--hash=sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1 \
11285
--hash=sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669
113-
# via
114-
# pytest
115-
# tox
116-
pyproject-api==1.6.1 \
117-
--hash=sha256:1817dc018adc0d1ff9ca1ed8c60e1623d5aaca40814b953af14a9cf9a5cae538 \
118-
--hash=sha256:4c0116d60476b0786c88692cf4e325a9814965e2469c5998b830bba16b183675
119-
# via tox
86+
# via pytest
12087
pyright==1.1.367 \
12188
--hash=sha256:89de6502ae02f1552d0c4df4b46867887a419849f379db617695ef9308cf01eb \
12289
--hash=sha256:b1e5522ceb246ee6bc293a43d6d0162719d6467c1f1e9b81cee741aa11cdacbd
@@ -138,18 +105,7 @@ responses==0.25.2 \
138105
tomli==2.0.1 \
139106
--hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \
140107
--hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f
141-
# via
142-
# pyproject-api
143-
# pytest
144-
# tox
145-
tox==4.15.1 \
146-
--hash=sha256:53a092527d65e873e39213ebd4bd027a64623320b6b0326136384213f95b7076 \
147-
--hash=sha256:f00a5dc4222b358e69694e47e3da0227ac41253509bca9f45aa8f012053e8d9d
148-
# via -r requirements/test.in
149-
virtualenv==20.26.2 \
150-
--hash=sha256:82bf0f4eebbb78d36ddaee0283d43fe5736b53880b8a8cdcd37390a07ac3741c \
151-
--hash=sha256:a624db5e94f01ad993d476b9ee5346fdf7b9de43ccaee0e0197012dc838a0e9b
152-
# via tox
108+
# via pytest
153109
zstandard==0.22.0 \
154110
--hash=sha256:11f0d1aab9516a497137b41e3d3ed4bbf7b2ee2abc79e5c8b010ad286d7464bd \
155111
--hash=sha256:1958100b8a1cc3f27fa21071a55cb2ed32e9e5df4c3c6e661c193437f171cba2 \

‎taskcluster/config.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@ trust-domain: taskgraph
33
treeherder:
44
group-names:
55
'I': 'Docker Image Builds'
6-
'doc': 'Documentation tasks'
76
'Fetch': 'Fetch tasks'
7+
'check': 'Checks and lints'
8+
'doc': 'Documentation tasks'
9+
'unit': 'Unit test tasks'
810

911
index:
1012
products:

‎taskcluster/docker/python/Dockerfile‎

Lines changed: 6 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@
22
# License, v. 2.0. If a copy of the MPL was not distributed with this
33
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
44

5-
FROM python:3.12-slim
6-
MAINTAINER Release Engineering <[email protected]>
5+
#%ARG PYTHON_VERSION
6+
FROM python:$PYTHON_VERSION-slim
7+
LABEL maintainer="Release Engineering <[email protected]>"
78

89
# Add worker user
910
RUN mkdir /builds && \
@@ -16,13 +17,10 @@ RUN mkdir /builds && \
1617
RUN apt-get update \
1718
&& apt-get install -y --reinstall ca-certificates \
1819
&& apt-get install -y --force-yes --no-install-recommends \
19-
mercurial \
2020
build-essential \
21-
zlib1g-dev \
22-
libssl-dev \
23-
curl \
24-
git \
25-
build-essential zlib1g-dev libffi-dev libssl-dev libbz2-dev libreadline-dev libsqlite3-dev liblzma-dev pkg-config
21+
mercurial \
22+
git
23+
2624

2725
ENV SHELL=/bin/bash \
2826
HOME=/builds/worker \
@@ -31,13 +29,5 @@ ENV SHELL=/bin/bash \
3129
VOLUME /builds/worker/checkouts
3230
VOLUME /builds/worker/.cache
3331

34-
# pyenv
35-
# %ARG PYENV_VERSIONS
36-
ENV PYENV_ROOT=/builds/worker/.pyenv \
37-
PATH=/builds/worker/.pyenv/bin:/builds/worker/.pyenv/shims:$PATH
38-
# %include taskcluster/scripts/pyenv-setup
39-
ADD topsrcdir/taskcluster/scripts/pyenv-setup /builds/worker/pyenv-setup
40-
RUN chmod +x /builds/worker/pyenv-setup && /builds/worker/pyenv-setup "$PYENV_VERSIONS"
41-
4232
# Set a default command useful for debugging
4333
CMD ["/bin/bash", "--login"]

‎taskcluster/kinds/check/kind.yml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# This Source Code Form is subject to the terms of the Mozilla Public
2+
# License, v. 2.0. If a copy of the MPL was not distributed with this
3+
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4+
---
5+
6+
loader: taskgraph.loader.transform:loader
7+
8+
transforms:
9+
- taskgraph.transforms.run
10+
- taskgraph.transforms.task
11+
12+
task-defaults:
13+
attributes:
14+
retrigger: true
15+
code-review: true
16+
worker-type: t-linux
17+
worker:
18+
docker-image: {in-tree: py38}
19+
max-run-time: 600
20+
env:
21+
LC_ALL: "C"
22+
PYTHONUTF8: "0"
23+
PYTHONCOERCECLOCALE: "0"
24+
HGENCODING: "utf-8"
25+
treeherder:
26+
kind: test
27+
platform: test/opt
28+
tier: 1
29+
run:
30+
using: run-task
31+
cwd: '{checkout}'
32+
cache-dotcache: true
33+
34+
tasks:
35+
type:
36+
description: "Run type checking against src with Pyright"
37+
treeherder:
38+
symbol: check(type)
39+
run:
40+
command: >-
41+
pip install --user -r requirements/test.txt --require-hashes &&
42+
pip install --user --no-deps . &&
43+
pyright

‎taskcluster/kinds/codecov/kind.yml‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,28 @@
55
loader: taskgraph.loader.transform:loader
66

77
transforms:
8-
- taskgraph.transforms.run:transforms
9-
- taskgraph.transforms.task:transforms
8+
- taskgraph.transforms.from_deps
9+
- taskgraph.transforms.run
10+
- taskgraph.transforms.task
1011

1112
kind-dependencies:
1213
- fetch
13-
- tests
14+
- test
1415

1516
tasks:
1617
upload:
1718
description: "Upload coverage.xml to codecov.io"
19+
from-deps:
20+
kinds: [test]
21+
group-by: all
22+
unique-kinds: false
23+
set-name: null
24+
fetches:
25+
test:
26+
- artifact: coverage.{python_version}
1827
worker-type: t-linux
1928
worker:
20-
docker-image: {in-tree: python}
29+
docker-image: {in-tree: py312}
2130
max-run-time: 1800
2231
env:
2332
MOZ_FETCHES_DIR: /builds/worker/fetches
@@ -28,17 +37,15 @@ tasks:
2837
symbol: codecov
2938
scopes:
3039
- secrets:get:project/releng/taskgraph/ci
31-
dependencies:
32-
tests: tests-unit
3340
fetches:
3441
fetch:
3542
- codecov-uploader
36-
tests:
37-
- coverage.xml
3843
run-on-tasks-for: ["github-push", "github-pull-request"]
3944
run:
4045
using: run-task
4146
cwd: '{checkout}'
4247
command: >-
4348
pip install --user -r requirements/test.txt --require-hashes &&
49+
coverage combine --data-file $MOZ_FETCHES_DIR/coverage $MOZ_FETCHES_DIR &&
50+
coverage xml --data-file $MOZ_FETCHES_DIR/coverage -o $MOZ_FETCHES_DIR/coverage.xml &&
4451
python taskcluster/scripts/codecov-upload.py

‎taskcluster/kinds/doc/kind.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ transforms:
1212
task-defaults:
1313
worker-type: t-linux
1414
worker:
15-
docker-image: {in-tree: python}
15+
docker-image: {in-tree: py312}
1616
max-run-time: 3600
1717
treeherder:
1818
kind: test

‎taskcluster/kinds/docker-image/kind.yml‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,31 @@ tasks:
2323
symbol: I(fetch)
2424
index-task:
2525
symbol: I(idx)
26-
python:
27-
symbol: I(py)
26+
py38:
27+
definition: python
28+
symbol: I(py38)
2829
args:
29-
PYENV_VERSIONS: "3.12, 3.11, 3.10, 3.9, 3.8"
30+
PYTHON_VERSION: "3.8"
31+
py39:
32+
definition: python
33+
symbol: I(py39)
34+
args:
35+
PYTHON_VERSION: "3.9"
36+
py310:
37+
definition: python
38+
symbol: I(py310)
39+
args:
40+
PYTHON_VERSION: "3.10"
41+
py311:
42+
definition: python
43+
symbol: I(py311)
44+
args:
45+
PYTHON_VERSION: "3.11"
46+
py312:
47+
definition: python
48+
symbol: I(py312)
49+
args:
50+
PYTHON_VERSION: "3.12"
3051
run-task:
3152
symbol: I(rt)
3253
skopeo:

0 commit comments

Comments
 (0)