Skip to content

Commit f776f88

Browse files
djsaundeDan Saunders
djsaunde
and
Dan Saunders
authored
adding codecov reporting (axolotl-ai-cloud#2372) [skip ci]
* adding codecov reporting * update codecov-action to v5 * fix --------- Co-authored-by: Dan Saunders <[email protected]>
1 parent 69eda20 commit f776f88

File tree

8 files changed

+155
-12
lines changed

8 files changed

+155
-12
lines changed

‎.coveragerc

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
[run]
2+
source = axolotl
3+
omit =
4+
*/tests/*
5+
setup.py
6+
7+
[report]
8+
exclude_lines =
9+
pragma: no cover
10+
def __repr__
11+
raise NotImplementedError
12+
if __name__ == .__main__.:
13+
pass
14+
raise ImportError

‎.github/workflows/tests.yml

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,9 +102,16 @@ jobs:
102102
103103
- name: Run tests
104104
run: |
105-
pytest -v -n8 --dist loadfile --ignore=tests/e2e/ --ignore=tests/patched/ --ignore=tests/cli/ tests/
106-
pytest -v tests/patched/
107-
pytest -v tests/cli/
105+
pytest -v -n8 --dist loadfile --ignore=tests/e2e/ --ignore=tests/patched/ --ignore=tests/cli/ tests/ --cov=axolotl --cov-report=xml
106+
pytest -v tests/patched/ --cov=axolotl --cov-append --cov-report=xml
107+
pytest -v tests/cli/ --cov=axolotl --cov-append --cov-report=xml
108+
109+
- name: Upload coverage to Codecov
110+
uses: codecov/codecov-action@v5
111+
with:
112+
files: ./coverage.xml
113+
flags: unittests,pytorch-${{ matrix.pytorch_version }}
114+
fail_ci_if_error: false
108115

109116
- name: cleanup pip cache
110117
run: |

‎README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
<p align="center">
1010
<img src="https://img.shields.io/github/license/axolotl-ai-cloud/axolotl.svg?color=blue" alt="GitHub License">
1111
<img src="https://github.com/axolotl-ai-cloud/axolotl/actions/workflows/tests.yml/badge.svg" alt="tests">
12+
<a href="https://codecov.io/gh/axolotl-ai-cloud/axolotl"><img src="https://codecov.io/gh/axolotl-ai-cloud/axolotl/branch/main/graph/badge.svg" alt="codecov"></a>
1213
<a href="https://github.com/axolotl-ai-cloud/axolotl/releases"><img src="https://img.shields.io/github/release/axolotl-ai-cloud/axolotl.svg" alt="Releases"></a>
1314
<br/>
1415
<a href="https://github.com/axolotl-ai-cloud/axolotl/graphs/contributors"><img src="https://img.shields.io/github/contributors-anon/axolotl-ai-cloud/axolotl?color=yellow&style=flat-square" alt="contributors" style="height: 20px;"></a>

‎cicd/cicd.sh

Lines changed: 56 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,59 @@ set -e
33

44
python -c "import torch; assert '$PYTORCH_VERSION' in torch.__version__"
55

6-
pytest -v --durations=10 -n8 --ignore=tests/e2e/ --ignore=tests/patched/ --ignore=tests/cli /workspace/axolotl/tests/
7-
pytest -v --durations=10 /workspace/axolotl/tests/e2e/patched/lora_kernels # running these with the other patches causes a failure
8-
pytest -v --durations=10 --ignore=tests/e2e/patched/lora_kernels /workspace/axolotl/tests/e2e/patched
9-
pytest -v --durations=10 -n1 /workspace/axolotl/tests/e2e/solo/
10-
pytest -v --durations=10 /workspace/axolotl/tests/e2e/integrations/
11-
pytest -v --durations=10 /workspace/axolotl/tests/cli
12-
pytest -v --durations=10 --ignore=tests/e2e/solo/ --ignore=tests/e2e/patched/ --ignore=tests/e2e/multigpu/ --ignore=tests/e2e/integrations/ --ignore=tests/cli /workspace/axolotl/tests/e2e/
6+
# Run unit tests with initial coverage report
7+
pytest -v --durations=10 -n8 \
8+
--ignore=tests/e2e/ \
9+
--ignore=tests/patched/ \
10+
--ignore=tests/cli \
11+
/workspace/axolotl/tests/ \
12+
--cov=axolotl \
13+
--cov-report=xml:coverage.xml
14+
15+
# Run lora kernels tests with coverage append
16+
pytest -v --durations=10 \
17+
/workspace/axolotl/tests/e2e/patched/lora_kernels \
18+
--cov=axolotl \
19+
--cov-append
20+
21+
# Run patched tests excluding lora kernels with coverage append
22+
pytest -v --durations=10 \
23+
--ignore=tests/e2e/patched/lora_kernels \
24+
/workspace/axolotl/tests/e2e/patched \
25+
--cov=axolotl \
26+
--cov-append
27+
28+
# Run solo tests with coverage append
29+
pytest -v --durations=10 -n1 \
30+
/workspace/axolotl/tests/e2e/solo/ \
31+
--cov=axolotl \
32+
--cov-append
33+
34+
# Run integration tests with coverage append
35+
pytest -v --durations=10 \
36+
/workspace/axolotl/tests/e2e/integrations/ \
37+
--cov=axolotl \
38+
--cov-append
39+
40+
pytest -v --durations=10 /workspace/axolotl/tests/cli \
41+
--cov=axolotl \
42+
--cov-append
43+
44+
# Run remaining e2e tests with coverage append and final report
45+
pytest -v --durations=10 \
46+
--ignore=tests/e2e/solo/ \
47+
--ignore=tests/e2e/patched/ \
48+
--ignore=tests/e2e/multigpu/ \
49+
--ignore=tests/e2e/integrations/ \
50+
--ignore=tests/cli \
51+
/workspace/axolotl/tests/e2e/ \
52+
--cov=axolotl \
53+
--cov-append \
54+
--cov-report=xml:coverage.xml
55+
56+
# Upload coverage to Codecov
57+
if [ -f e2e-coverage.xml ]; then
58+
codecov -f e2e-coverage.xml -F e2e,pytorch-${PYTORCH_VERSION}
59+
else
60+
echo "Coverage file not found. Coverage report may have failed."
61+
fi

‎cicd/multigpu.sh

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,22 @@ set -e
44
# only run one test at a time so as not to OOM the GPU
55
pytest -v --durations=10 -n2 /workspace/axolotl/tests/e2e/multigpu/ --ignore=/workspace/axolotl/tests/e2e/multigpu/solo/
66
pytest -v --durations=10 -n1 /workspace/axolotl/tests/e2e/multigpu/solo/
7+
8+
# Only run two tests at a time to avoid OOM on GPU (with coverage collection)
9+
pytest -v -n2 \
10+
--ignore=/workspace/axolotl/tests/e2e/multigpu/solo/
11+
/workspace/axolotl/tests/e2e/multigpu/ \
12+
--cov=axolotl \
13+
--cov-report=xml:multigpu-coverage.xml
14+
15+
pytest -v --durations=10 -n1 /workspace/axolotl/tests/e2e/multigpu/solo/ \
16+
--cov=axolotl \
17+
--cov-append \
18+
--cov-report=xml:multigpu-coverage.xml
19+
20+
# Upload coverage to Codecov
21+
if [ -f multigpu-coverage.xml ]; then
22+
codecov -f multigpu-coverage.xml -F multigpu,docker-tests,pytorch-${PYTORCH_VERSION}
23+
else
24+
echo "Coverage file not found. Coverage report may have failed."
25+
fi

‎codecov.yml

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
codecov:
2+
require_ci_to_pass: yes
3+
4+
coverage:
5+
precision: 2
6+
round: down
7+
range: "70...100"
8+
status:
9+
project:
10+
default:
11+
# basic
12+
target: auto
13+
threshold: 0%
14+
base: auto
15+
# advanced
16+
branches: null
17+
if_no_uploads: error
18+
if_not_found: success
19+
if_ci_failed: error
20+
only_pulls: false
21+
flags: null
22+
paths: null
23+
patch:
24+
default:
25+
# basic
26+
target: auto
27+
threshold: 0%
28+
base: auto
29+
# advanced
30+
branches: null
31+
if_no_uploads: error
32+
if_not_found: success
33+
if_ci_failed: error
34+
only_pulls: false
35+
flags: null
36+
paths: null
37+
38+
parsers:
39+
gcov:
40+
branch_detection:
41+
conditional: yes
42+
loop: yes
43+
method: no
44+
macro: no
45+
46+
comment:
47+
layout: "reach,diff,flags,files,footer"
48+
behavior: default
49+
require_changes: no
50+
require_base: no
51+
require_head: yes

‎requirements-dev.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
pre-commit
21
black
32
mypy
3+
pre-commit
44
types-requests
55
quartodoc
66
jupyter

‎requirements-tests.txt

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
1+
codecov
12
pytest
2-
pytest-xdist
3+
pytest-cov
34
pytest-retry
45
pytest-sugar
6+
pytest-xdist
57
tbparse

0 commit comments

Comments
 (0)