Skip to content

Commit f34eef5

Browse files
winglianwinglian
authored
update doc and use P2P=LOC for brittle grpo test (axolotl-ai-cloud#2649)
* update doc and skip brittle grpo test * fix the path to run the multigpu tests * increase timeout, use LOC instead of NVL * typo * use hf cache from s3 backed cloudfront * mark grpo as flaky test dues to vllm start
1 parent c7b6790 commit f34eef5

File tree

6 files changed

+131
-110
lines changed

6 files changed

+131
-110
lines changed

‎.github/workflows/multi-gpu-e2e.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: docker-multigpu-tests-biweekly
33
on:
44
pull_request:
55
paths:
6-
- 'tests/e2e/multigpu/*.py'
6+
- 'tests/e2e/multigpu/**.py'
77
- 'requirements.txt'
88
- 'setup.py'
99
- 'pyproject.toml'

‎.github/workflows/tests.yml

Lines changed: 121 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -44,96 +44,102 @@ jobs:
4444
env:
4545
SKIP: no-commit-to-branch
4646

47-
preload-cache:
48-
name: Preload HF cache
49-
runs-on: ubuntu-latest
50-
strategy:
51-
fail-fast: false
52-
matrix:
53-
python_version: ["3.11"]
54-
pytorch_version: ["2.6.0"]
55-
timeout-minutes: 20
56-
57-
env:
58-
AXOLOTL_IS_CI_CACHE_PRELOAD: "1"
59-
60-
steps:
61-
- name: Check out repository code
62-
uses: actions/checkout@v4
63-
64-
- name: Restore HF cache
65-
id: hf-cache-restore
66-
uses: actions/cache/restore@v4
67-
with:
68-
path: |
69-
/home/runner/.cache/huggingface/hub/datasets--*
70-
/home/runner/.cache/huggingface/hub/models--*
71-
key: ${{ runner.os }}-hf-hub-cache-v2
72-
73-
- name: Setup Python
74-
uses: actions/setup-python@v5
75-
with:
76-
python-version: ${{ matrix.python_version }}
77-
cache: 'pip' # caching pip dependencies
78-
79-
- name: upgrade pip
80-
run: |
81-
pip3 install --upgrade pip
82-
pip3 install --upgrade packaging==23.2 setuptools==75.8.0 wheel
83-
84-
- name: Install PyTorch
85-
run: |
86-
pip3 install torch==${{ matrix.pytorch_version }}
87-
88-
- name: Install dependencies
89-
run: |
90-
pip3 show torch
91-
pip3 install --no-build-isolation -U -e .
92-
python scripts/unsloth_install.py | sh
93-
python scripts/cutcrossentropy_install.py | sh
94-
pip3 install -r requirements-dev.txt -r requirements-tests.txt
95-
96-
- name: Make sure PyTorch version wasn't clobbered
97-
run: |
98-
python -c "import torch; assert '${{ matrix.pytorch_version }}' in torch.__version__"
99-
100-
- name: Ensure axolotl CLI was installed
101-
run: |
102-
axolotl --help
103-
104-
- name: Pre-Download dataset fixture
105-
run: |
106-
huggingface-cli download --repo-type=dataset axolotl-ai-internal/axolotl-oss-dataset-fixtures
107-
108-
- name: Run tests
109-
run: |
110-
pytest -v tests/conftest.py
111-
112-
- name: Upload coverage to Codecov
113-
uses: codecov/codecov-action@v5
114-
with:
115-
token: ${{ secrets.CODECOV_TOKEN }}
116-
files: ./coverage.xml
117-
flags: unittests,pytorch-${{ matrix.pytorch_version }}
118-
fail_ci_if_error: false
119-
120-
- name: cleanup pip cache
121-
run: |
122-
find "$(pip cache dir)/http-v2" -type f -mtime +14 -exec rm {} \;
123-
124-
- name: Save HF cache
125-
id: hf-cache
126-
uses: actions/cache/save@v4
127-
with:
128-
path: |
129-
/home/runner/.cache/huggingface/hub/datasets--*
130-
/home/runner/.cache/huggingface/hub/models--*
131-
key: ${{ steps.hf-cache-restore.outputs.cache-primary-key }}
47+
# preload-cache:
48+
# name: Preload HF cache
49+
# runs-on: ubuntu-latest
50+
# strategy:
51+
# fail-fast: false
52+
# matrix:
53+
# python_version: ["3.11"]
54+
# pytorch_version: ["2.6.0"]
55+
# timeout-minutes: 20
56+
#
57+
# env:
58+
# AXOLOTL_IS_CI_CACHE_PRELOAD: "1"
59+
#
60+
# steps:
61+
# - name: Check out repository code
62+
# uses: actions/checkout@v4
63+
#
64+
# - name: Restore HF cache
65+
# id: hf-cache-restore
66+
# uses: actions/cache/restore@v4
67+
# with:
68+
# path: |
69+
# /home/runner/.cache/huggingface/hub/datasets--*
70+
# /home/runner/.cache/huggingface/hub/models--*
71+
# key: ${{ runner.os }}-hf-hub-cache-v2
72+
#
73+
# - name: Restore Cache from S3
74+
# id: hf-cache-restore-s3
75+
# run: |
76+
# mkdir -p /home/runner/.cache/huggingface/hub
77+
# curl -L https://d1dttdx32dkk5p.cloudfront.net/hf-cache.tar.zst | tar -xf - -C /home/runner/.cache/huggingface/hub/ --use-compress-program unzstd
78+
#
79+
# - name: Setup Python
80+
# uses: actions/setup-python@v5
81+
# with:
82+
# python-version: ${{ matrix.python_version }}
83+
# cache: 'pip' # caching pip dependencies
84+
#
85+
# - name: upgrade pip
86+
# run: |
87+
# pip3 install --upgrade pip
88+
# pip3 install --upgrade packaging==23.2 setuptools==75.8.0 wheel
89+
#
90+
# - name: Install PyTorch
91+
# run: |
92+
# pip3 install torch==${{ matrix.pytorch_version }}
93+
#
94+
# - name: Install dependencies
95+
# run: |
96+
# pip3 show torch
97+
# pip3 install --no-build-isolation -U -e .
98+
# python scripts/unsloth_install.py | sh
99+
# python scripts/cutcrossentropy_install.py | sh
100+
# pip3 install -r requirements-dev.txt -r requirements-tests.txt
101+
#
102+
# - name: Make sure PyTorch version wasn't clobbered
103+
# run: |
104+
# python -c "import torch; assert '${{ matrix.pytorch_version }}' in torch.__version__"
105+
#
106+
# - name: Ensure axolotl CLI was installed
107+
# run: |
108+
# axolotl --help
109+
#
110+
# - name: Pre-Download dataset fixture
111+
# run: |
112+
# huggingface-cli download --repo-type=dataset axolotl-ai-internal/axolotl-oss-dataset-fixtures
113+
#
114+
# - name: Run tests
115+
# run: |
116+
# pytest -v tests/conftest.py
117+
#
118+
# - name: Upload coverage to Codecov
119+
# uses: codecov/codecov-action@v5
120+
# with:
121+
# token: ${{ secrets.CODECOV_TOKEN }}
122+
# files: ./coverage.xml
123+
# flags: unittests,pytorch-${{ matrix.pytorch_version }}
124+
# fail_ci_if_error: false
125+
#
126+
# - name: cleanup pip cache
127+
# run: |
128+
# find "$(pip cache dir)/http-v2" -type f -mtime +14 -exec rm {} \;
129+
#
130+
# - name: Save HF cache
131+
# id: hf-cache
132+
# uses: actions/cache/save@v4
133+
# with:
134+
# path: |
135+
# /home/runner/.cache/huggingface/hub/datasets--*
136+
# /home/runner/.cache/huggingface/hub/models--*
137+
# key: ${{ steps.hf-cache-restore.outputs.cache-primary-key }}
132138

133139
pytest:
134140
name: PyTest
135141
runs-on: ubuntu-latest
136-
needs: [preload-cache]
142+
# needs: [preload-cache]
137143
strategy:
138144
fail-fast: false
139145
matrix:
@@ -145,14 +151,20 @@ jobs:
145151
- name: Check out repository code
146152
uses: actions/checkout@v4
147153

148-
- name: Restore HF cache
149-
id: hf-cache-restore
150-
uses: actions/cache/restore@v4
151-
with:
152-
path: |
153-
/home/runner/.cache/huggingface/hub/datasets--*
154-
/home/runner/.cache/huggingface/hub/models--*
155-
key: ${{ runner.os }}-hf-hub-cache-v2
154+
# - name: Restore HF cache
155+
# id: hf-cache-restore
156+
# uses: actions/cache/restore@v4
157+
# with:
158+
# path: |
159+
# /home/runner/.cache/huggingface/hub/datasets--*
160+
# /home/runner/.cache/huggingface/hub/models--*
161+
# key: ${{ runner.os }}-hf-hub-cache-v2
162+
163+
- name: Restore Cache from S3
164+
id: hf-cache-restore-s3
165+
run: |
166+
mkdir -p /home/runner/.cache/huggingface/hub
167+
curl -L https://d1dttdx32dkk5p.cloudfront.net/hf-cache.tar.zst | tar -xf - -C /home/runner/.cache/huggingface/hub/ --use-compress-program unzstd
156168
157169
- name: Setup Python
158170
uses: actions/setup-python@v5
@@ -210,7 +222,7 @@ jobs:
210222
pytest-sdist:
211223
name: PyTest from Source Dist
212224
runs-on: ubuntu-latest
213-
needs: [preload-cache]
225+
# needs: [preload-cache]
214226
strategy:
215227
fail-fast: false
216228
matrix:
@@ -222,14 +234,20 @@ jobs:
222234
- name: Check out repository code
223235
uses: actions/checkout@v4
224236

225-
- name: Restore HF cache
226-
id: hf-cache-restore
227-
uses: actions/cache/restore@v4
228-
with:
229-
path: |
230-
/home/runner/.cache/huggingface/hub/datasets--*
231-
/home/runner/.cache/huggingface/hub/models--*
232-
key: ${{ runner.os }}-hf-hub-cache-v2
237+
# - name: Restore HF cache
238+
# id: hf-cache-restore
239+
# uses: actions/cache/restore@v4
240+
# with:
241+
# path: |
242+
# /home/runner/.cache/huggingface/hub/datasets--*
243+
# /home/runner/.cache/huggingface/hub/models--*
244+
# key: ${{ runner.os }}-hf-hub-cache-v2
245+
246+
- name: Restore Cache from S3
247+
id: hf-cache-restore-s3
248+
run: |
249+
mkdir -p /home/runner/.cache/huggingface/hub
250+
curl -L https://d1dttdx32dkk5p.cloudfront.net/hf-cache.tar.zst | tar -xf - -C /home/runner/.cache/huggingface/hub/ --use-compress-program unzstd
233251
234252
- name: Setup Python
235253
uses: actions/setup-python@v5

‎cicd/e2e_tests.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
@app.function(
77
image=cicd_image,
88
gpu=GPU_CONFIG,
9-
timeout=60 * 60,
9+
timeout=90 * 60, # 90 min
1010
cpu=8.0,
1111
memory=131072,
1212
volumes=VOLUME_CONFIG,

‎codecov.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ coverage:
1919
if_no_uploads: error
2020
if_not_found: success
2121
if_ci_failed: error
22-
only_pulls: false
22+
only_pulls: true
2323
flags: null
2424
paths: null
2525
patch:

‎docs/config.qmd

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -505,6 +505,7 @@ save_strategy: # Set to `"no"` to skip checkpoint saves, `"epoch"` at end of eac
505505
save_steps: # Leave empty to save at each epoch, integer for every N steps. float for fraction of total steps
506506
saves_per_epoch: # number of times per epoch to save a checkpoint, mutually exclusive with save_steps
507507
save_total_limit: # Checkpoints saved at a time
508+
save_only_model: # Save only the model weights, skipping the optimizer. Using this means you can't resume from checkpoints.
508509
# Maximum number of iterations to train for. It precedes num_epochs which means that
509510
# if both are set, num_epochs will not be guaranteed.
510511
# e.g., when 1 epoch is 1000 steps => `num_epochs: 2` and `max_steps: 100` will train for 100 steps

‎tests/e2e/multigpu/solo/test_grpo.py

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ def transform_fn(example, tokenizer=None):
166166
"""
167167
)
168168

169+
@pytest.mark.skip(reason="flaky test")
169170
@pytest.mark.parametrize(
170171
"num_gpus",
171172
[1, 2],
@@ -227,7 +228,7 @@ def test_llama_dora(self, temp_dir, num_gpus):
227228

228229
current_env = os.environ.copy()
229230
env = {
230-
"NCCL_P2P_LEVEL": "NVL",
231+
"NCCL_P2P_LEVEL": "LOC",
231232
**current_env,
232233
"CUDA_VISIBLE_DEVICES": "1",
233234
"VLLM_DISABLE_COMPILE_CACHE": "1",
@@ -257,14 +258,15 @@ def test_llama_dora(self, temp_dir, num_gpus):
257258
f"{get_torch_dist_unique_port()}",
258259
],
259260
env={
260-
"NCCL_P2P_LEVEL": "NVL",
261+
"NCCL_P2P_LEVEL": "LOC",
261262
"NCCL_DEBUG": "INFO",
262263
**current_env,
263264
},
264265
)
265266
finally:
266267
recursive_kill(vllm_process)
267268

269+
@pytest.mark.skip(reason="flaky test")
268270
@pytest.mark.parametrize(
269271
"num_gpus",
270272
[1, 2],
@@ -320,7 +322,7 @@ def test_llama_fft(self, temp_dir, num_gpus):
320322

321323
current_env = os.environ.copy()
322324
env = {
323-
"NCCL_P2P_LEVEL": "NVL", # nccl can be brittle, assume P2P isn't reliable
325+
"NCCL_P2P_LEVEL": "LOC", # nccl can be brittle, assume P2P isn't reliable
324326
**current_env,
325327
"CUDA_VISIBLE_DEVICES": "1",
326328
"VLLM_DISABLE_COMPILE_CACHE": "1",
@@ -350,7 +352,7 @@ def test_llama_fft(self, temp_dir, num_gpus):
350352
f"{get_torch_dist_unique_port()}",
351353
],
352354
env={
353-
"NCCL_P2P_LEVEL": "NVL",
355+
"NCCL_P2P_LEVEL": "LOC",
354356
"NCCL_DEBUG": "INFO",
355357
**current_env,
356358
},

0 commit comments

Comments
 (0)