Skip to content

Commit b64be96

Browse files
alexbadenalexbaden
authored
Merge branch 'main' into alex/2585
2 parents ae3ba41 + 816d7ef commit b64be96

40 files changed

+1104
-515
lines changed

‎.github/actions/setup-pytorch/action.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ runs:
8383
uses: ./.github/actions/load
8484
env:
8585
# Increase this value to reset cache
86-
CACHE_NUMBER: 12
86+
CACHE_NUMBER: 14
8787
with:
8888
path: pytorch
8989
key: pytorch-$PYTORCH_CACHE_KEY-$CACHE_NUMBER
@@ -120,7 +120,7 @@ runs:
120120
cd pytorch
121121
pip install wheel
122122
pip install -r requirements.txt
123-
python setup.py bdist_wheel
123+
USE_STATIC_MKL=1 python setup.py bdist_wheel
124124
125125
- name: Install PyTorch (built from source)
126126
if: ${{ inputs.mode == 'source' }}

‎.github/workflows/integration-tests.yml‎

Lines changed: 99 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,12 @@ concurrency:
2121
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
2222
permissions: read-all
2323
env:
24+
TRITON_BUILD_WITH_CCACHE: "true"
2425
TRITON_BUILD_WITH_CLANG_LLD: "TRUE"
2526
TRITON_USE_ASSERT_ENABLED_LLVM: "TRUE"
2627
TRITON_DISABLE_LINE_INFO: 1
2728
PROTON_SKIP_PC_SAMPLING_TEST: 1
29+
CCACHE_COMPRESS: "true"
2830
jobs:
2931
Runner-Preparation:
3032
runs-on: ubuntu-latest
@@ -39,6 +41,11 @@ jobs:
3941
if: github.event_name == 'pull_request'
4042
run: |
4143
echo "enable_integration=true" >> $GITHUB_ENV
44+
- name: Decide manual trigger integration test enablement
45+
# Always enable integration tests when manually triggered
46+
if: github.event_name == 'workflow_dispatch'
47+
run: |
48+
echo "enable_integration=true" >> $GITHUB_ENV
4249
- name: Checkout post-submit commits
4350
if: github.event_name == 'push'
4451
uses: actions/checkout@v4
@@ -154,6 +161,8 @@ jobs:
154161
strategy:
155162
matrix:
156163
runner: ${{fromJson(needs.Runner-Preparation.outputs.matrix-CUDA)}}
164+
env:
165+
RUNNER_TYPE: ${{ matrix.runner[0] }}
157166
steps:
158167
- name: Checkout
159168
uses: actions/checkout@v4
@@ -199,22 +208,28 @@ jobs:
199208
# "restore" step. This is to prevent the caches from accumulating stale
200209
# files over time.
201210
name: Restore cache of ccache and Triton compilation artifacts
202-
if: github.event_name != 'push'
211+
id: restore-build-cache
212+
if: github.ref != 'refs/heads/main'
203213
uses: actions/cache/restore@v4
204214
with:
205215
path: |
206216
~/.triton/cache
207-
~/.cache/ccache
217+
~/.ccache
208218
# Restore the most recent cache entry.
209-
restore-keys: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ runner.name }}-llvm-${{ steps.cache-key.outputs.llvm }}-
219+
restore-keys: |
220+
triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-llvm-${{ steps.cache-key.outputs.llvm }}-
221+
triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-
210222
# We expect this cache key never to hit and for us to fall back
211223
# unconditionally to the restore-key, so it doesn't actually matter
212224
# what we put here (so long as it doesn't hit an existing key).
213-
key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ runner.name }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }}
214-
- name: Inspect cache directory
225+
key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }}
226+
- name: Inspect cache directories
215227
run: |
216228
mkdir -p ~/.triton
217-
ls -alh ~/.triton
229+
du -h -d 1 ~/.triton
230+
231+
mkdir -p ~/.ccache
232+
du -h -d 1 ~/.ccache
218233
- name: Update PATH
219234
run: |
220235
echo "$HOME/.local/bin" >> $GITHUB_PATH
@@ -224,12 +239,14 @@ jobs:
224239
python3 -m pip install cython setuptools wheel cmake==3.24 ninja pytest-forked pytest-xdist lit
225240
- name: Install Triton
226241
env:
227-
TRITON_BUILD_WITH_CCACHE: "true"
228242
CUDA_HOME: "/usr/local/cuda"
229243
run: |
230244
echo "PATH is '$PATH'"
231245
cd python
232-
python3 -m pip install '.[tests]'
246+
ccache --zero-stats
247+
python3 -m pip install -v '.[tests]'
248+
- name: CCache Stats
249+
run: ccache --print-stats
233250
- name: Run lit tests
234251
run: |
235252
cd python
@@ -278,6 +295,13 @@ jobs:
278295
cd third_party/proton/test
279296
python3 -m pytest -s .
280297
cd ..
298+
- name: Inspect cache directories
299+
run: |
300+
mkdir -p ~/.triton
301+
du -h -d 1 ~/.triton
302+
303+
mkdir -p ~/.ccache
304+
du -h -d 1 ~/.ccache
281305
- # If we're on branch `main`, save the ccache Triton compilation artifacts
282306
# to the cache so they can be used by other (non-main) CI runs.
283307
#
@@ -287,22 +311,17 @@ jobs:
287311
if: github.ref == 'refs/heads/main'
288312
uses: actions/cache/save@v4
289313
with:
290-
path: ~/.triton/cache ~/.cache/ccache
291-
key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ runner.name }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }}
292-
- name: Inspect cache directories
293-
run: |
294-
mkdir -p ~/.triton
295-
ls -alh ~/.triton
296-
du -sh ~/.triton/**
297-
298-
mkdir -p ~/.cache/ccache
299-
ls -alh ~/.cache/ccache
300-
du -sh ~/.cache/ccache
314+
path: |
315+
~/.triton/cache
316+
~/.ccache
317+
key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }}
301318
Integration-Tests-AMD:
302319
needs: Runner-Preparation
303320
if: needs.Runner-Preparation.outputs.matrix-HIP != ''
304321
runs-on: ${{ matrix.runner }}
305322
timeout-minutes: 30
323+
env:
324+
RUNNER_TYPE: ${{ matrix.runner[1] }}
306325
strategy:
307326
matrix:
308327
runner: ${{fromJson(needs.Runner-Preparation.outputs.matrix-HIP)}}
@@ -355,40 +374,53 @@ jobs:
355374
# "restore" step. This is to prevent the caches from accumulating stale
356375
# files over time.
357376
name: Restore cache of ccache and Triton compilation artifacts
358-
if: github.event_name != 'push'
377+
id: restore-build-cache
378+
if: github.ref != 'refs/heads/main'
359379
uses: actions/cache/restore@v4
360380
with:
361381
path: |
362382
~/.triton/cache
363-
~/.cache/ccache
383+
~/.ccache
364384
# Restore the most recent cache entry.
365-
restore-keys: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ runner.name }}-llvm-${{ steps.cache-key.outputs.llvm }}-
385+
restore-keys: |
386+
triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-llvm-${{ steps.cache-key.outputs.llvm }}-
387+
triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-
366388
# We expect this cache key never to hit and for us to fall back
367389
# unconditionally to the restore-key, so it doesn't actually matter
368390
# what we put here (so long as it doesn't hit an existing key).
369-
key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ runner.name }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }}
370-
- name: Inspect cache directory
391+
key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }}
392+
- name: Inspect cache directories
371393
run: |
372394
mkdir -p ~/.triton
373-
ls -alh ~/.triton
395+
du -h -d 1 ~/.triton
396+
397+
mkdir -p ~/.ccache
398+
du -h -d 1 ~/.ccache
374399
- name: Update PATH
375400
run: |
376401
echo "/opt/rocm/llvm/bin" >> $GITHUB_PATH
377402
- name: Install pip dependencies
378403
run: |
379404
python3 -m pip install --upgrade pip
380405
python3 -m pip install lit
406+
- name: Install apt dependencies
407+
run: |
408+
apt update
409+
apt install ccache
381410
- name: Install Triton
382411
id: amd-install-triton
383412
run: |
384413
echo "PATH is '$PATH'"
385414
pip uninstall -y triton
386415
cd python
416+
ccache --zero-stats
387417
pip install -v -e '.[tests]'
388418
- name: Clean up after an unsuccessful build
389419
if: ${{ !success() && steps.amd-install-triton.outcome != 'success' }}
390420
run: |
391421
rm -rf ~/.triton
422+
- name: CCache Stats
423+
run: ccache --print-stats
392424
- name: Run lit tests
393425
run: |
394426
cd python
@@ -431,6 +463,13 @@ jobs:
431463
cd python
432464
cd "build/$(ls build | grep -i cmake)"
433465
ctest -j32
466+
- name: Inspect cache directories
467+
run: |
468+
mkdir -p ~/.triton
469+
du -h -d 1 ~/.triton
470+
471+
mkdir -p ~/.ccache
472+
du -h -d 1 ~/.ccache
434473
- # If we're on branch `main`, save the ccache Triton compilation artifacts
435474
# to the cache so they can be used by other (non-main) CI runs.
436475
#
@@ -440,28 +479,23 @@ jobs:
440479
if: github.ref == 'refs/heads/main'
441480
uses: actions/cache/save@v4
442481
with:
443-
path: ~/.triton/cache ~/.cache/ccache
444-
key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ runner.name }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }}
445-
- name: Inspect cache directories
446-
run: |
447-
mkdir -p ~/.triton
448-
ls -alh ~/.triton
449-
du -sh ~/.triton/**
450-
451-
mkdir -p ~/.cache/ccache
452-
ls -alh ~/.cache/ccache
453-
du -sh ~/.cache/ccache
482+
path: |
483+
~/.triton/cache
484+
~/.ccache
485+
key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }}
454486
- name: Clean up caches
455487
run: |
456488
rm -rf ~/.triton/cache
457489
Build-Tests:
458490
needs: Runner-Preparation
459491
if: needs.Runner-Preparation.outputs.matrix-MACOS != ''
460492
runs-on: ${{ matrix.runner }}
461-
timeout-minutes: 30
493+
timeout-minutes: 40
462494
strategy:
463495
matrix:
464496
runner: ${{fromJson(needs.Runner-Preparation.outputs.matrix-MACOS)}}
497+
env:
498+
RUNNER_TYPE: ${{ matrix.runner[0] }}
465499
steps:
466500
- name: Checkout
467501
uses: actions/checkout@v4
@@ -470,7 +504,7 @@ jobs:
470504
- name: Install brew dependencies
471505
run: |
472506
brew update
473-
brew install ccache llvm@19 lld
507+
brew install ccache llvm@19 lld coreutils
474508
- name: Compute cache keys
475509
id: cache-key
476510
run: |
@@ -511,22 +545,28 @@ jobs:
511545
# "restore" step. This is to prevent the caches from accumulating stale
512546
# files over time.
513547
name: Restore cache of ccache and Triton compilation artifacts
514-
if: github.event_name != 'push'
548+
id: restore-build-cache
549+
if: github.ref != 'refs/heads/main'
515550
uses: actions/cache/restore@v4
516551
with:
517552
path: |
518553
~/.triton/cache
519-
~/.cache/ccache
554+
~/.ccache
520555
# Restore the most recent cache entry.
521-
restore-keys: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ runner.name }}-llvm-${{ steps.cache-key.outputs.llvm }}-
556+
restore-keys: |
557+
triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-llvm-${{ steps.cache-key.outputs.llvm }}-
558+
triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-
522559
# We expect this cache key never to hit and for us to fall back
523560
# unconditionally to the restore-key, so it doesn't actually matter
524561
# what we put here (so long as it doesn't hit an existing key).
525-
key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ runner.name }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }}
526-
- name: Inspect cache directory
562+
key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }}
563+
- name: Inspect cache directories
527564
run: |
528565
mkdir -p ~/.triton
529-
ls -alh ~/.triton
566+
du -h -d 1 ~/.triton
567+
568+
mkdir -p ~/.ccache
569+
du -h -d 1 ~/.ccache
530570
- name: Update PATH
531571
run: |
532572
echo "$HOME/.local/bin" >> $GITHUB_PATH
@@ -539,7 +579,6 @@ jobs:
539579
python3 -m pip install cython setuptools wheel cmake==3.24 ninja pytest-xdist lit pybind11
540580
- name: Install Triton
541581
env:
542-
TRITON_BUILD_WITH_CCACHE: "true"
543582
TRITON_BUILD_WITH_O1: "true"
544583
# macos-latest has 3 vcpus and 7GB DRAM, to save memory we limit the number of jobs to 3
545584
# https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories
@@ -548,7 +587,17 @@ jobs:
548587
source ~/.venv/bin/activate
549588
echo "PATH is '$PATH'"
550589
cd python
551-
python3 -m pip install --no-build-isolation .
590+
ccache --zero-stats
591+
python3 -m pip install -v --no-build-isolation .
592+
- name: CCache Stats
593+
run: ccache --print-stats
594+
- name: Inspect cache directories
595+
run: |
596+
mkdir -p ~/.triton
597+
du -h -d 1 ~/.triton
598+
599+
mkdir -p ~/.ccache
600+
du -h -d 1 ~/.ccache
552601
- # If we're on branch `main`, save the ccache Triton compilation artifacts
553602
# to the cache so they can be used by other (non-main) CI runs.
554603
#
@@ -558,14 +607,7 @@ jobs:
558607
if: github.ref == 'refs/heads/main'
559608
uses: actions/cache/save@v4
560609
with:
561-
path: ~/.triton/cache ~/.cache/ccache
562-
key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ runner.name }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }}
563-
- name: Inspect cache directories
564-
run: |
565-
mkdir -p ~/.triton
566-
ls -alh ~/.triton
567-
du -sh ~/.triton/**
568-
569-
mkdir -p ~/.cache/ccache
570-
ls -alh ~/.cache/ccache
571-
du -sh ~/.cache/ccache
610+
path: |
611+
~/.triton/cache
612+
~/.ccache
613+
key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }}

0 commit comments

Comments
 (0)