Skip to content
This repository was archived by the owner on Sep 28, 2024. It is now read-only.

Commit aaf7d45

Browse files
avik-palavik-pal
authored
Merge pull request #12 from LuxDL/ap/ci_improve
standardization across LuxDL packages
2 parents 44c39bb + 56d5d00 commit aaf7d45

27 files changed

+354
-373
lines changed

‎.buildkite/pipeline.yml‎

Lines changed: 20 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,55 +1,26 @@
11
steps:
2-
- label: ":julia: Julia {{matrix.julia}} + CUDA GPU"
3-
plugins:
4-
- JuliaCI/julia#v1:
5-
version: "{{matrix.julia}}"
6-
- JuliaCI/julia-test#v1:
7-
test_args: "--quickfail"
8-
- JuliaCI/julia-coverage#v1:
9-
codecov: true
10-
dirs:
11-
- src
12-
- ext
2+
- label: "Triggering Pipelines (Pull Request)"
3+
if: "build.pull_request.base_branch == 'main'"
134
agents:
145
queue: "juliagpu"
15-
cuda: "*"
16-
env:
17-
BACKEND_GROUP: "CUDA"
18-
if: build.message !~ /\[skip tests\]/
19-
timeout_in_minutes: 240
20-
matrix:
21-
setup:
22-
julia:
23-
- "1"
24-
25-
- label: ":julia: Julia: {{matrix.julia}} + AMD GPU"
266
plugins:
27-
- JuliaCI/julia#v1:
28-
version: "{{matrix.julia}}"
29-
- JuliaCI/julia-test#v1:
30-
test_args: "--quickfail"
31-
- JuliaCI/julia-coverage#v1:
32-
codecov: true
33-
dirs:
34-
- src
35-
- ext
36-
env:
37-
JULIA_AMDGPU_CORE_MUST_LOAD: "1"
38-
JULIA_AMDGPU_HIP_MUST_LOAD: "1"
39-
JULIA_AMDGPU_DISABLE_ARTIFACTS: "1"
40-
BACKEND_GROUP: "AMDGPU"
7+
- monebag/monorepo-diff#v2.5.9:
8+
diff: ".buildkite/scripts/diff.sh $BUILDKITE_COMMIT"
9+
interpolation: false
10+
watch:
11+
- path:
12+
- "src/"
13+
- "ext/"
14+
- "test/"
15+
- "Project.toml"
16+
- ".buildkite/"
17+
config:
18+
command: "buildkite-agent pipeline upload .buildkite/testing.yml"
19+
agents:
20+
queue: "juliagpu"
21+
22+
- label: "Triggering Pipelines (Main Branch / Tag)"
23+
if: build.branch == "main" || build.tag != null
4124
agents:
4225
queue: "juliagpu"
43-
rocm: "*"
44-
rocmgpu: "*"
45-
if: build.message !~ /\[skip tests\]/
46-
timeout_in_minutes: 240
47-
matrix:
48-
setup:
49-
julia:
50-
- "1"
51-
52-
env:
53-
RETESTITEMS_NWORKERS: 4
54-
RETESTITEMS_NWORKER_THREADS: 2
55-
SECRET_CODECOV_TOKEN: "vn5M+4wSwUFje6fl6UB/Q/rTmLHu3OlCCMgoPOXPQHYpLZTLz2hOHsV44MadAnxw8MsNVxLKZlXBKqP3IydU9gUfV7QUBtnvbUmIvgUHbr+r0bVaIVVhw6cnd0s8/b+561nU483eRJd35bjYDOlO+V5eDxkbdh/0bzLefXNXy5+ALxsBYzsp75Sx/9nuREfRqWwU6S45mne2ZlwCDpZlFvBDXQ2ICKYXpA45MpxhW9RuqfpQdi6sSR6I/HdHkV2cuJO99dqqh8xfUy6vWPC/+HUVrn9ETsrXtayX1MX3McKj869htGICpR8vqd311HTONYVprH2AN1bJqr5MOIZ8Xg==;U2FsdGVkX1+W55pTI7zq+NwYrbK6Cgqe+Gp8wMCmXY+W10aXTB0bS6zshiDYSQ1Y3piT91xFyNhS+9AsajY0yQ=="
26+
command: "buildkite-agent pipeline upload .buildkite/testing.yml"

‎.buildkite/scripts/diff.sh‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
set -ueo pipefail
3+
4+
# Script to output the diff where the branch was created
5+
# Usage: ./diff.sh $BUILDKITE_COMMIT
6+
7+
COMMIT_HASH=$1
8+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
9+
10+
BRANCH_POINT_COMMIT=$($SCRIPT_DIR/find_branch_point.sh "$COMMIT_HASH")
11+
echo >&2 "Cannot find latest build. Running diff against: $BRANCH_POINT_COMMIT"
12+
diff=$(git diff --name-only "$BRANCH_POINT_COMMIT")
13+
echo "$diff"

‎.buildkite/scripts/find_branch_point.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash
2+
set -ue
3+
4+
diff -u <(git rev-list --first-parent "$1") \
5+
<(git rev-list --first-parent main) | \
6+
sed -ne 's/^ //p' | head -1

‎.buildkite/testing.yml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
steps:
2+
- label: ":julia: Julia {{matrix.julia}} + CUDA GPU"
3+
plugins:
4+
- JuliaCI/julia#v1:
5+
version: "{{matrix.julia}}"
6+
- JuliaCI/julia-test#v1:
7+
test_args: "--quickfail"
8+
- JuliaCI/julia-coverage#v1:
9+
codecov: true
10+
commands: |
11+
printf "[LuxTestUtils]\ntarget_modules = [\"NeuralOperators\", \"Lux\", \"LuxLib\"]\n[LuxLib]\ninstability_check = \"error\"\n[LuxCore]\ninstability_check = \"error\"\n" > LocalPreferences.toml
12+
agents:
13+
queue: "juliagpu"
14+
cuda: "*"
15+
env:
16+
BACKEND_GROUP: "CUDA"
17+
if: build.message !~ /\[skip tests\]/
18+
timeout_in_minutes: 60
19+
matrix:
20+
setup:
21+
julia:
22+
- "1"
23+
24+
- label: ":julia: Julia: {{matrix.julia}} + AMD GPU"
25+
plugins:
26+
- JuliaCI/julia#v1:
27+
version: "{{matrix.julia}}"
28+
- JuliaCI/julia-test#v1:
29+
test_args: "--quickfail"
30+
- JuliaCI/julia-coverage#v1:
31+
codecov: true
32+
commands: |
33+
printf "[LuxTestUtils]\ntarget_modules = [\"NeuralOperators\", \"Lux\", \"LuxLib\"]\n[LuxLib]\ninstability_check = \"error\"\n[LuxCore]\ninstability_check = \"error\"\n" > LocalPreferences.toml
34+
env:
35+
JULIA_AMDGPU_CORE_MUST_LOAD: "1"
36+
JULIA_AMDGPU_HIP_MUST_LOAD: "1"
37+
JULIA_AMDGPU_DISABLE_ARTIFACTS: "1"
38+
BACKEND_GROUP: "AMDGPU"
39+
agents:
40+
queue: "juliagpu"
41+
rocm: "*"
42+
rocmgpu: "*"
43+
if: build.message !~ /\[skip tests\]/
44+
timeout_in_minutes: 60
45+
matrix:
46+
setup:
47+
julia:
48+
- "1"
49+
50+
env:
51+
RETESTITEMS_NWORKERS: 4
52+
RETESTITEMS_NWORKER_THREADS: 2
53+
SECRET_CODECOV_TOKEN: "vn5M+4wSwUFje6fl6UB/Q/rTmLHu3OlCCMgoPOXPQHYpLZTLz2hOHsV44MadAnxw8MsNVxLKZlXBKqP3IydU9gUfV7QUBtnvbUmIvgUHbr+r0bVaIVVhw6cnd0s8/b+561nU483eRJd35bjYDOlO+V5eDxkbdh/0bzLefXNXy5+ALxsBYzsp75Sx/9nuREfRqWwU6S45mne2ZlwCDpZlFvBDXQ2ICKYXpA45MpxhW9RuqfpQdi6sSR6I/HdHkV2cuJO99dqqh8xfUy6vWPC/+HUVrn9ETsrXtayX1MX3McKj869htGICpR8vqd311HTONYVprH2AN1bJqr5MOIZ8Xg==;U2FsdGVkX1+W55pTI7zq+NwYrbK6Cgqe+Gp8wMCmXY+W10aXTB0bS6zshiDYSQ1Y3piT91xFyNhS+9AsajY0yQ=="

‎.github/workflows/CI.yml‎

Lines changed: 93 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,24 +3,48 @@ on:
33
pull_request:
44
branches:
55
- main
6+
paths:
7+
- "src/**"
8+
- "ext/**"
9+
- "test/**"
10+
- "Project.toml"
11+
- ".github/workflows/CI.yml"
612
push:
713
branches:
814
- main
15+
916
concurrency:
1017
# Skip intermediate builds: always.
1118
# Cancel intermediate builds: only if it is a pull request build.
1219
group: ${{ github.workflow }}-${{ github.ref }}
1320
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
21+
1422
jobs:
15-
test:
16-
runs-on: ubuntu-latest
23+
ci:
24+
name: Julia ${{ matrix.version }} - ${{ matrix.os }}
25+
if: ${{ !contains(github.event.head_commit.message, '[skip tests]') }}
26+
runs-on: ${{ matrix.os }}
1727
strategy:
1828
fail-fast: false
1929
matrix:
2030
version:
2131
- "1"
32+
os:
33+
- ubuntu-latest
34+
- macos-latest
35+
- windows-latest
2236
steps:
2337
- uses: actions/checkout@v4
38+
- uses: DamianReeves/write-file-action@master
39+
with:
40+
path: "LocalPreferences.toml"
41+
contents: |
42+
[LuxTestUtils]
43+
target_modules = ["NeuralOperators", "Lux", "LuxLib"]
44+
[LuxLib]
45+
instability_check = "error"
46+
[LuxCore]
47+
instability_check = "error"
2448
- uses: julia-actions/setup-julia@v2
2549
with:
2650
version: ${{ matrix.version }}
@@ -36,16 +60,78 @@ jobs:
3660
${{ runner.os }}-
3761
- uses: julia-actions/julia-buildpkg@v1
3862
- uses: julia-actions/julia-runtest@v1
39-
env:
40-
BACKEND_GROUP: "CPU"
41-
RETESTITEMS_NWORKERS: 4
42-
RETESTITEMS_NWORKER_THREADS: 2
4363
- uses: julia-actions/julia-processcoverage@v1
64+
- uses: codecov/codecov-action@v4
65+
with:
66+
files: lcov.info
67+
token: ${{ secrets.CODECOV_TOKEN }}
68+
verbose: true
69+
fail_ci_if_error: true
70+
71+
downgrade:
72+
if: ${{ !contains(github.event.head_commit.message, '[skip tests]') && github.base_ref == github.event.repository.default_branch }}
73+
name: Downgrade Julia ${{ matrix.version }}
74+
runs-on: ubuntu-latest
75+
strategy:
76+
fail-fast: false
77+
matrix:
78+
version: ["1"]
79+
steps:
80+
- uses: actions/checkout@v4
81+
- uses: DamianReeves/write-file-action@master
4482
with:
45-
directories: src,ext
83+
path: "LocalPreferences.toml"
84+
contents: |
85+
[LuxTestUtils]
86+
target_modules = ["NeuralOperators", "Lux", "LuxLib"]
87+
[LuxLib]
88+
instability_check = "error"
89+
[LuxCore]
90+
instability_check = "error"
91+
- uses: julia-actions/setup-julia@v2
92+
with:
93+
version: ${{ matrix.version }}
94+
- uses: julia-actions/julia-downgrade-compat@v1
95+
- uses: julia-actions/julia-buildpkg@v1
96+
- uses: julia-actions/julia-runtest@v1
97+
- uses: julia-actions/julia-processcoverage@v1
4698
- uses: codecov/codecov-action@v4
4799
with:
48100
files: lcov.info
49101
token: ${{ secrets.CODECOV_TOKEN }}
50102
verbose: true
51103
fail_ci_if_error: true
104+
105+
invalidations:
106+
# Only run on PRs to the default branch.
107+
# In the PR trigger above branches can be specified only explicitly whereas this check should work for master, main, or any other default branch
108+
if: github.base_ref == github.event.repository.default_branch
109+
runs-on: ubuntu-latest
110+
steps:
111+
- uses: julia-actions/setup-julia@v2
112+
with:
113+
version: "1"
114+
- uses: actions/checkout@v4
115+
- uses: julia-actions/julia-buildpkg@v1
116+
- uses: julia-actions/julia-invalidations@v1
117+
id: invs_pr
118+
119+
- uses: actions/checkout@v4
120+
with:
121+
ref: ${{ github.event.repository.default_branch }}
122+
- uses: julia-actions/julia-buildpkg@v1
123+
- uses: julia-actions/julia-invalidations@v1
124+
id: invs_default
125+
126+
- name: Report invalidation counts
127+
run: |
128+
echo "Invalidations on default branch: ${{ steps.invs_default.outputs.total }} (${{ steps.invs_default.outputs.deps }} via deps)" >> $GITHUB_STEP_SUMMARY
129+
echo "This branch: ${{ steps.invs_pr.outputs.total }} (${{ steps.invs_pr.outputs.deps }} via deps)" >> $GITHUB_STEP_SUMMARY
130+
- name: Check if the PR does increase number of invalidations
131+
if: steps.invs_pr.outputs.total > steps.invs_default.outputs.total
132+
run: exit 1
133+
134+
env:
135+
BACKEND_GROUP: "CPU"
136+
RETESTITEMS_NWORKERS: 4
137+
RETESTITEMS_NWORKER_THREADS: 2

‎.github/workflows/Downgrade.yml‎

Lines changed: 0 additions & 41 deletions
This file was deleted.

‎.github/workflows/FormatCheck.yml‎

Lines changed: 0 additions & 9 deletions
This file was deleted.

‎.github/workflows/Invalidations.yml‎

Lines changed: 0 additions & 40 deletions
This file was deleted.

0 commit comments

Comments
 (0)