Skip to content

Commit 41eb4df

Browse files
marcromeynmarcromeyn
authored
Merge branch 'main' into torch/agg
2 parents 8aeb11e + 004ff77 commit 41eb4df

28 files changed

+841
-101
lines changed

.github/workflows/set-stable-branch.yaml renamed to .github/workflows/branches.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
name: Set Stable Branch
1+
name: branches
22

33
on:
44
workflow_dispatch:
55
release:
66
types: [published, deleted]
77

88
jobs:
9-
set-stable-branch:
9+
set-stable:
1010
uses: NVIDIA-Merlin/.github/.github/workflows/set-stable-branch.yaml@main

‎.github/workflows/check-base-branch.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
name: Require Development Base Branch
1+
name: branches
22

33
on:
44
pull_request:
55
types: [synchronize, opened, reopened, labeled, unlabeled]
66

77
jobs:
8-
check:
8+
require-base-branch:
99
uses: NVIDIA-Merlin/.github/.github/workflows/check-base-branch.yml@main

‎.github/workflows/cpu-horovod.yml‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: horovod (cpu)
1+
name: horovod
22

33
on:
44
push:
@@ -11,7 +11,29 @@ concurrency:
1111
cancel-in-progress: true
1212

1313
jobs:
14+
check-changes:
15+
runs-on: ubuntu-latest
16+
steps:
17+
- uses: actions/checkout@v3
18+
- name: Set up Python
19+
uses: actions/setup-python@v4
20+
with:
21+
python-version: 3.8
22+
- name: Install dependencies
23+
run: |
24+
python -m pip install --upgrade pip
25+
pip install GitPython
26+
pip install . --no-deps
27+
- name: Get changed backends
28+
id: backend_check
29+
run: |
30+
echo "changed=$(python ci/get_changed_backends.py --backend tensorflow --branch ${{github.base_ref}})" >> $GITHUB_OUTPUT
31+
outputs:
32+
needs_testing: ${{ steps.backend_check.outputs.changed }}
33+
1434
tests:
35+
needs: check-changes
36+
if: ${{needs.check-changes.outputs.needs_testing == 'true' || github.ref == 'refs/heads/main'}}
1537
runs-on: ${{ matrix.os }}
1638
strategy:
1739
matrix:
@@ -45,9 +67,13 @@ jobs:
4567
run: |
4668
ref_type=${{ github.ref_type }}
4769
branch=main
70+
pr_flag="false"
4871
if [[ $ref_type == "tag"* ]]
4972
then
5073
git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +refs/heads/release*:refs/remotes/origin/release*
5174
branch=$(git branch -r --contains ${{ github.ref_name }} --list '*release*' --format "%(refname:short)" | sed -e 's/^origin\///')
5275
fi
53-
cd ${{ github.workspace }}; tox -e py38-horovod-cpu -- $branch
76+
if [[ "${{ github.ref }}" != 'refs/heads/main' ]]; then
77+
extra_pytest_markers="and changed"
78+
fi
79+
cd ${{ github.workspace }}; EXTRA_PYTEST_MARKERS=$extra_pytest_markers MERLIN_BRANCH=$branch COMPARE_BRANCH=${{ github.base_ref }} tox -e py38-horovod-cpu

‎.github/workflows/cpu-nvtabular.yml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,29 @@ concurrency:
1414
cancel-in-progress: true
1515

1616
jobs:
17+
check-changes:
18+
runs-on: ubuntu-latest
19+
steps:
20+
- uses: actions/checkout@v3
21+
- name: Set up Python
22+
uses: actions/setup-python@v4
23+
with:
24+
python-version: 3.8
25+
- name: Install dependencies
26+
run: |
27+
python -m pip install --upgrade pip
28+
pip install GitPython
29+
pip install . --no-deps
30+
- name: Get changed backends
31+
id: backend_check
32+
run: |
33+
echo "changed=$(python ci/get_changed_backends.py --backend datasets --branch ${{github.base_ref}})" >> $GITHUB_OUTPUT
34+
outputs:
35+
needs_testing: ${{ steps.backend_check.outputs.changed }}
36+
1737
tests:
38+
needs: check-changes
39+
if: ${{needs.check-changes.outputs.needs_testing == 'true' || github.ref == 'refs/heads/main'}}
1840
runs-on: ${{ matrix.os }}
1941
strategy:
2042
matrix:

‎.github/workflows/cpu-systems.yml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,29 @@ concurrency:
1414
cancel-in-progress: true
1515

1616
jobs:
17+
check-changes:
18+
runs-on: ubuntu-latest
19+
steps:
20+
- uses: actions/checkout@v3
21+
- name: Set up Python
22+
uses: actions/setup-python@v4
23+
with:
24+
python-version: 3.8
25+
- name: Install dependencies
26+
run: |
27+
python -m pip install --upgrade pip
28+
pip install GitPython
29+
pip install . --no-deps
30+
- name: Get changed backends
31+
id: backend_check
32+
run: |
33+
echo "changed=$(python ci/get_changed_backends.py --backend datasets --branch ${{github.base_ref}})" >> $GITHUB_OUTPUT
34+
outputs:
35+
needs_testing: ${{ steps.backend_check.outputs.changed }}
36+
1737
tests:
38+
needs: check-changes
39+
if: ${{needs.check-changes.outputs.needs_testing == 'true' || github.ref == 'refs/heads/main'}}
1840
runs-on: ${{ matrix.os }}
1941
strategy:
2042
matrix:

‎.github/workflows/cpu-t4r.yml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,29 @@ on:
1010
branches: [ main ]
1111

1212
jobs:
13+
check-changes:
14+
runs-on: ubuntu-latest
15+
steps:
16+
- uses: actions/checkout@v3
17+
- name: Set up Python
18+
uses: actions/setup-python@v4
19+
with:
20+
python-version: 3.8
21+
- name: Install dependencies
22+
run: |
23+
python -m pip install --upgrade pip
24+
pip install GitPython
25+
pip install . --no-deps
26+
- name: Get changed backends
27+
id: backend_check
28+
run: |
29+
echo "changed=$(python ci/get_changed_backends.py --backend 'datasets|torch' --branch ${{github.base_ref}})" >> $GITHUB_OUTPUT
30+
outputs:
31+
needs_testing: ${{ steps.backend_check.outputs.changed }}
32+
1333
tests:
34+
needs: check-changes
35+
if: ${{needs.check-changes.outputs.needs_testing == 'true' || github.ref == 'refs/heads/main'}}
1436
runs-on: ${{ matrix.os }}
1537
strategy:
1638
matrix:

‎.github/workflows/datasets.yml‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,29 @@ concurrency:
1111
cancel-in-progress: true
1212

1313
jobs:
14+
check-changes:
15+
runs-on: ubuntu-latest
16+
steps:
17+
- uses: actions/checkout@v3
18+
- name: Set up Python
19+
uses: actions/setup-python@v4
20+
with:
21+
python-version: 3.8
22+
- name: Install dependencies
23+
run: |
24+
python -m pip install --upgrade pip
25+
pip install GitPython
26+
pip install . --no-deps
27+
- name: Get changed backends
28+
id: backend_check
29+
run: |
30+
echo "changed=$(python ci/get_changed_backends.py --backend datasets --branch ${{github.base_ref}})" >> $GITHUB_OUTPUT
31+
outputs:
32+
needs_testing: ${{ steps.backend_check.outputs.changed }}
33+
1434
tests:
35+
needs: check-changes
36+
if: ${{needs.check-changes.outputs.needs_testing == 'true' || github.ref == 'refs/heads/main'}}
1537
runs-on: ${{ matrix.os }}
1638
strategy:
1739
matrix:
@@ -51,4 +73,8 @@ jobs:
5173
python setup.py develop
5274
- name: Run unittests
5375
run: |
54-
make tests-datasets
76+
if [[ "${{ github.ref }}" == 'refs/heads/main' ]]; then
77+
make tests-datasets
78+
else
79+
make tests-datasets-changed
80+
fi

‎.github/workflows/gpu-multi.yml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
name: gpu-multi
2+
3+
on:
4+
workflow_dispatch:
5+
push:
6+
branches: [ main ]
7+
tags:
8+
- "v[0-9]+.[0-9]+.[0-9]+"
9+
pull_request:
10+
branches: [ main ]
11+
types: [opened, synchronize, reopened]
12+
13+
concurrency:
14+
group: ${{ github.workflow }}-${{ github.ref }}
15+
cancel-in-progress: true
16+
17+
jobs:
18+
check-changes-tf:
19+
runs-on: ubuntu-latest
20+
steps:
21+
- uses: actions/checkout@v3
22+
- name: Set up Python
23+
uses: actions/setup-python@v4
24+
with:
25+
python-version: 3.8
26+
- name: Install dependencies
27+
run: |
28+
python -m pip install --upgrade pip
29+
pip install GitPython
30+
pip install . --no-deps
31+
- name: Get changed backends
32+
id: backend_check
33+
run: |
34+
echo "changed=$(python ci/get_changed_backends.py --backend tensorflow --branch ${{github.base_ref}})" >> $GITHUB_OUTPUT
35+
outputs:
36+
needs_testing: ${{ steps.backend_check.outputs.changed }}
37+
38+
tensorflow:
39+
needs: check-changes-tf
40+
if: ${{needs.check-changes-tf.outputs.needs_testing == 'true' || github.ref == 'refs/heads/main'}}
41+
runs-on: 2GPU
42+
43+
steps:
44+
- uses: actions/checkout@v3
45+
with:
46+
fetch-depth: 0
47+
- name: Run tests
48+
run: |
49+
ref_type=${{ github.ref_type }}
50+
branch=main
51+
pr_flag="false"
52+
if [[ $ref_type == "tag"* ]]
53+
then
54+
git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +refs/heads/release*:refs/remotes/origin/release*
55+
branch=$(git branch -r --contains ${{ github.ref_name }} --list '*release*' --format "%(refname:short)" | sed -e 's/^origin\///')
56+
fi
57+
if [[ "${{ github.ref }}" != 'refs/heads/main' ]]; then
58+
extra_pytest_markers="and changed"
59+
fi
60+
cd ${{ github.workspace }}; EXTRA_PYTEST_MARKERS=$extra_pytest_markers MERLIN_BRANCH=$branch COMPARE_BRANCH=${{ github.base_ref }} tox -e py38-multi-gpu

.github/workflows/gpu-ci.yml renamed to .github/workflows/gpu.yml

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: GPU CI
1+
name: gpu-ci
22

33
on:
44
workflow_dispatch:
@@ -16,7 +16,7 @@ concurrency:
1616

1717
jobs:
1818
gpu-ci:
19-
runs-on: 2GPU
19+
runs-on: 1GPU
2020

2121
steps:
2222
- uses: actions/checkout@v3
@@ -26,9 +26,13 @@ jobs:
2626
run: |
2727
ref_type=${{ github.ref_type }}
2828
branch=main
29+
pr_flag="false"
2930
if [[ $ref_type == "tag"* ]]
3031
then
3132
git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +refs/heads/release*:refs/remotes/origin/release*
3233
branch=$(git branch -r --contains ${{ github.ref_name }} --list '*release*' --format "%(refname:short)" | sed -e 's/^origin\///')
3334
fi
34-
cd ${{ github.workspace }}; tox -e py38-gpu -- $branch
35+
if [[ "${{ github.ref }}" != 'refs/heads/main' ]]; then
36+
extra_pytest_markers="and changed"
37+
fi
38+
cd ${{ github.workspace }}; EXTRA_PYTEST_MARKERS=$extra_pytest_markers MERLIN_BRANCH=$branch COMPARE_BRANCH=${{ github.base_ref }} tox -e py38-gpu

‎.github/workflows/implicit.yml‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,29 @@ concurrency:
1111
cancel-in-progress: true
1212

1313
jobs:
14+
check-changes:
15+
runs-on: ubuntu-latest
16+
steps:
17+
- uses: actions/checkout@v3
18+
- name: Set up Python
19+
uses: actions/setup-python@v4
20+
with:
21+
python-version: 3.8
22+
- name: Install dependencies
23+
run: |
24+
python -m pip install --upgrade pip
25+
pip install GitPython
26+
pip install . --no-deps
27+
- name: Get changed backends
28+
id: backend_check
29+
run: |
30+
echo "changed=$(python ci/get_changed_backends.py --backend implicit --branch ${{github.base_ref}})" >> $GITHUB_OUTPUT
31+
outputs:
32+
needs_testing: ${{ steps.backend_check.outputs.changed }}
33+
1434
tests:
35+
needs: check-changes
36+
if: ${{needs.check-changes.outputs.needs_testing == 'true' || github.ref == 'refs/heads/main'}}
1537
runs-on: ${{ matrix.os }}
1638
strategy:
1739
matrix:
@@ -51,4 +73,8 @@ jobs:
5173
python setup.py develop
5274
- name: Run unittests
5375
run: |
54-
make tests-implicit
76+
if [[ "${{ github.ref }}" == 'refs/heads/main' ]]; then
77+
make tests-implicit
78+
else
79+
make tests-implicit-changed
80+
fi

0 commit comments

Comments
 (0)