Improve CI

.github/codecov.yml

@@ -0,0 +1,8 @@
+# Allow atmost 5% coverage drop from main branch.
+coverage:
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 5%
+    patch: false

.github/workflows/_wait_for_checks.yml

@@ -0,0 +1,29 @@
+name: Wait for checks
+
+on:
+  workflow_call:
+    inputs:
+      match_pattern:
+        required: true
+        type: string
+      delay:
+        required: false
+        type: string
+        default: 10s
+
+jobs:
+  wait:
+    runs-on: ubuntu-latest
+    permissions:
+      checks: read
+    steps:
+      - name: Wait for checks (PRs only)
+        if: github.event_name == 'pull_request' || startsWith(github.ref, 'refs/heads/pull-request/')
+        uses: poseidon/[email protected]
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          match_pattern: ${{ inputs.match_pattern }}
+          delay: ${{ inputs.delay }}
+      - name: No-op for non-PR events
+        if: github.event_name != 'pull_request' && !startsWith(github.ref, 'refs/heads/pull-request/')
+        run: echo "Not a pull_request event"

.github/workflows/gpu_tests.yml

@@ -3,22 +3,47 @@ name: GPU tests
 on:
   push:
     branches: ["pull-request/[0-9]+"]
-    # TODO: paths cannot be used since push happens to copied PR and only latest commit to PR is used
-    # paths:
-    #   - ".github/workflows/gpu_tests.yml"
-    #   - "modelopt/**"
-    #   - "tests/gpu/**"
-    #   - "pyproject.toml"
-    #   - "setup.py"
-    #   - "tox.ini"
+    # NOTE: paths cannot be used since push happens to copied PR and only latest commit to PR is used
 
 # Cancel previous runs if new commit is pushed to the same PR
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+  group: ${{ github.workflow }}-${{ github.sha }}
   cancel-in-progress: true
 
 jobs:
+  check-file-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      any_changed: ${{ steps.changed-tests.outputs.any_changed }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: get-pr-info
+        uses: nv-gha-runners/get-pr-info@main
+      - name: Check for changes in test-relevant directories
+        id: changed-tests
+        uses: step-security/[email protected]
+        with:
+          files: |
+            .github/workflows/gpu_tests.yml
+            modelopt/**
+            tests/gpu/**
+            tox.ini
+            pyproject.toml
+            setup.py
+          base_sha: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.ref }}
+  wait-checks:
+    needs: [check-file-changes]
+    if: needs.check-file-changes.outputs.any_changed == 'true'
+    uses: ./.github/workflows/_wait_for_checks.yml
+    permissions:
+      checks: read
+    secrets: inherit
+    with:
+      match_pattern: '^DCO$|^linux$' # Wait for DCO and Unit tests / linux to pass
+      delay: 300s
   gpu-tests:
+    needs: [check-file-changes, wait-checks]
+    if: needs.check-file-changes.outputs.any_changed == 'true'
     # Runner list at https://github.com/nv-gha-runners/enterprise-runner-configuration/blob/main/docs/runner-groups.md
     runs-on: linux-amd64-gpu-h100-latest-1
     timeout-minutes: 60
@@ -30,7 +55,6 @@ jobs:
         PIP_CONSTRAINT: "" # Disable pip constraint for upgrading packages
     steps:
       - uses: actions/checkout@v4
-      - name: Setup proxy cache
-        uses: nv-gha-runners/setup-proxy-cache@main
+      - uses: nv-gha-runners/setup-proxy-cache@main
       - name: Run gpu tests
         run: pip install tox-current-env && tox -e py312-cuda12-gpu --current-env

.github/workflows/pages.yml

@@ -8,7 +8,7 @@ on:
 
 # Cancel previous runs if new commit is pushed
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
   cancel-in-progress: true
 
 # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages

.github/workflows/unit_tests.yml

@@ -22,11 +22,19 @@ on:
 
 # Cancel previous runs if new commit is pushed
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
   cancel-in-progress: true
 
 jobs:
-  unit-tests:
+  check-dco:
+    uses: ./.github/workflows/_wait_for_checks.yml
+    permissions:
+      checks: read
+    secrets: inherit
+    with:
+      match_pattern: '^DCO$'
+  linux:
+    needs: [check-dco]
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
@@ -42,8 +50,9 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: true
           verbose: true
-  unit-tests-windows:
+  windows:
     if: github.event_name == 'pull_request'
+    needs: [linux]
     runs-on: windows-latest
     timeout-minutes: 30
     steps:
@@ -53,3 +62,48 @@ jobs:
           python-version: "3.12"
       - name: Run unit tests (without coverage)
         run: pip install tox && tox -e py312-torch28-unit
+  multi-py:
+    if: github.event_name == 'pull_request'
+    needs: [linux]
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        py: [10, 11]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.${{ matrix.py }}"
+      - name: Run unit tests
+        run: pip install tox && tox -e py3${{ matrix.py }}-torch28-unit
+  multi-torch:
+    if: github.event_name == 'pull_request'
+    needs: [linux]
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        torch: [25, 26, 27]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Run unit tests
+        run: pip install tox && tox -e py312-torch${{ matrix.torch }}-unit
+  partial-install:
+    if: github.event_name == 'pull_request'
+    needs: [linux]
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        test-env: [onnx, torch]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Run unit tests
+        run: pip install tox && tox -e py312-partial-unit-${{ matrix.test-env }}