chore(ci): Combine code coverage in CI before uploading to codecov (#24)

* chore(ci): Combine code coverage in CI before uploading to codecov

* fix(ci): Account for parallel mode creating multiple coverage files

* test: Debug coverage files

* fix: Debug coverage artifacts

* test: Simplify code coverage artifact paths

* test: Inspect generated coverage files as they are missing before upload

* fix: Ensure upload-artifact action includes gitignored `.coverage` file

* test: Debug coverage combining step

* fix: Coverage artifact ID incrementing

* fix: Fail fast and use correct directory when combining coverage files

* fix: Only upload the provided combined coverage file to codecov

* chore(ci): Simplify the logic and error handling for combining coverage

* fix: Debug coverage using nox target

* fix: Lint

* chore: Incorporate PR review suggestions

Author: m1so

.github/workflows/ci.yml

@@ -334,62 +334,99 @@ jobs:
         run: poetry install --no-interaction --only-root
 
       - name: Run unit tests
-        id: test-unit
         env:
           TOOLKIT_VERSION: ${{ steps.version.outputs.VERSION }}
-          PYTHON_VERSION: ${{ matrix.python-version }}
-          COVERAGE_FILE: coverage/.coverage.${{ matrix.python-version }}
         run: |
-          set -euo pipefail
-
-          # Create coverage directory
-          mkdir -p coverage
-
-          # Run tests with coverage
           poetry run pytest tests/unit \
             --cov=deepnote_toolkit \
             --cov=installer \
             --cov=deepnote_core \
             --cov-branch \
             --cov-report=term-missing:skip-covered \
-            --cov-report=xml:coverage/coverage-${PYTHON_VERSION}.xml \
-            --cov-report=json:coverage/coverage-${PYTHON_VERSION}.json \
             --junitxml=junit.xml \
             -o junit_family=legacy
 
-          # Check if coverage file was generated
-          if [ -f "coverage/.coverage.${PYTHON_VERSION}" ]; then
-            echo "coverage_generated=true" >> $GITHUB_OUTPUT
-            echo "Coverage files found:"
-            ls -la coverage/
-          else
-            echo "coverage_generated=false" >> $GITHUB_OUTPUT
-            echo "Warning: No coverage file generated"
-          fi
-
       - name: Per-version coverage summary
-        if: steps.test-unit.outputs.coverage_generated == 'true'
-        env:
-          PYTHON_VERSION: ${{ matrix.python-version }}
         run: |
-          echo "## Python ${PYTHON_VERSION} Coverage" >> $GITHUB_STEP_SUMMARY
-          poetry run coverage report --data-file=coverage/.coverage.${PYTHON_VERSION} --format=markdown >> $GITHUB_STEP_SUMMARY
+          echo "## Python ${{ matrix.python-version }} Coverage" >> $GITHUB_STEP_SUMMARY
+          poetry run coverage report --format=markdown >> $GITHUB_STEP_SUMMARY
 
-      - name: Upload test results to Codecov (these are results not coverage reports)
+      - name: Upload test results to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/test-results-action@47f89e9acb64b76debcd5ea40642d25a4adced9f # v1
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           flags: python-${{ matrix.python-version }}
 
-      - name: Upload coverage to Codecov
+      - name: Upload coverage artifacts
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: coverage-${{ matrix.python-version }}
+          path: .coverage
+          retention-days: 1
+          include-hidden-files: true
+          if-no-files-found: error
+
+  coverage-combine:
+    name: Combine and Upload Coverage
+    runs-on: ubuntu-latest
+    needs: tests-unit
+    if: always()
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: '3.11'
+
+      - name: Install coverage
+        run: pip install coverage[toml]
+
+      - name: Download all coverage artifacts
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
+        with:
+          pattern: coverage-*
+          path: coverage-artifacts/
+
+      - name: Combine coverage files
+        run: |
+          shopt -s nullglob
+          mkdir -p coverage-data
+          
+          i=0
+          for file in coverage-artifacts/*/.coverage; do
+            cp "$file" "coverage-data/.coverage.$i"
+            i=$((i + 1))
+          done
+          
+          coverage combine coverage-data/
+          coverage xml -o coverage-data/coverage.xml
+          coverage report
+
+          echo "## Combined Coverage Report" >> $GITHUB_STEP_SUMMARY
+          coverage report --format=markdown >> $GITHUB_STEP_SUMMARY
+
+      - name: Upload combined coverage to Codecov
         uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          slug: deepnote/deepnote-toolkit
-          files: ./coverage/coverage-${{ matrix.python-version }}.xml
+          slug: ${{ github.repository }}
+          files: ./coverage-data/coverage.xml
+          flags: combined
+          disable_search: true
           fail_ci_if_error: ${{ github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push' }}
 
+      - name: Upload combined coverage report
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: coverage-combined-report
+          path: coverage-data/coverage.xml
+          retention-days: 30
+
   audit-prod:
     name: Audit - Production
     runs-on: ubuntu-latest

noxfile.py

@@ -146,17 +146,54 @@ def coverage_report(session):
 
     import pathlib
 
-    coverage_dir = pathlib.Path("coverage")
+    # Use absolute path relative to session.invoked_from
+    project_root = pathlib.Path(session.invoked_from)
+    coverage_dir = project_root / "coverage"
+
     if not coverage_dir.exists():
-        session.error("No coverage directory found. Run tests with --coverage first.")
+        session.error("No coverage directory found. Run tests with nox -s unit first.")
+
+    # Check if we have a combined coverage file or individual files
+    combined_file = coverage_dir / ".coverage"
+    coverage_files = sorted(coverage_dir.glob(".coverage.*"))
+
+    if not combined_file.exists() and not coverage_files:
+        session.error("No coverage files found. Run tests with nox -s unit first.")
+
+    if coverage_files:
+        session.log(f"Combining {len(coverage_files)} coverage files")
+        session.run(
+            "coverage",
+            "combine",
+            f"--data-file={combined_file}",
+            *[str(f) for f in coverage_files],
+        )
+    else:
+        session.log("Using existing combined coverage file")
 
-    # Combine all coverage files from coverage directory
+    # Generate reports in coverage directory
     session.run(
-        "coverage", "combine", "--data-file=coverage/.coverage", "coverage/.coverage.*"
+        "coverage", "report", f"--data-file={combined_file}", "--format=markdown"
+    )
+    session.run(
+        "coverage",
+        "html",
+        f"--data-file={combined_file}",
+        "-d",
+        str(coverage_dir / "htmlcov"),
+    )
+    session.run(
+        "coverage",
+        "xml",
+        f"--data-file={combined_file}",
+        "-o",
+        str(coverage_dir / "coverage.xml"),
+        "-i",
+    )
+    session.run(
+        "coverage",
+        "json",
+        f"--data-file={combined_file}",
+        "-o",
+        str(coverage_dir / "coverage.json"),
     )
-
-    # Generate reports in coverage directory
-    session.run("coverage", "report", "--format=markdown")
-    session.run("coverage", "html", "-d", "coverage/htmlcov")
-    session.run("coverage", "xml", "-o", "coverage/coverage.xml", "-i")
-    session.run("coverage", "json", "-o", "coverage/coverage.json")