Add code coverage combination and reporting (#1674)

SUMMARY:
Update code coverage support to include results in the job summaries, as
well as combine the results from the multiple jobs in the
"test-checks.yaml" workflow.

Some open questions:
* Should the reporting in the summary skip empty files (files with no
Python code/statements)?
  * Current: yes
* Should the reporting use different types of sorting (e.g., by
coverage)?
  * Current: file path/name (tool default)
* Should the GitHub jobs have coverage summaries for intermediate jobs
or just the combined results?
  * Current: all (though it does seem more verbose than is useful)
* Note: The coverage result artifacts will still be attached to the jobs
even without summaries, they are currently required for the job that
combines the results.
* Should coverage be enabled by default for eithe of the workflows (here
or another PR)?
* Current: no (adds ~1 minute to base test run wall time and ~6-7
minutes to transformers run time)

TEST PLAN:
For examples, check these runs and the summary for each job in the run:

* base:
https://github.com/vllm-project/llm-compressor/actions/runs/16471546227
(49% combined coverage)
* transformers:
https://github.com/vllm-project/llm-compressor/actions/runs/16472609090
(65% coverage)

Signed-off-by: Domenic Barbuzzi <[email protected]>

Author: dbarbuzzi

.github/workflows/test-check-transformers.yaml

@@ -122,4 +122,9 @@ jobs:
             .coverage
             coverage-html
             coverage.json
+          include-hidden-files: true
           retention-days: 5
+      - name: "Report coverage"
+        if: (success() || failure()) && inputs.code_coverage
+        run: |
+          coverage report --data-file=".coverage" --skip-empty --format="markdown" > "$GITHUB_STEP_SUMMARY"

.github/workflows/test-check.yaml

@@ -18,6 +18,8 @@ jobs:
 
   base-tests:
     runs-on: ubuntu-22.04
+    env:
+      COVERAGE_FILE: ".coverage.base"
     steps:
       - uses: actions/setup-python@v5
         with:
@@ -53,13 +55,20 @@ jobs:
         with:
           name: base-tests-coverage-results
           path: |
-            .coverage
+            .coverage*
             coverage-html
             coverage.json
+          include-hidden-files: true
           retention-days: 5
+      - name: "Report coverage"
+        if: (success() || failure()) && inputs.code_coverage
+        run: |
+          coverage report --data-file="$COVERAGE_FILE" --skip-empty --format="markdown" > "$GITHUB_STEP_SUMMARY"
 
   pytorch-tests:
     runs-on: ubuntu-22.04
+    env:
+      COVERAGE_FILE: ".coverage.pytorch"
     steps:
       - uses: actions/setup-python@v5
         with:
@@ -96,14 +105,20 @@ jobs:
         with:
           name: pytorch-tests-coverage-results
           path: |
-            .coverage
+            .coverage*
             coverage-html
             coverage.json
+          include-hidden-files: true
           retention-days: 5
-
+      - name: "Report coverage"
+        if: (success() || failure()) && inputs.code_coverage
+        run: |
+          coverage report --data-file="$COVERAGE_FILE" --skip-empty --format="markdown" > "$GITHUB_STEP_SUMMARY"
 
   compat-pytorch-1_9-pytorch-tests:
     runs-on: ubuntu-22.04
+    env:
+      COVERAGE_FILE: ".coverage.compat-pytorch-1.9"
     steps:
       - uses: actions/setup-python@v5
         with:
@@ -139,8 +154,60 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: compat-pytorch-tests-coverage-results
+          path: |
+            .coverage*
+            coverage-html
+            coverage.json
+          include-hidden-files: true
+          retention-days: 5
+      - name: "Report coverage"
+        if: (success() || failure()) && inputs.code_coverage
+        run: |
+          coverage report --data-file="$COVERAGE_FILE" --skip-empty --format="markdown" > "$GITHUB_STEP_SUMMARY"
+
+  combine-coverage:
+    runs-on: ubuntu-22.04
+    needs: [base-tests, pytorch-tests, compat-pytorch-1_9-pytorch-tests]
+    if: (success() || failure()) && inputs.code_coverage
+    steps:
+      - name: "Checkout llm-compressor"
+        uses: actions/checkout@v4
+
+      - name: "Download coverage artifacts"
+        uses: actions/download-artifact@v4
+        with:
+          merge-multiple: true
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: "Install dependencies"
+        run: |
+          pip3 install -U pip setuptools
+          pip3 install coverage setuptools-scm
+          make build  # need to build to generate the version.py file
+
+      - name: "Combine and report coverage"
+        run: |
+          cat << EOF > .coveragerc
+          [paths]
+          source =
+              src/
+              */site-packages/
+          EOF
+          coverage combine
+          coverage report --skip-empty --format="markdown" >> "$GITHUB_STEP_SUMMARY"
+          coverage html --directory coverage-html
+          coverage json -o coverage.json
+
+      - name: "Upload coverage report"
+        uses: actions/upload-artifact@v4
+        with:
+          name: combined-coverage-results
           path: |
             .coverage
             coverage-html
             coverage.json
+          include-hidden-files: true
           retention-days: 5