support branch coverage for testing (#24980)

fixes #24976

Author: eleanorjboyd

build/test-requirements.txt

@@ -28,7 +28,6 @@ namedpipe; platform_system == "Windows"
 # typing for Django files
 django-stubs
 
-# for coverage
 coverage
 pytest-cov
 pytest-json

python_files/tests/pytestadapter/test_coverage.py

@@ -5,7 +5,9 @@
 import pathlib
 import sys
 
+import coverage
 import pytest
+from packaging.version import Version
 
 script_dir = pathlib.Path(__file__).parent.parent
 sys.path.append(os.fspath(script_dir))
@@ -34,9 +36,9 @@ def test_simple_pytest_coverage():
     cov_folder_path = TEST_DATA_PATH / "coverage_gen"
     actual = runner_with_cwd_env(args, cov_folder_path, env_add)
     assert actual
-    coverage = actual[-1]
-    assert coverage
-    results = coverage["result"]
+    cov = actual[-1]
+    assert cov
+    results = cov["result"]
     assert results
     assert len(results) == 3
     focal_function_coverage = results.get(os.fspath(TEST_DATA_PATH / "coverage_gen" / "reverse.py"))
@@ -46,6 +48,12 @@ def test_simple_pytest_coverage():
     assert set(focal_function_coverage.get("lines_covered")) == {4, 5, 7, 9, 10, 11, 12, 13, 14, 17}
     assert len(set(focal_function_coverage.get("lines_missed"))) >= 3
 
+    coverage_version = Version(coverage.__version__)
+    # only include check for branches if the version is >= 7.7.0
+    if coverage_version >= Version("7.7.0"):
+        assert focal_function_coverage.get("executed_branches") == 4
+        assert focal_function_coverage.get("total_branches") == 6
+
 
 coverage_gen_file_path = TEST_DATA_PATH / "coverage_gen" / "coverage.json"
 
@@ -77,9 +85,9 @@ def test_coverage_gen_report(cleanup_coverage_gen_file):  # noqa: ARG001
     print("cov_folder_path", cov_folder_path)
     actual = runner_with_cwd_env(args, cov_folder_path, env_add)
     assert actual
-    coverage = actual[-1]
-    assert coverage
-    results = coverage["result"]
+    cov = actual[-1]
+    assert cov
+    results = cov["result"]
     assert results
     assert len(results) == 3
     focal_function_coverage = results.get(os.fspath(TEST_DATA_PATH / "coverage_gen" / "reverse.py"))
@@ -88,6 +96,11 @@ def test_coverage_gen_report(cleanup_coverage_gen_file):  # noqa: ARG001
     assert focal_function_coverage.get("lines_missed") is not None
     assert set(focal_function_coverage.get("lines_covered")) == {4, 5, 7, 9, 10, 11, 12, 13, 14, 17}
     assert set(focal_function_coverage.get("lines_missed")) == {18, 19, 6}
+    coverage_version = Version(coverage.__version__)
+    # only include check for branches if the version is >= 7.7.0
+    if coverage_version >= Version("7.7.0"):
+        assert focal_function_coverage.get("executed_branches") == 4
+        assert focal_function_coverage.get("total_branches") == 6
     # assert that the coverage file was created at the right path
     assert os.path.exists(coverage_gen_file_path)  # noqa: PTH110
 
@@ -123,9 +136,9 @@ def test_coverage_w_omit_config():
     actual = runner_with_cwd_env([], cov_folder_path, env_add)
     assert actual
     print("actual", json.dumps(actual, indent=2))
-    coverage = actual[-1]
-    assert coverage
-    results = coverage["result"]
+    cov = actual[-1]
+    assert cov
+    results = cov["result"]
     assert results
     # assert one file is reported and one file (as specified in pyproject.toml) is omitted
     assert len(results) == 1

python_files/tests/unittestadapter/test_coverage.py

@@ -8,7 +8,9 @@
 import pathlib
 import sys
 
+import coverage
 import pytest
+from packaging.version import Version
 
 sys.path.append(os.fspath(pathlib.Path(__file__).parent))
 
@@ -40,9 +42,9 @@ def test_basic_coverage():
     )
 
     assert actual
-    coverage = actual[-1]
-    assert coverage
-    results = coverage["result"]
+    cov = actual[-1]
+    assert cov
+    results = cov["result"]
     assert results
     assert len(results) == 3
     focal_function_coverage = results.get(os.fspath(TEST_DATA_PATH / "coverage_ex" / "reverse.py"))
@@ -51,6 +53,11 @@ def test_basic_coverage():
     assert focal_function_coverage.get("lines_missed") is not None
     assert set(focal_function_coverage.get("lines_covered")) == {4, 5, 7, 9, 10, 11, 12, 13, 14}
     assert set(focal_function_coverage.get("lines_missed")) == {6}
+    coverage_version = Version(coverage.__version__)
+    # only include check for branches if the version is >= 7.7.0
+    if coverage_version >= Version("7.7.0"):
+        assert focal_function_coverage.get("executed_branches") == 3
+        assert focal_function_coverage.get("total_branches") == 4
 
 
 @pytest.mark.parametrize("manage_py_file", ["manage.py", "old_manage.py"])
@@ -79,9 +86,9 @@ def test_basic_django_coverage(manage_py_file):
     )
 
     assert actual
-    coverage = actual[-1]
-    assert coverage
-    results = coverage["result"]
+    cov = actual[-1]
+    assert cov
+    results = cov["result"]
     assert results
     assert len(results) == 16
     polls_views_coverage = results.get(str(data_path / "polls" / "views.py"))
@@ -90,3 +97,10 @@ def test_basic_django_coverage(manage_py_file):
     assert polls_views_coverage.get("lines_missed") is not None
     assert set(polls_views_coverage.get("lines_covered")) == {3, 4, 6}
     assert set(polls_views_coverage.get("lines_missed")) == {7}
+
+    model_cov = results.get(str(data_path / "polls" / "models.py"))
+    coverage_version = Version(coverage.__version__)
+    # only include check for branches if the version is >= 7.7.0
+    if coverage_version >= Version("7.7.0"):
+        assert model_cov.get("executed_branches") == 1
+        assert model_cov.get("total_branches") == 2

python_files/unittestadapter/execution.py

@@ -12,6 +12,8 @@
 from types import TracebackType
 from typing import Dict, List, Optional, Set, Tuple, Type, Union
 
+from packaging.version import Version
+
 # Adds the scripts directory to the PATH as a workaround for enabling shell for test execution.
 path_var_name = "PATH" if "PATH" in os.environ else "Path"
 os.environ[path_var_name] = (
@@ -316,21 +318,29 @@ def send_run_data(raw_data, test_run_pipe):
     # For unittest COVERAGE_ENABLED is to the root of the workspace so correct data is collected
     cov = None
     is_coverage_run = os.environ.get("COVERAGE_ENABLED") is not None
+    include_branches = False
     if is_coverage_run:
         print(
             "COVERAGE_ENABLED env var set, starting coverage. workspace_root used as parent dir:",
             workspace_root,
         )
         import coverage
 
+        coverage_version = Version(coverage.__version__)
+        # only include branches if coverage version is 7.7.0 or greater (as this was when the api saves)
+        if coverage_version >= Version("7.7.0"):
+            include_branches = True
+
         source_ar: List[str] = []
         if workspace_root:
             source_ar.append(workspace_root)
         if top_level_dir:
             source_ar.append(top_level_dir)
         if start_dir:
             source_ar.append(os.path.abspath(start_dir))  # noqa: PTH100
-        cov = coverage.Coverage(branch=True, source=source_ar)  # is at least 1 of these required??
+        cov = coverage.Coverage(
+            branch=include_branches, source=source_ar
+        )  # is at least 1 of these required??
         cov.start()
 
     # If no error occurred, we will have test ids to run.
@@ -362,12 +372,22 @@ def send_run_data(raw_data, test_run_pipe):
         file_coverage_map: Dict[str, FileCoverageInfo] = {}
         for file in file_set:
             analysis = cov.analysis2(file)
+            taken_file_branches = 0
+            total_file_branches = -1
+
+            if include_branches:
+                branch_stats: dict[int, tuple[int, int]] = cov.branch_stats(file)
+                total_file_branches = sum([total_exits for total_exits, _ in branch_stats.values()])
+                taken_file_branches = sum([taken_exits for _, taken_exits in branch_stats.values()])
+
             lines_executable = {int(line_no) for line_no in analysis[1]}
             lines_missed = {int(line_no) for line_no in analysis[3]}
             lines_covered = lines_executable - lines_missed
             file_info: FileCoverageInfo = {
                 "lines_covered": list(lines_covered),  # list of int
                 "lines_missed": list(lines_missed),  # list of int
+                "executed_branches": taken_file_branches,
+                "total_branches": total_file_branches,
             }
             file_coverage_map[file] = file_info
 

python_files/unittestadapter/pvsc_utils.py

@@ -75,6 +75,8 @@ class ExecutionPayloadDict(TypedDict):
 class FileCoverageInfo(TypedDict):
     lines_covered: List[int]
     lines_missed: List[int]
+    executed_branches: int
+    total_branches: int
 
 
 class CoveragePayloadDict(Dict):

python_files/vscode_pytest/__init__.py

@@ -13,6 +13,7 @@
 from typing import TYPE_CHECKING, Any, Dict, Generator, Literal, TypedDict
 
 import pytest
+from packaging.version import Version
 
 if TYPE_CHECKING:
     from pluggy import Result
@@ -61,6 +62,7 @@ def __init__(self, message):
 collected_tests_so_far = []
 TEST_RUN_PIPE = os.getenv("TEST_RUN_PIPE")
 SYMLINK_PATH = None
+INCLUDE_BRANCHES = False
 
 
 def pytest_load_initial_conftests(early_config, parser, args):  # noqa: ARG001
@@ -70,6 +72,9 @@ def pytest_load_initial_conftests(early_config, parser, args):  # noqa: ARG001
         raise VSCodePytestError(
             "\n \nERROR: pytest-cov is not installed, please install this before running pytest with coverage as pytest-cov is required. \n"
         )
+    if "--cov-branch" in args:
+        global INCLUDE_BRANCHES
+        INCLUDE_BRANCHES = True
 
     global TEST_RUN_PIPE
     TEST_RUN_PIPE = os.getenv("TEST_RUN_PIPE")
@@ -363,6 +368,8 @@ def check_skipped_condition(item):
 class FileCoverageInfo(TypedDict):
     lines_covered: list[int]
     lines_missed: list[int]
+    executed_branches: int
+    total_branches: int
 
 
 def pytest_sessionfinish(session, exitstatus):
@@ -436,6 +443,15 @@ def pytest_sessionfinish(session, exitstatus):
         # load the report and build the json result to return
         import coverage
 
+        coverage_version = Version(coverage.__version__)
+        global INCLUDE_BRANCHES
+        # only include branches if coverage version is 7.7.0 or greater (as this was when the api saves)
+        if coverage_version < Version("7.7.0") and INCLUDE_BRANCHES:
+            print(
+                "Plugin warning[vscode-pytest]: Branch coverage not supported in this coverage versions < 7.7.0. Please upgrade coverage package if you would like to see branch coverage."
+            )
+            INCLUDE_BRANCHES = False
+
         try:
             from coverage.exceptions import NoSource
         except ImportError:
@@ -448,9 +464,8 @@ def pytest_sessionfinish(session, exitstatus):
         file_coverage_map: dict[str, FileCoverageInfo] = {}
 
         # remove files omitted per coverage report config if any
-        omit_files = cov.config.report_omit
-        if omit_files:
-            print("Plugin info[vscode-pytest]: Omit files/rules: ", omit_files)
+        omit_files: list[str] | None = cov.config.report_omit
+        if omit_files is not None:
             for pattern in omit_files:
                 for file in list(file_set):
                     if pathlib.Path(file).match(pattern):
@@ -459,6 +474,18 @@ def pytest_sessionfinish(session, exitstatus):
         for file in file_set:
             try:
                 analysis = cov.analysis2(file)
+                taken_file_branches = 0
+                total_file_branches = -1
+
+                if INCLUDE_BRANCHES:
+                    branch_stats: dict[int, tuple[int, int]] = cov.branch_stats(file)
+                    total_file_branches = sum(
+                        [total_exits for total_exits, _ in branch_stats.values()]
+                    )
+                    taken_file_branches = sum(
+                        [taken_exits for _, taken_exits in branch_stats.values()]
+                    )
+
             except NoSource:
                 # as per issue 24308 this best way to handle this edge case
                 continue
@@ -473,6 +500,8 @@ def pytest_sessionfinish(session, exitstatus):
             file_info: FileCoverageInfo = {
                 "lines_covered": list(lines_covered),  # list of int
                 "lines_missed": list(lines_missed),  # list of int
+                "executed_branches": taken_file_branches,
+                "total_branches": total_file_branches,
             }
             # convert relative path to absolute path
             if not pathlib.Path(file).is_absolute():

python_files/vscode_pytest/run_pytest_script.py

@@ -47,7 +47,7 @@ def run_pytest(args):
                 coverage_enabled = True
                 break
         if not coverage_enabled:
-            args = [*args, "--cov=."]
+            args = [*args, "--cov=.", "--cov-branch"]
 
     run_test_ids_pipe = os.environ.get("RUN_TEST_IDS_PIPE")
     if run_test_ids_pipe:

src/client/testing/testController/common/resultResolver.ts

@@ -17,7 +17,13 @@ import {
     Range,
 } from 'vscode';
 import * as util from 'util';
-import { CoveragePayload, DiscoveredTestPayload, ExecutionTestPayload, ITestResultResolver } from './types';
+import {
+    CoveragePayload,
+    DiscoveredTestPayload,
+    ExecutionTestPayload,
+    FileCoverageMetrics,
+    ITestResultResolver,
+} from './types';
 import { TestProvider } from '../../types';
 import { traceError, traceVerbose } from '../../../logging';
 import { Testing } from '../../../common/utils/localize';
@@ -120,16 +126,25 @@ export class PythonResultResolver implements ITestResultResolver {
         }
         for (const [key, value] of Object.entries(payload.result)) {
             const fileNameStr = key;
-            const fileCoverageMetrics = value;
+            const fileCoverageMetrics: FileCoverageMetrics = value;
             const linesCovered = fileCoverageMetrics.lines_covered ? fileCoverageMetrics.lines_covered : []; // undefined if no lines covered
             const linesMissed = fileCoverageMetrics.lines_missed ? fileCoverageMetrics.lines_missed : []; // undefined if no lines missed
+            const executedBranches = fileCoverageMetrics.executed_branches;
+            const totalBranches = fileCoverageMetrics.total_branches;
 
             const lineCoverageCount = new TestCoverageCount(
                 linesCovered.length,
                 linesCovered.length + linesMissed.length,
             );
+            let fileCoverage: FileCoverage;
             const uri = Uri.file(fileNameStr);
-            const fileCoverage = new FileCoverage(uri, lineCoverageCount);
+            if (totalBranches === -1) {
+                // branch coverage was not enabled and should not be displayed
+                fileCoverage = new FileCoverage(uri, lineCoverageCount);
+            } else {
+                const branchCoverageCount = new TestCoverageCount(executedBranches, totalBranches);
+                fileCoverage = new FileCoverage(uri, lineCoverageCount, branchCoverageCount);
+            }
             runInstance.addCoverage(fileCoverage);
 
             // create detailed coverage array for each file (only line coverage on detailed, not branch)

src/client/testing/testController/common/types.ts

@@ -222,6 +222,8 @@ export type FileCoverageMetrics = {
     lines_covered: number[];
     // eslint-disable-next-line camelcase
     lines_missed: number[];
+    executed_branches: number;
+    total_branches: number;
 };
 
 export type ExecutionTestPayload = {

src/test/testing/common/testingAdapter.test.ts

@@ -711,6 +711,8 @@ suite('End to End Tests: test adapters', () => {
             // since only one test was run, the other test in the same file will have missed coverage lines
             assert.strictEqual(simpleFileCov.lines_covered.length, 3, 'Expected 1 line to be covered in even.py');
             assert.strictEqual(simpleFileCov.lines_missed.length, 1, 'Expected 3 lines to be missed in even.py');
+            assert.strictEqual(simpleFileCov.executed_branches, 1, 'Expected 1 branch to be executed in even.py');
+            assert.strictEqual(simpleFileCov.total_branches, 2, 'Expected 2 branches in even.py');
             return Promise.resolve();
         };
 
@@ -759,6 +761,8 @@ suite('End to End Tests: test adapters', () => {
             // since only one test was run, the other test in the same file will have missed coverage lines
             assert.strictEqual(simpleFileCov.lines_covered.length, 3, 'Expected 1 line to be covered in even.py');
             assert.strictEqual(simpleFileCov.lines_missed.length, 1, 'Expected 3 lines to be missed in even.py');
+            assert.strictEqual(simpleFileCov.executed_branches, 1, 'Expected 1 branch to be executed in even.py');
+            assert.strictEqual(simpleFileCov.total_branches, 2, 'Expected 2 branches in even.py');
 
             return Promise.resolve();
         };