support pytest-subtests plugin

Author: eleanorjboyd

build/test-requirements.txt

@@ -36,6 +36,7 @@ pytest-json
 
 # for pytest-describe related tests
 pytest-describe
+pytest-subtests
 
 # for pytest-ruff related tests
 pytest-ruff

python_files/tests/pytestadapter/.data/test_pytest_subtests_plugin.py

@@ -0,0 +1,8 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+def test_a(subtests):
+    with subtests.test(msg="test_a"):
+        assert 1 == 1
+    with subtests.test(msg="Second subtest"):
+        assert 2 == 1

python_files/tests/pytestadapter/expected_execution_test_output.py

@@ -6,6 +6,7 @@
 TEST_ADD_FUNCTION = "unittest_folder/test_add.py::TestAddFunction::"
 SUCCESS = "success"
 FAILURE = "failure"
+SUBTEST_FAILURE = "subtest-failure"
 
 # This is the expected output for the unittest_folder execute tests
 # └── unittest_folder
@@ -735,6 +736,41 @@
     },
 }
 
+# This is the expected output for the test_pytest_subtests_plugin.py file.
+# └── test_pytest_subtests_plugin.py
+#    └── test_a
+#       └── test_a [test_a]: subtest-success
+#       └── test_a [Second subtest]: subtest-failure
+test_pytest_subtests_plugin_path = TEST_DATA_PATH / "test_pytest_subtests_plugin.py"
+pytest_subtests_plugin_expected_execution_output = {
+    get_absolute_test_id(
+        "test_pytest_subtests_plugin.py::test_a**{test_a [test_a]}**",
+        test_pytest_subtests_plugin_path,
+    ): {
+        "test": get_absolute_test_id(
+            "test_pytest_subtests_plugin.py::test_a**{test_a [test_a]}**",
+            test_pytest_subtests_plugin_path,
+        ),
+        "outcome": "subtest-success",
+        "message": None,
+        "traceback": None,
+        "subtest": None,
+    },
+    get_absolute_test_id(
+        "test_pytest_subtests_plugin.py::test_a**{test_a [Second subtest]}**",
+        test_pytest_subtests_plugin_path,
+    ): {
+        "test": get_absolute_test_id(
+            "test_pytest_subtests_plugin.py::test_a**{test_a [Second subtest]}**",
+            test_pytest_subtests_plugin_path,
+        ),
+        "outcome": SUBTEST_FAILURE,
+        "message": "ERROR MESSAGE",
+        "traceback": None,
+        "subtest": None,
+    },
+}
+
 skip_test_fixture_path = TEST_DATA_PATH / "skip_test_fixture.py"
 skip_test_fixture_execution_expected_output = {
     get_absolute_test_id("skip_test_fixture.py::test_docker_client", skip_test_fixture_path): {

python_files/tests/pytestadapter/test_execution.py

@@ -232,6 +232,34 @@ def test_pytest_execution(test_ids, expected_const):
     assert actual_result_dict == expected_const
 
 
+def test_pytest_subtest_plugin():
+    test_subtest_plugin_path = TEST_DATA_PATH / "test_pytest_subtests_plugin.py"
+
+    test_a_id = get_absolute_test_id(
+        "test_subtest_plugin_path::test_a",
+        test_subtest_plugin_path,
+    )
+    args = [test_a_id]
+    expected_const = expected_execution_test_output.pytest_subtests_plugin_expected_execution_output
+    actual = runner(args)
+    assert actual
+    actual_list: List[Dict[str, Dict[str, Any]]] = actual
+    assert len(actual_list) == len(expected_const)
+    actual_result_dict = {}
+    if actual_list is not None:
+        for actual_item in actual_list:
+            assert all(item in actual_item for item in ("status", "cwd", "result"))
+            assert actual_item.get("status") == "success"
+            assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH)
+            actual_result_dict.update(actual_item["result"])
+    for key in actual_result_dict:
+        if actual_result_dict[key]["outcome"] == "subtest-failure":
+            actual_result_dict[key]["message"] = "ERROR MESSAGE"
+        if actual_result_dict[key]["traceback"] is not None:
+            actual_result_dict[key]["traceback"] = "TRACEBACK"
+    assert actual_result_dict == expected_const
+
+
 def test_symlink_run():
     """Test to test pytest discovery with the command line arg --rootdir specified as a symlink path.
 

python_files/vscode_pytest/__init__.py

@@ -271,8 +271,19 @@ def pytest_report_teststatus(report, config):  # noqa: ARG001
             node_path = map_id_to_path[report.nodeid]
         except KeyError:
             node_path = cwd
-        # Calculate the absolute test id and use this as the ID moving forward.
+
         absolute_node_id = get_absolute_test_id(report.nodeid, node_path)
+        parent_test_name = report.nodeid.split("::")[-1]
+        if report.head_line and report.head_line != parent_test_name:
+            # add parent node to collected_tests_so_far to not double report
+            collected_tests_so_far.append(absolute_node_id)
+            # If the report has a head_line, then it is a pytest-subtest
+            # and we need to adjust the nodeid to reflect the subtest.
+            if report_value == "failure":
+                report_value = "subtest-failure"
+            elif report_value == "success":
+                report_value = "subtest-success"
+            absolute_node_id = absolute_node_id + "**{" + report.head_line + "}**"
         if absolute_node_id not in collected_tests_so_far:
             collected_tests_so_far.append(absolute_node_id)
             item_result = create_test_outcome(

src/client/testing/testController/common/resultResolver.ts

@@ -246,7 +246,7 @@ export class PythonResultResolver implements ITestResultResolver {
                     }
                 } else if (testItem.outcome === 'subtest-failure') {
                     // split on [] or () based on how the subtest is setup.
-                    const [parentTestCaseId, subtestId] = splitTestNameWithRegex(keyTemp);
+                    const [parentTestCaseId, subtestId] = splitTestNameWithRegex(keyTemp, this.testProvider);
                     const parentTestItem = this.runIdToTestItem.get(parentTestCaseId);
                     const data = testItem;
                     // find the subtest's parent test item
@@ -288,7 +288,7 @@ export class PythonResultResolver implements ITestResultResolver {
                     }
                 } else if (testItem.outcome === 'subtest-success') {
                     // split on [] or () based on how the subtest is setup.
-                    const [parentTestCaseId, subtestId] = splitTestNameWithRegex(keyTemp);
+                    const [parentTestCaseId, subtestId] = splitTestNameWithRegex(keyTemp, this.testProvider);
                     const parentTestItem = this.runIdToTestItem.get(parentTestCaseId);
 
                     // find the subtest's parent test item

src/client/testing/testController/common/utils.ts

@@ -18,6 +18,7 @@ import {
 import { Deferred, createDeferred } from '../../../common/utils/async';
 import { createReaderPipe, generateRandomPipeName } from '../../../common/pipes/namedPipes';
 import { EXTENSION_ROOT_DIR } from '../../../constants';
+import { TestProvider } from '../../types';
 
 export function fixLogLinesNoTrailing(content: string): string {
     const lines = content.split(/\r?\n/g);
@@ -272,10 +273,15 @@ export function createDiscoveryErrorPayload(
  * @param testName The full test name string.
  * @returns A tuple where the first item is the parent test name and the second item is the subtest section or `testName` if no subtest section exists.
  */
-export function splitTestNameWithRegex(testName: string): [string, string] {
+export function splitTestNameWithRegex(testName: string, testProvider: TestProvider): [string, string] {
     // If a match is found, return the parent test name and the subtest (whichever was captured between parenthesis or square brackets).
     // Otherwise, return the entire testName for the parent and entire testName for the subtest.
-    const regex = /^(.*?) ([\[(].*[\])])$/;
+    let regex: RegExp;
+    if (testProvider === 'pytest') {
+        regex = /^(.*?)\*\*{(.*?)}\*\*$/;
+    } else {
+        regex = /^(.*?) ([\[(].*[\])])$/;
+    }
     const match = testName.match(regex);
     if (match) {
         return [match[1].trim(), match[2] || match[3] || testName];

src/test/testing/testController/utils.unit.test.ts

@@ -1,5 +1,5 @@
-// // Copyright (c) Microsoft Corporation. All rights reserved.
-// // Licensed under the MIT License.
+// // // Copyright (c) Microsoft Corporation. All rights reserved.
+// // // Licensed under the MIT License.
 
 // import * as assert from 'assert';
 // import {
@@ -106,7 +106,7 @@
 //         assert.deepStrictEqual(rpcContent.remainingRawData, '');
 //     });
 
-//     suite('Test Controller Utils: Other', () => {
+//     suite('Test Controller Utils: Unittest', () => {
 //         interface TestCase {
 //             name: string;
 //             input: string;