Add flag for expected test failures, so we can get rid of the nasty seperate pytest run and stuff. (#56)

Author: brunns

.github/workflows/test.yml

@@ -26,9 +26,7 @@ jobs:
         pip install -U pip setuptools wheel
         pip install -r dev-requirements.txt
     - name: Run tests
-      run: pytest --ignore-glob="*.shouldfail.yml"
-    - name: Run test with expected failures
-      run: pytest pytest_mypy_plugins/tests/*.shouldfail.yml 2>&1 | grep "5 failed"
+      run: pytest 
 
   lint:
     runs-on: ubuntu-latest

pytest_mypy_plugins/collect.py

@@ -101,6 +101,7 @@ def collect(self) -> Iterator["YamlTestItem"]:
                 test_name = f"{test_name_prefix}{test_name_suffix}"
                 main_file = File(path="main.py", content=pystache.render(raw_test["main"], params))
                 test_files = [main_file] + parse_test_files(raw_test.get("files", []))
+                expect_fail = raw_test.get("expect_fail", False)
                 regex = raw_test.get("regex", False)
 
                 expected_output = []
@@ -130,6 +131,7 @@ def collect(self) -> Iterator["YamlTestItem"]:
                         expected_output=expected_output,
                         parsed_test_data=raw_test,
                         mypy_config=additional_mypy_config,
+                        expect_fail=expect_fail,
                     )
 
     def _eval_skip(self, skip_if: str) -> bool:

pytest_mypy_plugins/item.py

@@ -130,11 +130,13 @@ def __init__(
         disable_cache: bool,
         mypy_config: str,
         parsed_test_data: Dict[str, Any],
+        expect_fail: bool,
     ) -> None:
         super().__init__(name, parent, config)
         self.files = files
         self.environment_variables = environment_variables
         self.disable_cache = disable_cache
+        self.expect_fail = expect_fail
         self.expected_output = expected_output
         self.starting_lineno = starting_lineno
         self.additional_mypy_config = mypy_config
@@ -280,7 +282,15 @@ def runtest(self) -> None:
                 for line in mypy_output.splitlines():
                     output_line = replace_fpath_with_module_name(line, rootdir=execution_path)
                     output_lines.append(output_line)
-                assert_expected_matched_actual(expected=self.expected_output, actual=output_lines)
+                try:
+                    assert_expected_matched_actual(expected=self.expected_output, actual=output_lines)
+                except TypecheckAssertionError as e:
+                    if not self.expect_fail:
+                        raise e
+                else:
+                    if self.expect_fail:
+                        raise TypecheckAssertionError("Expected failure, but test passed")
+
         finally:
             temp_dir.cleanup()
             # remove created modules and all their dependants from cache

pytest_mypy_plugins/tests/test-regex_assertions.shouldfail.yml

@@ -20,3 +20,23 @@
   main: |
     a = 'hello'
     reveal_type(a)  # NR: .*str.*
+
+- case: rexex_but_not_turned_on
+  expect_fail: yes
+  main: |
+    a = 'hello'
+    reveal_type(a)  # N: .*str.*
+
+- case: rexex_but_turned_off
+  expect_fail: yes
+  regex: no
+  main: |
+    a = 'hello'
+    reveal_type(a)  # N: .*str.*
+
+- case: regext_does_not_match
+  expect_fail: yes
+  regex: no
+  main: |
+    a = 'hello'
+    reveal_type(a)  # NR: .*banana.*

pytest_mypy_plugins/tests/test-regex_assertions.yml

@@ -88,3 +88,17 @@
   main: |
     a = 1
     a.lower()  # E: "int" has no attribute "lower"
+
+- case: fail_if_message_does_not_match
+  expect_fail: yes
+  main: |
+    a = 'hello'
+    reveal_type(a)  # N: Some other message
+
+- case: fail_if_message_from_outdoes_not_match
+  expect_fail: yes
+  main: |
+    a = 'abc'
+    reveal_type(a)
+  out: |
+    main:2: note: Some other message