[stubtest] Verify __all__ exists in stub

Closes: #13300

Author: srittau

mypy/stubtest.py

@@ -348,6 +348,8 @@ def verify_mypyfile(
             # Only verify the contents of the stub's __all__
             # if the stub actually defines __all__
             yield from _verify_exported_names(object_path, stub, runtime_all_as_set)
+        else:
+            yield Error(object_path + ["__all__"], "is not present in stub", stub, runtime)
     else:
         runtime_all_as_set = None
 

mypy/test/teststubtest.py

@@ -169,12 +169,14 @@ def run_stubtest_with_stderr(
     filtered_output = remove_color_code(
         output.getvalue()
         # remove cwd as it's not available from outside
-        .replace(os.path.realpath(tmp_dir) + os.sep, "").replace(tmp_dir + os.sep, "")
+        .replace(os.path.realpath(tmp_dir) + os.sep, "")
+        .replace(tmp_dir + os.sep, "")
     )
     filtered_outerr = remove_color_code(
         outerr.getvalue()
         # remove cwd as it's not available from outside
-        .replace(os.path.realpath(tmp_dir) + os.sep, "").replace(tmp_dir + os.sep, "")
+        .replace(os.path.realpath(tmp_dir) + os.sep, "")
+        .replace(tmp_dir + os.sep, "")
     )
     return filtered_output, filtered_outerr
 
@@ -1403,7 +1405,7 @@ def test_all_at_runtime_not_stub(self) -> Iterator[Case]:
             runtime="""
             __all__ = []
             Z = 5""",
-            error=None,
+            error="__all__",
         )
 
     @collect_cases
@@ -1443,7 +1445,7 @@ def h(x: str): ...
             runtime="",
             error="h",
         )
-        yield Case(stub="", runtime="__all__ = []", error=None)  # dummy case
+        yield Case(stub="", runtime="__all__ = []", error="__all__")  # dummy case
         yield Case(stub="", runtime="__all__ += ['y']\ny = 5", error="y")
         yield Case(stub="", runtime="__all__ += ['g']\ndef g(): pass", error="g")
         # Here we should only check that runtime has B, since the stub explicitly re-exports it
@@ -2435,11 +2437,6 @@ def test_output(self) -> None:
         assert output == expected
 
     def test_ignore_flags(self) -> None:
-        output = run_stubtest(
-            stub="", runtime="__all__ = ['f']\ndef f(): pass", options=["--ignore-missing-stub"]
-        )
-        assert output == "Success: no issues found in 1 module\n"
-
         output = run_stubtest(stub="", runtime="def f(): pass", options=["--ignore-missing-stub"])
         assert output == "Success: no issues found in 1 module\n"
 
@@ -2488,18 +2485,14 @@ def test_allowlist(self) -> None:
                     def good() -> None: ...
                     def bad(number: int) -> None: ...
                     def also_bad(number: int) -> None: ...
-                    """.lstrip(
-                        "\n"
-                    )
+                    """.lstrip("\n")
                 ),
                 runtime=textwrap.dedent(
                     """
                     def good(): pass
                     def bad(asdf): pass
                     def also_bad(asdf): pass
-                    """.lstrip(
-                        "\n"
-                    )
+                    """.lstrip("\n")
                 ),
                 options=["--allowlist", allowlist.name, "--generate-allowlist"],
             )