[mypyc] Fail run test if default driver does not find test cases (#19420)

If a `run-` test doesn't define a custom `driver.py`, the default one
looks for functions prefixed `test_` and calls them. If the prefix is
missing or misspelled, or there is some other issue that causes the test
to have no test cases, the test succeeds without running the cases.

To try to prevent this, the default driver will now fail if it doesn't
find any test cases. Existing tests are changed to conform to this
requirement in this PR.

For most tests that meant simply moving the statements from top level to
a function, but some were revealed to have been broken and weren't
actually run, for example because `[typing fixtures/typing-full.pyi]`
was at the top of the test instead of at the bottom, which made the test
setup code ignore all of the test case.

Author: p-sawicki

mypyc/test-data/driver/driver.py

@@ -11,10 +11,12 @@
 import native
 
 failures = []
+tests_run = 0
 
 for name in dir(native):
     if name.startswith('test_'):
         test_func = getattr(native, name)
+        tests_run += 1
         try:
             test_func()
         except Exception as e:
@@ -46,3 +48,5 @@ def extract_line(tb):
     print(f'<< {failures[-1][0]} >>')
     sys.stdout.flush()
     raise failures[-1][1][1]
+
+assert tests_run > 0, 'Default test driver did not find any functions prefixed "test_" to run.'

mypyc/test-data/fixtures/ir.py

@@ -350,7 +350,12 @@ class GeneratorExit(BaseException): pass
 
 def any(i: Iterable[_T]) -> bool: pass
 def all(i: Iterable[_T]) -> bool: pass
-def sum(i: Iterable[_T]) -> int: pass
+@overload
+def sum(i: Iterable[bool]) -> int: pass
+@overload
+def sum(i: Iterable[_T]) -> _T: pass
+@overload
+def sum(i: Iterable[_T], start: _T) -> _T: pass
 def reversed(object: Sequence[_T]) -> Iterator[_T]: ...
 def id(o: object) -> int: pass
 # This type is obviously wrong but the test stubs don't have Sized anymore

mypyc/test-data/run-bools.test

@@ -223,7 +223,8 @@ def test_mixed_comparisons_i64() -> None:
             assert gt_mixed_i64(n, x) == (n > int(x))
 
 [case testBoolMixInt]
-y = False
-print((y or 0) and True)
+def test_mix() -> None:
+    y = False
+    print((y or 0) and True)
 [out]
 0

mypyc/test-data/run-bytes.test

@@ -277,7 +277,6 @@ class bytes_subclass(bytes):
         return b'spook'
 
 [case testBytesFormatting]
-[typing fixtures/typing-full.pyi]
 from testutil import assertRaises
 
 # https://www.python.org/dev/peps/pep-0461/
@@ -314,6 +313,7 @@ def test_bytes_formatting_2() -> None:
     aa = b'\xe4\xbd\xa0\xe5\xa5\xbd%b' % b'\xe4\xbd\xa0\xe5\xa5\xbd'
     assert aa == b'\xe4\xbd\xa0\xe5\xa5\xbd\xe4\xbd\xa0\xe5\xa5\xbd'
     assert aa.decode() == '你好你好'
+[typing fixtures/typing-full.pyi]
 
 
 class A:

mypyc/test-data/run-classes.test

@@ -1278,9 +1278,10 @@ class Bar(Foo):
     def f(self, *args: int, **kwargs: int) -> None:
         print("stuff", args, kwargs)
 
-z: Foo = Bar()
-z.f(1, z=50)
-z.f()
+def test_override() -> None:
+    z: Foo = Bar()
+    z.f(1, z=50)
+    z.f()
 
 [out]
 stuff (1,) {'z': 50}
@@ -1300,18 +1301,19 @@ class Foo:
 def baz_f(self: Any, *args: int, **kwargs: int) -> None:
     print("Baz", args, kwargs)
 
-# Make an "interpreted" subtype of Foo
-type2: Any = type
-Bar = type2('Bar', (Foo,), {})
-Baz = type2('Baz', (Foo,), {'f': baz_f})
+def test_override() -> None:
+    # Make an "interpreted" subtype of Foo
+    type2: Any = type
+    Bar = type2('Bar', (Foo,), {})
+    Baz = type2('Baz', (Foo,), {'f': baz_f})
 
-y: Foo = Bar()
-y.f(1, z=2)
-y.f()
+    y: Foo = Bar()
+    y.f(1, z=2)
+    y.f()
 
-z: Foo = Baz()
-z.f(1, z=2)
-z.f()
+    z: Foo = Baz()
+    z.f(1, z=2)
+    z.f()
 
 [out]
 Foo 1 2
@@ -1330,9 +1332,10 @@ class Bar(Foo):
     def f(self, x: Optional[int]=None) -> None:
         print(x)
 
-z: Foo = Bar()
-z.f(1)
-z.f()
+def test_override() -> None:
+    z: Foo = Bar()
+    z.f(1)
+    z.f()
 
 [out]
 1
@@ -1349,10 +1352,11 @@ class Bar(Foo):
     def f(self, *args: int, **kwargs: int) -> None:
         print("Bar", args, kwargs)
 
-z: Foo = Bar()
-z.f(1, z=2)
-z.f(1, 2, 3)
-# z.f(x=5)  # Not tested because we (knowingly) do the wrong thing and pass it as positional
+def test_override() -> None:
+    z: Foo = Bar()
+    z.f(1, z=2)
+    z.f(1, 2, 3)
+    # z.f(x=5)  # Not tested because we (knowingly) do the wrong thing and pass it as positional
 
 [out]
 Bar (1,) {'z': 2}
@@ -1370,10 +1374,11 @@ class Bar(Foo):
     def f(self, x: int = 10, *args: int, **kwargs: int) -> None:
         print("Bar", x, args, kwargs)
 
-z: Foo = Bar()
-z.f(1, z=2)
-z.f(1, 2, 3)
-z.f()
+def test_override() -> None:
+    z: Foo = Bar()
+    z.f(1, z=2)
+    z.f(1, 2, 3)
+    z.f()
 
 [out]
 Bar 1 () {'z': 2}
@@ -1397,18 +1402,19 @@ class Foo:
 def baz_f(self, a: int=30, y: int=50) -> None:
     print("Baz", a, y)
 
-# Make an "interpreted" subtype of Foo
-type2: Any = type
-Baz = type2('Baz', (Foo,), {'f': baz_f})
+def test_override() -> None:
+    # Make an "interpreted" subtype of Foo
+    type2: Any = type
+    Baz = type2('Baz', (Foo,), {'f': baz_f})
 
-z: Foo = Baz()
-z.f()
-z.f(y=1)
-z.f(1, 2)
-# Not tested because we don't (and probably won't) match cpython here
-# from testutil import assertRaises
-# with assertRaises(TypeError):
-#     z.f(x=7)
+    z: Foo = Baz()
+    z.f()
+    z.f(y=1)
+    z.f(1, 2)
+    # Not tested because we don't (and probably won't) match cpython here
+    # from testutil import assertRaises
+    # with assertRaises(TypeError):
+    #     z.f(x=7)
 
 [out]
 Baz 30 50
@@ -2591,7 +2597,8 @@ class Base:
 class Derived(Base):
     pass
 
-assert Derived()() == 1
+def test_inherited() -> None:
+    assert Derived()() == 1
 
 [case testClassWithFinalAttribute]
 from typing import Final

mypyc/test-data/run-dunders-special.test

@@ -6,4 +6,5 @@ class UsesNotImplemented:
     def __eq__(self, b: object) -> bool:
         return NotImplemented
 
-assert UsesNotImplemented() != object()
+def test_not_implemented() -> None:
+    assert UsesNotImplemented() != object()

mypyc/test-data/run-functions.test

@@ -1235,13 +1235,11 @@ from contextlib import contextmanager
 def f() -> Iterator[None]:
     yield
 
-def g() -> None:
+def test_special_case() -> None:
     a = ['']
     with f():
         a.pop()
 
-g()
-
 [case testUnpackKwargsCompiled]
 from typing import TypedDict
 from typing_extensions import Unpack
@@ -1253,8 +1251,9 @@ class Person(TypedDict):
 def foo(**kwargs: Unpack[Person]) -> None:
     print(kwargs["name"])
 
-# This is not really supported yet, just test that we behave reasonably.
-foo(name='Jennifer', age=38)
+def test_unpack() -> None:
+    # This is not really supported yet, just test that we behave reasonably.
+    foo(name='Jennifer', age=38)
 [typing fixtures/typing-full.pyi]
 [out]
 Jennifer
@@ -1269,8 +1268,9 @@ def foo() -> None:
     print(inner.__dict__)  # type: ignore[attr-defined]
     print(inner.x)  # type: ignore[attr-defined]
 
-if sys.version_info >= (3, 12):  # type: ignore
-    foo()
+def test_nested() -> None:
+    if sys.version_info >= (3, 12):  # type: ignore
+        foo()
 [out]
 [out version>=3.12]
 {}
@@ -1285,7 +1285,8 @@ def bar() -> None:
     functools.update_wrapper(inner, bar)  # type: ignore
     print(inner.__dict__)  # type: ignore
 
-bar()
+def test_update() -> None:
+    bar()
 [typing fixtures/typing-full.pyi]
 [out]
 {'__module__': 'native', '__name__': 'bar', '__qualname__': 'bar', '__doc__': None, '__wrapped__': <built-in function bar>}

mypyc/test-data/run-generators.test

@@ -617,16 +617,22 @@ else:
 from typing import Iterator
 
 class Foo:
-    flag: bool
+    flag = False
 
 class C:
-    foo: Foo
+    foo = Foo()
 
     def genf(self) -> Iterator[None]:
         self.foo.flag = True
         yield
         self.foo.flag = False
 
+def test_near_yield() -> None:
+    c = C()
+    for x in c.genf():
+        pass
+    assert c.foo.flag == False
+
 [case testGeneratorEarlyReturnWithBorrows]
 from typing import Iterator
 class Bar:
@@ -639,6 +645,12 @@ class Foo:
             return
         yield 0
 
+def test_early_return() -> None:
+    foo = Foo()
+    for x in foo.f():
+        pass
+    assert foo.bar.bar == 1
+
 [case testBorrowingInGeneratorInTupleAssignment]
 from typing import Iterator
 

mypyc/test-data/run-generics.test

@@ -27,22 +27,23 @@ def fn_typeddict(t: T) -> None:
     print([x for x in t.keys()])
     print({k: v for k, v in t.items()})
 
-fn_mapping({})
-print("=====")
-fn_mapping({"a": 1, "b": 2})
-print("=====")
+def test_mapping() -> None:
+    fn_mapping({})
+    print("=====")
+    fn_mapping({"a": 1, "b": 2})
+    print("=====")
 
-fn_union({"a": 1, "b": 2})
-print("=====")
-fn_union({"a": "1", "b": "2"})
-print("=====")
+    fn_union({"a": 1, "b": 2})
+    print("=====")
+    fn_union({"a": "1", "b": "2"})
+    print("=====")
 
-orig: Union[Dict[str, int], Dict[str, str]] = {"a": 1, "b": 2}
-fn_union(orig)
-print("=====")
+    orig: Union[Dict[str, int], Dict[str, str]] = {"a": 1, "b": 2}
+    fn_union(orig)
+    print("=====")
 
-td: TD = {"foo": 1}
-fn_typeddict(td)
+    td: TD = {"foo": 1}
+    fn_typeddict(td)
 [typing fixtures/typing-full.pyi]
 [out]
 \[]
@@ -96,8 +97,9 @@ def deco(func: Callable[P, int]) -> Callable[P, int]:
 def f(x: int, y: str) -> int:
     return x
 
-assert f(1, 'a') == 1
-assert f(2, y='b') == 2
+def test_usable() -> None:
+    assert f(1, 'a') == 1
+    assert f(2, y='b') == 2
 [out]
 \[1, 'a']
 {}

mypyc/test-data/run-imports.test

@@ -212,9 +212,10 @@ import shared
 def do_import() -> None:
     import a
 
-assert shared.counter == 0
-do_import()
-assert shared.counter == 1
+def test_lazy() -> None:
+    assert shared.counter == 0
+    do_import()
+    assert shared.counter == 1
 
 [file a.py]
 import shared
@@ -224,9 +225,10 @@ shared.counter += 1
 counter = 0
 
 [case testDelayedImport]
-import a
-print("inbetween")
-import b
+def test_delayed() -> None:
+    import a
+    print("inbetween")
+    import b
 
 [file a.py]
 print("first")
@@ -240,19 +242,21 @@ inbetween
 last
 
 [case testImportErrorLineNumber]
-try:
-    import enum
-    import dataclasses, missing  # type: ignore[import]
-except ImportError as e:
-    line = e.__traceback__.tb_lineno # type: ignore[attr-defined]
-    assert line == 3, f"traceback's line number is {line}, expected 3"
+def test_error() -> None:
+    try:
+        import enum
+        import dataclasses, missing  # type: ignore[import]
+    except ImportError as e:
+        line = e.__traceback__.tb_lineno # type: ignore[attr-defined]
+        assert line == 4, f"traceback's line number is {line}, expected 4"
 
 [case testImportGroupIsolation]
 def func() -> None:
     import second
 
-import first
-func()
+def test_isolation() -> None:
+    import first
+    func()
 
 [file first.py]
 print("first")