(very) initial steps on conformance tests

Author: carljm

conformance/results/mypy/overloads_basic.toml

@@ -1,12 +1,21 @@
-conformant = "Pass"
+conformant = "Partial"
+notes = """
+Does not allow an overload with no implementation in an abstract base class.
+"""
 output = """
-overloads_basic.py:37: error: No overload variant of "__getitem__" of "Bytes" matches argument type "str"  [call-overload]
-overloads_basic.py:37: note: Possible overload variants:
-overloads_basic.py:37: note:     def __getitem__(self, int, /) -> int
-overloads_basic.py:37: note:     def __getitem__(self, slice[Any, Any, Any], /) -> bytes
-overloads_basic.py:62: error: Single overload definition, multiple required  [misc]
-overloads_basic.py:74: error: An overloaded function outside a stub file must have an implementation  [no-overload-impl]
+overloads_basic.py:41: error: No overload variant of "__getitem__" of "Bytes" matches argument type "str"  [call-overload]
+overloads_basic.py:41: note: Possible overload variants:
+overloads_basic.py:41: note:     def __getitem__(self, int, /) -> int
+overloads_basic.py:41: note:     def __getitem__(self, slice[Any, Any, Any], /) -> bytes
+overloads_basic.py:66: error: Single overload definition, multiple required  [misc]
+overloads_basic.py:78: error: An overloaded function outside a stub file must have an implementation  [no-overload-impl]
+overloads_basic.py:101: error: An overloaded function outside a stub file must have an implementation  [no-overload-impl]
+overloads_basic.py:116: error: Overload does not consistently use the "@staticmethod" decorator on all function signatures.  [misc]
+overloads_basic.py:126: error: Overloaded function implementation does not accept all possible arguments of signature 1  [misc]
+overloads_basic.py:126: error: Overloaded function implementation does not accept all possible arguments of signature 2  [misc]
+overloads_basic.py:129: error: Overload does not consistently use the "@classmethod" decorator on all function signatures.  [misc]
 """
-conformance_automated = "Pass"
+conformance_automated = "Fail"
 errors_diff = """
+Line 101: Unexpected errors ['overloads_basic.py:101: error: An overloaded function outside a stub file must have an implementation  [no-overload-impl]']
 """

conformance/results/mypy/version.toml

@@ -1,2 +1,2 @@
-version = "mypy 1.14.0"
-test_duration = 1.6
+version = "mypy 1.14.1"
+test_duration = 1.7

conformance/results/pyre/overloads_basic.toml

@@ -1,11 +1,22 @@
-conformant = "Pass"
+conformant = "Partial"
 notes = """
+Does not allow an overload with no implementation in a Protocol or an abstract base class.
 """
 output = """
-overloads_basic.py:37:2 Incompatible parameter type [6]: In call `Bytes.__getitem__`, for 1st positional argument, expected `int` but got `str`.
-overloads_basic.py:63:0 Incompatible overload [43]: At least two overload signatures must be present.
-overloads_basic.py:75:0 Missing overload implementation [42]: Overloaded function `func2` must have an implementation.
+overloads_basic.py:41:2 Incompatible parameter type [6]: In call `Bytes.__getitem__`, for 1st positional argument, expected `int` but got `str`.
+overloads_basic.py:67:0 Incompatible overload [43]: At least two overload signatures must be present.
+overloads_basic.py:79:0 Missing overload implementation [42]: Overloaded function `func2` must have an implementation.
+overloads_basic.py:92:4 Missing overload implementation [42]: Overloaded function `MyProto.func3` must have an implementation.
+overloads_basic.py:102:4 Missing overload implementation [42]: Overloaded function `MyAbstractBase.func4` must have an implementation.
+overloads_basic.py:118:4 Incompatible overload [43]: The implementation of `C.func5` does not accept all possible arguments of overload defined on line `118`.
+overloads_basic.py:123:4 Incompatible overload [43]: The implementation of `C.func5` does not accept all possible arguments of overload defined on line `123`.
+overloads_basic.py:126:4 Incompatible overload [43]: This definition does not have the same decorators as the preceding overload(s).
+overloads_basic.py:131:4 Incompatible overload [43]: The implementation of `C.func6` does not accept all possible arguments of overload defined on line `131`.
+overloads_basic.py:136:4 Incompatible overload [43]: The implementation of `C.func6` does not accept all possible arguments of overload defined on line `136`.
+overloads_basic.py:139:4 Incompatible overload [43]: This definition does not have the same decorators as the preceding overload(s).
 """
-conformance_automated = "Pass"
+conformance_automated = "Fail"
 errors_diff = """
+Line 92: Unexpected errors ['overloads_basic.py:92:4 Missing overload implementation [42]: Overloaded function `MyProto.func3` must have an implementation.']
+Line 102: Unexpected errors ['overloads_basic.py:102:4 Missing overload implementation [42]: Overloaded function `MyAbstractBase.func4` must have an implementation.']
 """

conformance/results/pyre/version.toml

@@ -1,2 +1,2 @@
 version = "pyre 0.9.23"
-test_duration = 7.3
+test_duration = 5.9

conformance/results/pyright/overloads_basic.toml

@@ -1,11 +1,31 @@
-conformant = "Pass"
+conformant = "Partial"
+notes = """
+Does not allow an overload with no implementation in an abstract base class.
+"""
 output = """
-overloads_basic.py:37:1 - error: No overloads for "__getitem__" match the provided arguments (reportCallIssue)
-overloads_basic.py:37:1 - error: Argument of type "Literal['']" cannot be assigned to parameter "__s" of type "slice[Any, Any, Any]" in function "__getitem__"
+overloads_basic.py:41:1 - error: No overloads for "__getitem__" match the provided arguments (reportCallIssue)
+overloads_basic.py:41:1 - error: Argument of type "Literal['']" cannot be assigned to parameter "__s" of type "slice[Any, Any, Any]" in function "__getitem__"
   "Literal['']" is not assignable to "slice[Any, Any, Any]" (reportArgumentType)
-overloads_basic.py:63:5 - error: "func1" is marked as overload, but additional overloads are missing (reportInconsistentOverload)
-overloads_basic.py:75:5 - error: "func2" is marked as overload, but no implementation is provided (reportNoOverloadImplementation)
+overloads_basic.py:67:5 - error: "func1" is marked as overload, but additional overloads are missing (reportInconsistentOverload)
+overloads_basic.py:79:5 - error: "func2" is marked as overload, but no implementation is provided (reportNoOverloadImplementation)
+overloads_basic.py:102:9 - error: "func4" is marked as overload, but no implementation is provided (reportNoOverloadImplementation)
+overloads_basic.py:118:9 - error: Overloads for "func5" use @staticmethod inconsistently (reportInconsistentOverload)
+overloads_basic.py:126:9 - error: Overloaded implementation is not consistent with signature of overload 1
+  Type "(self: Self@C, x: int | str) -> (int | str)" is not assignable to type "(x: int) -> int"
+    Parameter name mismatch: "x" versus "self"
+    Parameter 1: type "int" is incompatible with type "Self@C"
+      Type "int" is not assignable to type "Self@C"
+    Extra parameter "x" (reportInconsistentOverload)
+overloads_basic.py:126:9 - error: Overloaded implementation is not consistent with signature of overload 2
+  Type "(self: Self@C, x: int | str) -> (int | str)" is not assignable to type "(x: str) -> str"
+    Parameter name mismatch: "x" versus "self"
+    Parameter 1: type "str" is incompatible with type "Self@C"
+      Type "str" is not assignable to type "Self@C"
+    Extra parameter "x" (reportInconsistentOverload)
+overloads_basic.py:131:9 - error: Overloads for "func6" use @classmethod inconsistently (reportInconsistentOverload)
+overloads_basic.py:139:15 - warning: Instance methods should take a "self" parameter (reportSelfClsParameterName)
 """
-conformance_automated = "Pass"
+conformance_automated = "Fail"
 errors_diff = """
+Line 102: Unexpected errors ['overloads_basic.py:102:9 - error: "func4" is marked as overload, but no implementation is provided (reportNoOverloadImplementation)']
 """

conformance/results/pyright/version.toml

@@ -1,2 +1,2 @@
 version = "pyright 1.1.391"
-test_duration = 1.2
+test_duration = 1.5

conformance/results/pytype/overloads_basic.toml

@@ -2,28 +2,57 @@ conformant = "Partial"
 notes = """
 Does not reject a function with a single @overload signature.
 Does not reject a function with @overload signature but no implementation.
+Does not allow an overload with no implementation in a Protocol or an abstract base class.
+Does not exempt overloads from checking of return type in body, when also decorated with `@staticmethod`.
+Does not error on overloads inconsistently decorated with `@staticmethod` or `@classmethod`.
 """
 output = """
-overloads_basic.py:31:20: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in __getitem__: bad return type [bad-return-type]
+overloads_basic.py:35:20: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in __getitem__: bad return type [bad-return-type]
 
             return b""
                    \u001b[1m\u001b[31m~~~\u001b[39m\u001b[0m
 
-overloads_basic.py:37:1: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: unsupported operand type(s) for item retrieval: Bytes and str [unsupported-operands]
+overloads_basic.py:41:1: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: unsupported operand type(s) for item retrieval: Bytes and str [unsupported-operands]
 
 b[""]  # E: no matching overload
 \u001b[1m\u001b[31m~~~~~\u001b[39m\u001b[0m
 
-overloads_basic.py:58:5: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in map: bad return type [bad-return-type]
+overloads_basic.py:62:5: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in map: bad return type [bad-return-type]
 
     pass
     \u001b[1m\u001b[31m~~~~\u001b[39m\u001b[0m
 
+overloads_basic.py:98:9: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in func3: @typing.overload-decorated 'MyProto.func3' object is not callable [not-callable]
+
+        ...
+        \u001b[1m\u001b[31m~~~\u001b[39m\u001b[0m
+
+overloads_basic.py:108:9: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in func4: @typing.overload-decorated 'MyAbstractBase.func4' object is not callable [not-callable]
+
+        ...
+        \u001b[1m\u001b[31m~~~\u001b[39m\u001b[0m
+
+overloads_basic.py:119:9: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in func5: bad return type [bad-return-type]
+
+        ...
+        \u001b[1m\u001b[31m~~~\u001b[39m\u001b[0m
+
+overloads_basic.py:124:9: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in func5: bad return type [bad-return-type]
+
+        ...
+        \u001b[1m\u001b[31m~~~\u001b[39m\u001b[0m
+
 """
 conformance_automated = "Fail"
 errors_diff = """
-Lines 62, 63: Expected error (tag 'func1')
-Lines 74, 75: Expected error (tag 'func2')
-Line 31: Unexpected errors ['overloads_basic.py:31:20: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in __getitem__: bad return type [bad-return-type]']
-Line 58: Unexpected errors ['overloads_basic.py:58:5: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in map: bad return type [bad-return-type]']
+Lines 66, 67: Expected error (tag 'func1')
+Lines 78, 79: Expected error (tag 'func2')
+Lines 116, 118, 123, 126: Expected error (tag 'func5')
+Lines 129, 131, 136, 139: Expected error (tag 'func6')
+Line 35: Unexpected errors ['overloads_basic.py:35:20: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in __getitem__: bad return type [bad-return-type]']
+Line 62: Unexpected errors ['overloads_basic.py:62:5: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in map: bad return type [bad-return-type]']
+Line 98: Unexpected errors ["overloads_basic.py:98:9: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in func3: @typing.overload-decorated 'MyProto.func3' object is not callable [not-callable]"]
+Line 108: Unexpected errors ["overloads_basic.py:108:9: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in func4: @typing.overload-decorated 'MyAbstractBase.func4' object is not callable [not-callable]"]
+Line 119: Unexpected errors ['overloads_basic.py:119:9: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in func5: bad return type [bad-return-type]']
+Line 124: Unexpected errors ['overloads_basic.py:124:9: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in func5: bad return type [bad-return-type]']
 """

conformance/results/pytype/version.toml

@@ -1,2 +1,2 @@
 version = "pytype 2024.10.11"
-test_duration = 37.2
+test_duration = 30.3

conformance/results/results.html

@@ -158,17 +158,17 @@ <h3>Python Type System Conformance Test Results</h3>
         </header>
         <div class="table_container"><table><tbody>
 <tr><th class="col1">&nbsp;</th>
-<th class='tc-header'><div class='tc-name'>mypy 1.14.0</div>
-<div class='tc-time'>1.6sec</div>
+<th class='tc-header'><div class='tc-name'>mypy 1.14.1</div>
+<div class='tc-time'>1.7sec</div>
 </th>
 <th class='tc-header'><div class='tc-name'>pyright 1.1.391</div>
-<div class='tc-time'>1.2sec</div>
+<div class='tc-time'>1.5sec</div>
 </th>
 <th class='tc-header'><div class='tc-name'>pyre 0.9.23</div>
-<div class='tc-time'>7.3sec</div>
+<div class='tc-time'>5.9sec</div>
 </th>
 <th class='tc-header'><div class='tc-name'>pytype 2024.10.11</div>
-<div class='tc-time'>37.2sec</div>
+<div class='tc-time'>30.3sec</div>
 </th>
 </tr>
 <tr><th class="column" colspan="5">
@@ -667,10 +667,10 @@ <h3>Python Type System Conformance Test Results</h3>
 <a class="test_group" href="https://typing.readthedocs.io/en/latest/spec/overload.html">Overloads</a>
 </th></tr>
 <tr><th class="column col1">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;overloads_basic</th>
-<th class="column col2 conformant">Pass</th>
-<th class="column col2 conformant">Pass</th>
-<th class="column col2 conformant">Pass</th>
-<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not reject a function with a single @overload signature.</p><p>Does not reject a function with @overload signature but no implementation.</p></span></div></th>
+<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not allow an overload with no implementation in an abstract base class.</p></span></div></th>
+<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not allow an overload with no implementation in an abstract base class.</p></span></div></th>
+<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not allow an overload with no implementation in a Protocol or an abstract base class.</p></span></div></th>
+<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not reject a function with a single @overload signature.</p><p>Does not reject a function with @overload signature but no implementation.</p><p>Does not allow an overload with no implementation in a Protocol or an abstract base class.</p><p>Does not exempt overloads from checking of return type in body, when also decorated with `@staticmethod`.</p><p>Does not error on overloads inconsistently decorated with `@staticmethod`.</p></span></div></th>
 </tr>
 <tr><th class="column" colspan="5">
 <a class="test_group" href="https://typing.readthedocs.io/en/latest/spec/exceptions.html">Exceptions</a>

conformance/tests/overloads_basic.py

@@ -1,16 +1,20 @@
 """
-Tests the basic typing.overload behavior described in PEP 484.
+Tests the behavior of typing.overload.
 """
 
 # Specification: https://typing.readthedocs.io/en/latest/spec/overload.html#overload
 
-# Note: The behavior of @overload is severely under-specified by PEP 484 leading
-# to significant divergence in behavior across type checkers. This is something
-# we will likely want to address in a future update to the typing spec. For now,
-# this conformance test will cover only the most basic functionality described
-# in PEP 484.
-
-from typing import Any, Callable, Iterable, Iterator, TypeVar, assert_type, overload
+from abc import ABC
+from typing import (
+    Any,
+    Callable,
+    Iterable,
+    Iterator,
+    Protocol,
+    TypeVar,
+    assert_type,
+    overload,
+)
 
 
 class Bytes:
@@ -58,7 +62,7 @@ def map(func: Any, iter1: Any, iter2: Any = ...) -> Any:
     pass
 
 
-# At least two overload signatures should be provided.
+# > At least two @overload-decorated definitions must be present.
 @overload  # E[func1]
 def func1() -> None:  # E[func1]: At least two overloads must be present
     ...
@@ -68,9 +72,9 @@ def func1() -> None:
     pass
 
 
-# > In regular modules, a series of @overload-decorated definitions must be
-# > followed by exactly one non-@overload-decorated definition (for the same
-# > function/method).
+# > The ``@overload``-decorated definitions must be followed by an overload
+# > implementation, which does not include an ``@overload`` decorator. Type
+# > checkers should report an error or warning if an implementation is missing.
 @overload  # E[func2]
 def func2(x: int) -> int:  # E[func2]: no implementation
     ...
@@ -79,3 +83,58 @@ def func2(x: int) -> int:  # E[func2]: no implementation
 @overload
 def func2(x: str) -> str:
     ...
+
+
+# > Overload definitions within stub files, protocols, and abstract base classes
+# > are exempt from this check.
+class MyProto(Protocol):
+    @overload
+    def func3(self, x: int) -> int:
+        ...
+
+
+    @overload
+    def func3(self, x: str) -> str:
+        ...
+
+class MyAbstractBase(ABC):
+    @overload
+    def func4(self, x: int) -> int:
+        ...
+
+
+    @overload
+    def func4(self, x: str) -> str:
+        ...
+
+
+# > If one overload signature is decorated with ``@staticmethod`` or
+# > ``@classmethod``, all overload signatures must be similarly decorated. The
+# > implementation, if present, must also have a consistent decorator. Type
+# > checkers should report an error if these conditions are not met.
+class C:
+    @overload  # E[func5]
+    @staticmethod
+    def func5(x: int) -> int:  # E[func5]
+        ...
+
+    @overload
+    @staticmethod
+    def func5(x: str) -> str:  # E[func5]
+        ...
+
+    def func5(self, x: int | str) -> int | str:  # E[func5]
+        return 1
+
+    @overload  # E[func6]
+    @classmethod
+    def func6(cls, x: int) -> int:  # E[func6]
+        ...
+
+    @overload
+    @classmethod
+    def func6(cls, x: str) -> str:  # E[func6]
+        ...
+
+    def func6(cls, x: int | str) -> int | str:  # E[func6]
+        return 1