add initial overloads_consistency tests

Author: carljm

conformance/results/mypy/overloads_consistency.toml

@@ -0,0 +1,8 @@
+conformant = "Pass"
+conformance_automated = "Pass"
+errors_diff = """
+"""
+output = """
+overloads_consistency.py:27: error: Overloaded function implementation cannot produce return type of signature 2  [misc]
+overloads_consistency.py:42: error: Overloaded function implementation does not accept all possible arguments of signature 2  [misc]
+"""

conformance/results/pyre/overloads_consistency.toml

@@ -0,0 +1,8 @@
+conformant = "Pass"
+conformance_automated = "Pass"
+errors_diff = """
+"""
+output = """
+overloads_consistency.py:24:0 Incompatible overload [43]: The return type of overloaded function `return_type` (`str`) is incompatible with the return type of the implementation (`int`).
+overloads_consistency.py:39:0 Incompatible overload [43]: The implementation of `parameter_type` does not accept all possible arguments of overload defined on line `39`.
+"""

conformance/results/pyre/version.toml

@@ -1,2 +1,2 @@
 version = "pyre 0.9.23"
-test_duration = 6.7
+test_duration = 6.6

conformance/results/pyright/overloads_consistency.toml

@@ -0,0 +1,13 @@
+conformant = "Pass"
+conformance_automated = "Pass"
+errors_diff = """
+"""
+output = """
+overloads_consistency.py:27:5 - error: Overloaded implementation is not consistent with signature of overload 2
+  Function return type "str" is incompatible with type "int"
+    "str" is not assignable to "int" (reportInconsistentOverload)
+overloads_consistency.py:42:5 - error: Overloaded implementation is not consistent with signature of overload 2
+  Type "(x: int) -> (int | str)" is not assignable to type "(x: str) -> str"
+    Parameter 1: type "str" is incompatible with type "int"
+      "str" is not assignable to "int" (reportInconsistentOverload)
+"""

conformance/results/pytype/overloads_consistency.toml

@@ -0,0 +1,23 @@
+conformant = "Fail"
+notes = """
+Doesn't appear to validate overload consistency at all.
+"""
+conformance_automated = "Fail"
+errors_diff = """
+Lines 24, 27: Expected error (tag 'return_type')
+Lines 39, 42: Expected error (tag 'parameter_type')
+Line 28: Unexpected errors ['overloads_consistency.py:28:12: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in return_type: bad return type [bad-return-type]']
+Line 43: Unexpected errors ['overloads_consistency.py:43:12: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in parameter_type: bad return type [bad-return-type]']
+"""
+output = """
+overloads_consistency.py:28:12: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in return_type: bad return type [bad-return-type]
+
+    return 1
+           \u001b[1m\u001b[31m~\u001b[39m\u001b[0m
+
+overloads_consistency.py:43:12: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in parameter_type: bad return type [bad-return-type]
+
+    return 1
+           \u001b[1m\u001b[31m~\u001b[39m\u001b[0m
+
+"""

conformance/results/pytype/overloads_definitions.toml

@@ -139,16 +139,16 @@ class Child(Base):  # E[override-final]
 \u001b[1m\u001b[31m\u001b[39m\u001b[0m
     @overload # E[override-final]
 \u001b[1m\u001b[31m~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\u001b[39m\u001b[0m
-    def final_method(self, x: int) -> int:  # E[override-final]
-\u001b[1m\u001b[31m~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\u001b[39m\u001b[0m
+    def final_method(self, x: int) -> int:
+\u001b[1m\u001b[31m~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\u001b[39m\u001b[0m
         ...
 \u001b[1m\u001b[31m~~~~~~~~~~~\u001b[39m\u001b[0m
 
 \u001b[1m\u001b[31m\u001b[39m\u001b[0m
     @overload
 \u001b[1m\u001b[31m~~~~~~~~~~~~~\u001b[39m\u001b[0m
-    def final_method(self, x: str) -> str:  # E[override-final]
-\u001b[1m\u001b[31m~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\u001b[39m\u001b[0m
+    def final_method(self, x: str) -> str:
+\u001b[1m\u001b[31m~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\u001b[39m\u001b[0m
         ...
 \u001b[1m\u001b[31m~~~~~~~~~~~\u001b[39m\u001b[0m
 

conformance/results/pytype/version.toml

@@ -1,2 +1,2 @@
 version = "pytype 2024.10.11"
-test_duration = 30.0
+test_duration = 30.3

conformance/results/results.html

@@ -165,10 +165,10 @@ <h3>Python Type System Conformance Test Results</h3>
 <div class='tc-time'>1.5sec</div>
 </th>
 <th class='tc-header'><div class='tc-name'>pyre 0.9.23</div>
-<div class='tc-time'>6.7sec</div>
+<div class='tc-time'>6.6sec</div>
 </th>
 <th class='tc-header'><div class='tc-name'>pytype 2024.10.11</div>
-<div class='tc-time'>30.0sec</div>
+<div class='tc-time'>30.3sec</div>
 </th>
 </tr>
 <tr><th class="column" colspan="5">
@@ -672,6 +672,12 @@ <h3>Python Type System Conformance Test Results</h3>
 <th class="column col2 conformant">Pass</th>
 <th class="column col2 partially-conformant">Partial</th>
 </tr>
+<tr><th class="column col1">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;overloads_consistency</th>
+<th class="column col2 not-conformant">Unknown</th>
+<th class="column col2 not-conformant">Unknown</th>
+<th class="column col2 not-conformant">Unknown</th>
+<th class="column col2 not-conformant">Unknown</th>
+</tr>
 <tr><th class="column col1">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;overloads_definitions</th>
 <th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not allow an overload with no implementation in an abstract base class.</p><p>Allows @override to be on all overloads and implementation, instead of just implementation.</p></span></div></th>
 <th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Allows @final/@override on all overloads and implementation; should be implementation-only.</p></span></div></th>

conformance/tests/overloads_consistency.py

@@ -0,0 +1,43 @@
+"""
+Tests consistency of overloads with implementation.
+"""
+
+from typing import overload
+
+# > If an overload implementation is defined, type checkers should validate
+# > that it is consistent with all of its associated overload signatures.
+# > The implementation should accept all potential sets of arguments
+# > that are accepted by the overloads and should produce all potential return
+# > types produced by the overloads. In typing terms, this means the input
+# > signature of the implementation should be :term:<assignable> to the input
+# > signatures of all overloads, and the return type of all overloads should be
+# > assignable to the return type of the implementation.
+
+# Return type of all overloads must be assignable to return type of
+# implementation:
+
+@overload
+def return_type(x: int) -> int:
+    ...
+
+@overload
+def return_type(x: str) -> str:  # E[return_type]
+    ...
+
+def return_type(x: int | str) -> int:  # E[return_type] an overload returns `str`, not assignable to `int`
+    return 1
+
+
+# Input signature of implementation must be assignable to signature of each
+# overload:
+
+@overload
+def parameter_type(x: int) -> int:
+    ...
+
+@overload
+def parameter_type(x: str) -> str:  # E[parameter_type]
+    ...
+
+def parameter_type(x: int) -> int | str:  # E[parameter_type] impl type of `x` must be assignable from overload types of `x`
+    return 1