Added a few more overload evaluation tests. (#1957)

* Added a few more overload evaluation tests.
These uncovered a few more cases where mypy and pyright do not conform to the standard.

* Incorporated feedback from Carl's code review.

Author: erictraut

conformance/results/mypy/overloads_evaluation.toml

@@ -3,41 +3,51 @@ notes = """
 Does not expand boolean arguments to Literal[True] and Literal[False].
 Does not expand enum arguments to literal variants.
 Does not expand tuple arguments to possible combinations.
+Does not evaluate Any in some cases where overload is ambiguous.
+Evaluates Any in some cases where overload is not ambiguous.
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 106: Unexpected errors ['overloads_evaluation.py:106: error: No overload variant of "expand_bool" matches argument type "bool"  [call-overload]']
-Line 107: Unexpected errors ['overloads_evaluation.py:107: error: Expression is of type "Any", not "Literal[0, 1]"  [assert-type]']
-Line 129: Unexpected errors ['overloads_evaluation.py:129: error: No overload variant of "expand_enum" matches argument type "Color"  [call-overload]']
-Line 130: Unexpected errors ['overloads_evaluation.py:130: error: Expression is of type "Any", not "Literal[0, 1]"  [assert-type]']
-Line 169: Unexpected errors ['overloads_evaluation.py:169: error: Argument 1 to "expand_tuple" has incompatible type "tuple[int, int | str]"; expected "tuple[int, int]"  [arg-type]']
-Line 170: Unexpected errors ['overloads_evaluation.py:170: error: Expression is of type "int", not "int | str"  [assert-type]']
+Line 134: Unexpected errors ['overloads_evaluation.py:134: error: No overload variant of "expand_bool" matches argument type "bool"  [call-overload]']
+Line 135: Unexpected errors ['overloads_evaluation.py:135: error: Expression is of type "Any", not "Literal[0, 1]"  [assert-type]']
+Line 160: Unexpected errors ['overloads_evaluation.py:160: error: No overload variant of "expand_enum" matches argument type "Color"  [call-overload]']
+Line 161: Unexpected errors ['overloads_evaluation.py:161: error: Expression is of type "Any", not "Literal[0, 1]"  [assert-type]']
+Line 204: Unexpected errors ['overloads_evaluation.py:204: error: Argument 1 to "expand_tuple" has incompatible type "tuple[int, int | str]"; expected "tuple[int, int]"  [arg-type]']
+Line 205: Unexpected errors ['overloads_evaluation.py:205: error: Expression is of type "int", not "int | str"  [assert-type]']
+Line 264: Unexpected errors ['overloads_evaluation.py:264: error: Expression is of type "list[Any]", not "Any"  [assert-type]']
+Line 280: Unexpected errors ['overloads_evaluation.py:280: error: Expression is of type "list[Any]", not "Any"  [assert-type]']
+Line 301: Unexpected errors ['overloads_evaluation.py:301: error: Expression is of type "Any", not "float"  [assert-type]']
+Line 345: Unexpected errors ['overloads_evaluation.py:345: error: Expression is of type "list[Any]", not "Any"  [assert-type]']
 """
 output = """
-overloads_evaluation.py:32: error: All overload variants of "example1" require at least one argument  [call-overload]
-overloads_evaluation.py:32: note: Possible overload variants:
-overloads_evaluation.py:32: note:     def example1(x: int, y: str) -> int
-overloads_evaluation.py:32: note:     def example1(x: str) -> str
-overloads_evaluation.py:40: error: No overload variant of "example1" matches argument types "int", "int"  [call-overload]
-overloads_evaluation.py:40: note: Possible overload variants:
-overloads_evaluation.py:40: note:     def example1(x: int, y: str) -> int
-overloads_evaluation.py:40: note:     def example1(x: str) -> str
-overloads_evaluation.py:45: error: No overload variant of "example1" matches argument type "int"  [call-overload]
-overloads_evaluation.py:45: note: Possible overload variants:
-overloads_evaluation.py:45: note:     def example1(x: int, y: str) -> int
-overloads_evaluation.py:45: note:     def example1(x: str) -> str
-overloads_evaluation.py:89: error: Argument 1 to "example2" has incompatible type "int | str"; expected "int"  [arg-type]
-overloads_evaluation.py:89: error: Argument 2 to "example2" has incompatible type "int | str"; expected "str"  [arg-type]
-overloads_evaluation.py:106: error: No overload variant of "expand_bool" matches argument type "bool"  [call-overload]
-overloads_evaluation.py:106: note: Possible overload variants:
-overloads_evaluation.py:106: note:     def expand_bool(x: Literal[False]) -> Literal[0]
-overloads_evaluation.py:106: note:     def expand_bool(x: Literal[True]) -> Literal[1]
-overloads_evaluation.py:107: error: Expression is of type "Any", not "Literal[0, 1]"  [assert-type]
-overloads_evaluation.py:129: error: No overload variant of "expand_enum" matches argument type "Color"  [call-overload]
-overloads_evaluation.py:129: note: Possible overload variants:
-overloads_evaluation.py:129: note:     def expand_enum(x: Literal[Color.RED]) -> Literal[0]
-overloads_evaluation.py:129: note:     def expand_enum(x: Literal[Color.BLUE]) -> Literal[1]
-overloads_evaluation.py:130: error: Expression is of type "Any", not "Literal[0, 1]"  [assert-type]
-overloads_evaluation.py:169: error: Argument 1 to "expand_tuple" has incompatible type "tuple[int, int | str]"; expected "tuple[int, int]"  [arg-type]
-overloads_evaluation.py:170: error: Expression is of type "int", not "int | str"  [assert-type]
+overloads_evaluation.py:38: error: All overload variants of "example1_1" require at least one argument  [call-overload]
+overloads_evaluation.py:38: note: Possible overload variants:
+overloads_evaluation.py:38: note:     def example1_1(x: int, y: str) -> int
+overloads_evaluation.py:38: note:     def example1_1(x: str) -> str
+overloads_evaluation.py:46: error: No overload variant of "example1_1" matches argument types "int", "int"  [call-overload]
+overloads_evaluation.py:46: note: Possible overload variants:
+overloads_evaluation.py:46: note:     def example1_1(x: int, y: str) -> int
+overloads_evaluation.py:46: note:     def example1_1(x: str) -> str
+overloads_evaluation.py:51: error: No overload variant of "example1_1" matches argument type "int"  [call-overload]
+overloads_evaluation.py:51: note: Possible overload variants:
+overloads_evaluation.py:51: note:     def example1_1(x: int, y: str) -> int
+overloads_evaluation.py:51: note:     def example1_1(x: str) -> str
+overloads_evaluation.py:115: error: Argument 1 to "example2" has incompatible type "int | str"; expected "int"  [arg-type]
+overloads_evaluation.py:115: error: Argument 2 to "example2" has incompatible type "int | str"; expected "str"  [arg-type]
+overloads_evaluation.py:134: error: No overload variant of "expand_bool" matches argument type "bool"  [call-overload]
+overloads_evaluation.py:134: note: Possible overload variants:
+overloads_evaluation.py:134: note:     def expand_bool(x: Literal[False]) -> Literal[0]
+overloads_evaluation.py:134: note:     def expand_bool(x: Literal[True]) -> Literal[1]
+overloads_evaluation.py:135: error: Expression is of type "Any", not "Literal[0, 1]"  [assert-type]
+overloads_evaluation.py:160: error: No overload variant of "expand_enum" matches argument type "Color"  [call-overload]
+overloads_evaluation.py:160: note: Possible overload variants:
+overloads_evaluation.py:160: note:     def expand_enum(x: Literal[Color.RED]) -> Literal[0]
+overloads_evaluation.py:160: note:     def expand_enum(x: Literal[Color.BLUE]) -> Literal[1]
+overloads_evaluation.py:161: error: Expression is of type "Any", not "Literal[0, 1]"  [assert-type]
+overloads_evaluation.py:204: error: Argument 1 to "expand_tuple" has incompatible type "tuple[int, int | str]"; expected "tuple[int, int]"  [arg-type]
+overloads_evaluation.py:205: error: Expression is of type "int", not "int | str"  [assert-type]
+overloads_evaluation.py:264: error: Expression is of type "list[Any]", not "Any"  [assert-type]
+overloads_evaluation.py:280: error: Expression is of type "list[Any]", not "Any"  [assert-type]
+overloads_evaluation.py:301: error: Expression is of type "Any", not "float"  [assert-type]
+overloads_evaluation.py:345: error: Expression is of type "list[Any]", not "Any"  [assert-type]
 """

conformance/results/pyre/overloads_evaluation.toml

@@ -9,35 +9,39 @@ Does not treat multiple matches due to gradual types as ambiguous.
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 81: Unexpected errors ['overloads_evaluation.py:81:23 Incompatible parameter type [6]: In call `example2`, for 2nd positional argument, expected `str` but got `Union[int, str]`.']
-Line 82: Unexpected errors ['overloads_evaluation.py:82:4 Assert type [70]: Expected `Union[int, str]` but got `str`.']
-Line 106: Unexpected errors ['overloads_evaluation.py:106:23 Incompatible parameter type [6]: In call `expand_bool`, for 1st positional argument, expected `typing_extensions.Literal[False]` but got `bool`.']
-Line 107: Unexpected errors ['overloads_evaluation.py:107:4 Assert type [70]: Expected `Union[typing_extensions.Literal[0], typing_extensions.Literal[1]]` but got `typing_extensions.Literal[0]`.']
-Line 129: Unexpected errors ['overloads_evaluation.py:129:23 Incompatible parameter type [6]: In call `expand_enum`, for 1st positional argument, expected `typing_extensions.Literal[Color.RED]` but got `Color`.']
-Line 130: Unexpected errors ['overloads_evaluation.py:130:4 Assert type [70]: Expected `Union[typing_extensions.Literal[0], typing_extensions.Literal[1]]` but got `typing_extensions.Literal[0]`.']
-Line 147: Unexpected errors ['overloads_evaluation.py:147:29 Incompatible parameter type [6]: In call `expand_type_union`, for 1st positional argument, expected `Type[int]` but got `Type[Union[int, str]]`.']
-Line 148: Unexpected errors ['overloads_evaluation.py:148:4 Assert type [70]: Expected `Union[int, str]` but got `int`.']
-Line 169: Unexpected errors ['overloads_evaluation.py:169:24 Incompatible parameter type [6]: In call `expand_tuple`, for 1st positional argument, expected `Tuple[int, int]` but got `Tuple[int, Union[int, str]]`.']
-Line 170: Unexpected errors ['overloads_evaluation.py:170:4 Assert type [70]: Expected `Union[int, str]` but got `int`.']
-Line 196: Unexpected errors ['overloads_evaluation.py:196:4 Assert type [70]: Expected `int` but got `str`.']
-Line 224: Unexpected errors ['overloads_evaluation.py:224:4 Assert type [70]: Expected `typing.Any` but got `int`.']
+Line 105: Unexpected errors ['overloads_evaluation.py:105:23 Incompatible parameter type [6]: In call `example2`, for 2nd positional argument, expected `str` but got `Union[int, str]`.']
+Line 106: Unexpected errors ['overloads_evaluation.py:106:4 Assert type [70]: Expected `Union[int, str]` but got `str`.']
+Line 134: Unexpected errors ['overloads_evaluation.py:134:23 Incompatible parameter type [6]: In call `expand_bool`, for 1st positional argument, expected `typing_extensions.Literal[False]` but got `bool`.']
+Line 135: Unexpected errors ['overloads_evaluation.py:135:4 Assert type [70]: Expected `Union[typing_extensions.Literal[0], typing_extensions.Literal[1]]` but got `typing_extensions.Literal[0]`.']
+Line 160: Unexpected errors ['overloads_evaluation.py:160:23 Incompatible parameter type [6]: In call `expand_enum`, for 1st positional argument, expected `typing_extensions.Literal[Color.RED]` but got `Color`.']
+Line 161: Unexpected errors ['overloads_evaluation.py:161:4 Assert type [70]: Expected `Union[typing_extensions.Literal[0], typing_extensions.Literal[1]]` but got `typing_extensions.Literal[0]`.']
+Line 180: Unexpected errors ['overloads_evaluation.py:180:29 Incompatible parameter type [6]: In call `expand_type_union`, for 1st positional argument, expected `Type[int]` but got `Type[Union[int, str]]`.']
+Line 181: Unexpected errors ['overloads_evaluation.py:181:4 Assert type [70]: Expected `Union[int, str]` but got `int`.']
+Line 204: Unexpected errors ['overloads_evaluation.py:204:24 Incompatible parameter type [6]: In call `expand_tuple`, for 1st positional argument, expected `Tuple[int, int]` but got `Tuple[int, Union[int, str]]`.']
+Line 205: Unexpected errors ['overloads_evaluation.py:205:4 Assert type [70]: Expected `Union[int, str]` but got `int`.']
+Line 234: Unexpected errors ['overloads_evaluation.py:234:4 Assert type [70]: Expected `int` but got `str`.']
+Line 264: Unexpected errors ['overloads_evaluation.py:264:4 Assert type [70]: Expected `typing.Any` but got `List[int]`.']
+Line 280: Unexpected errors ['overloads_evaluation.py:280:4 Assert type [70]: Expected `typing.Any` but got `List[int]`.']
+Line 345: Unexpected errors ['overloads_evaluation.py:345:4 Assert type [70]: Expected `typing.Any` but got `List[int]`.']
 """
 output = """
-overloads_evaluation.py:32:0 Missing argument [20]: Call `example1` expects argument `x`.
-overloads_evaluation.py:40:12 Incompatible parameter type [6]: In call `example1`, for 2nd positional argument, expected `str` but got `int`.
-overloads_evaluation.py:45:9 Incompatible parameter type [6]: In call `example1`, for 1st positional argument, expected `str` but got `int`.
-overloads_evaluation.py:81:23 Incompatible parameter type [6]: In call `example2`, for 2nd positional argument, expected `str` but got `Union[int, str]`.
-overloads_evaluation.py:82:4 Assert type [70]: Expected `Union[int, str]` but got `str`.
-overloads_evaluation.py:89:13 Incompatible parameter type [6]: In call `example2`, for 1st positional argument, expected `int` but got `Union[int, str]`.
-overloads_evaluation.py:89:16 Incompatible parameter type [6]: In call `example2`, for 2nd positional argument, expected `str` but got `Union[int, str]`.
-overloads_evaluation.py:106:23 Incompatible parameter type [6]: In call `expand_bool`, for 1st positional argument, expected `typing_extensions.Literal[False]` but got `bool`.
-overloads_evaluation.py:107:4 Assert type [70]: Expected `Union[typing_extensions.Literal[0], typing_extensions.Literal[1]]` but got `typing_extensions.Literal[0]`.
-overloads_evaluation.py:129:23 Incompatible parameter type [6]: In call `expand_enum`, for 1st positional argument, expected `typing_extensions.Literal[Color.RED]` but got `Color`.
-overloads_evaluation.py:130:4 Assert type [70]: Expected `Union[typing_extensions.Literal[0], typing_extensions.Literal[1]]` but got `typing_extensions.Literal[0]`.
-overloads_evaluation.py:147:29 Incompatible parameter type [6]: In call `expand_type_union`, for 1st positional argument, expected `Type[int]` but got `Type[Union[int, str]]`.
-overloads_evaluation.py:148:4 Assert type [70]: Expected `Union[int, str]` but got `int`.
-overloads_evaluation.py:169:24 Incompatible parameter type [6]: In call `expand_tuple`, for 1st positional argument, expected `Tuple[int, int]` but got `Tuple[int, Union[int, str]]`.
-overloads_evaluation.py:170:4 Assert type [70]: Expected `Union[int, str]` but got `int`.
-overloads_evaluation.py:196:4 Assert type [70]: Expected `int` but got `str`.
-overloads_evaluation.py:224:4 Assert type [70]: Expected `typing.Any` but got `int`.
+overloads_evaluation.py:38:0 Missing argument [20]: Call `example1_1` expects argument `x`.
+overloads_evaluation.py:46:14 Incompatible parameter type [6]: In call `example1_1`, for 2nd positional argument, expected `str` but got `int`.
+overloads_evaluation.py:51:11 Incompatible parameter type [6]: In call `example1_1`, for 1st positional argument, expected `str` but got `int`.
+overloads_evaluation.py:105:23 Incompatible parameter type [6]: In call `example2`, for 2nd positional argument, expected `str` but got `Union[int, str]`.
+overloads_evaluation.py:106:4 Assert type [70]: Expected `Union[int, str]` but got `str`.
+overloads_evaluation.py:115:13 Incompatible parameter type [6]: In call `example2`, for 1st positional argument, expected `int` but got `Union[int, str]`.
+overloads_evaluation.py:115:16 Incompatible parameter type [6]: In call `example2`, for 2nd positional argument, expected `str` but got `Union[int, str]`.
+overloads_evaluation.py:134:23 Incompatible parameter type [6]: In call `expand_bool`, for 1st positional argument, expected `typing_extensions.Literal[False]` but got `bool`.
+overloads_evaluation.py:135:4 Assert type [70]: Expected `Union[typing_extensions.Literal[0], typing_extensions.Literal[1]]` but got `typing_extensions.Literal[0]`.
+overloads_evaluation.py:160:23 Incompatible parameter type [6]: In call `expand_enum`, for 1st positional argument, expected `typing_extensions.Literal[Color.RED]` but got `Color`.
+overloads_evaluation.py:161:4 Assert type [70]: Expected `Union[typing_extensions.Literal[0], typing_extensions.Literal[1]]` but got `typing_extensions.Literal[0]`.
+overloads_evaluation.py:180:29 Incompatible parameter type [6]: In call `expand_type_union`, for 1st positional argument, expected `Type[int]` but got `Type[Union[int, str]]`.
+overloads_evaluation.py:181:4 Assert type [70]: Expected `Union[int, str]` but got `int`.
+overloads_evaluation.py:204:24 Incompatible parameter type [6]: In call `expand_tuple`, for 1st positional argument, expected `Tuple[int, int]` but got `Tuple[int, Union[int, str]]`.
+overloads_evaluation.py:205:4 Assert type [70]: Expected `Union[int, str]` but got `int`.
+overloads_evaluation.py:234:4 Assert type [70]: Expected `int` but got `str`.
+overloads_evaluation.py:264:4 Assert type [70]: Expected `typing.Any` but got `List[int]`.
+overloads_evaluation.py:280:4 Assert type [70]: Expected `typing.Any` but got `List[int]`.
+overloads_evaluation.py:345:4 Assert type [70]: Expected `typing.Any` but got `List[int]`.
 """

conformance/results/pyre/version.toml

@@ -1,2 +1,2 @@
 version = "pyre 0.9.23"
-test_duration = 6.3
+test_duration = 6.4

conformance/results/pyright/overloads_evaluation.toml

@@ -1,19 +1,24 @@
-conformant = "Pass"
-conformance_automated = "Pass"
+conformant = "Partial"
+notes = """
+Does not evaluate Any in some cases where overload is ambiguous.
+"""
+conformance_automated = "Fail"
 errors_diff = """
+Line 280: Unexpected errors ['overloads_evaluation.py:280:17 - error: "assert_type" mismatch: expected "Any" but received "list[int]" (reportAssertTypeFailure)']
 """
 output = """
-overloads_evaluation.py:32:1 - error: No overloads for "example1" match the provided arguments
+overloads_evaluation.py:38:1 - error: No overloads for "example1_1" match the provided arguments
   Argument types: () (reportCallIssue)
-overloads_evaluation.py:40:13 - error: Argument of type "Literal[1]" cannot be assigned to parameter "y" of type "str" in function "example1"
+overloads_evaluation.py:46:15 - error: Argument of type "Literal[1]" cannot be assigned to parameter "y" of type "str" in function "example1_1"
   "Literal[1]" is not assignable to "str" (reportArgumentType)
-overloads_evaluation.py:45:10 - error: Argument of type "Literal[1]" cannot be assigned to parameter "x" of type "str" in function "example1"
+overloads_evaluation.py:51:12 - error: Argument of type "Literal[1]" cannot be assigned to parameter "x" of type "str" in function "example1_1"
   "Literal[1]" is not assignable to "str" (reportArgumentType)
-overloads_evaluation.py:89:5 - error: No overloads for "example2" match the provided arguments (reportCallIssue)
-overloads_evaluation.py:89:14 - error: Argument of type "int | str" cannot be assigned to parameter "x" of type "int" in function "example2"
+overloads_evaluation.py:115:5 - error: No overloads for "example2" match the provided arguments (reportCallIssue)
+overloads_evaluation.py:115:14 - error: Argument of type "int | str" cannot be assigned to parameter "x" of type "int" in function "example2"
   Type "int | str" is not assignable to type "int"
     "str" is not assignable to "int" (reportArgumentType)
-overloads_evaluation.py:89:17 - error: Argument of type "int | str" cannot be assigned to parameter "y" of type "int" in function "example2"
+overloads_evaluation.py:115:17 - error: Argument of type "int | str" cannot be assigned to parameter "y" of type "int" in function "example2"
   Type "int | str" is not assignable to type "int"
     "str" is not assignable to "int" (reportArgumentType)
+overloads_evaluation.py:280:17 - error: "assert_type" mismatch: expected "Any" but received "list[int]" (reportAssertTypeFailure)
 """