Updated conformance test results for the latest versions of type chec… (#1904)

erictraut · web-flow · commit e52371772936 · 2024-12-20T10:32:35.000-08:00
Updated conformance test results for the latest versions of type checkers. Mypy 1.14: * Added support for @deprecated * Added support for default TypeVar values * Fixed conformance issues with enums * Fixed conformance issues with generics (constrained TypeVars and ParamSpecs) * Fixed conformance issues with TypedDict ReadOnly Pyright 1.1.391: * No changes
diff --git a/conformance/results/mypy/directives_deprecated.toml b/conformance/results/mypy/directives_deprecated.toml
@@ -1,21 +1,18 @@
-conformant = "Unsupported"
-notes = """
-Does not support @deprecated.
-"""
-conformance_automated = "Fail"
+conformant = "Pass"
+conformance_automated = "Pass"
 errors_diff = """
-Line 18: Expected 1 errors
-Line 24: Expected 1 errors
-Line 25: Expected 1 errors
-Line 30: Expected 1 errors
-Line 41: Expected 1 errors
-Line 42: Expected 1 errors
-Line 44: Expected 1 errors
-Line 47: Expected 1 errors
-Line 48: Expected 1 errors
-Line 58: Expected 1 errors
-Line 69: Expected 1 errors
-Line 98: Expected 1 errors
 """
 output = """
+directives_deprecated.py:18: error: class _directives_deprecated_library.Ham is deprecated: Use Spam instead  [deprecated]
+directives_deprecated.py:24: error: function _directives_deprecated_library.norwegian_blue is deprecated: It is pining for the fjords  [deprecated]
+directives_deprecated.py:25: error: function _directives_deprecated_library.norwegian_blue is deprecated: It is pining for the fjords  [deprecated]
+directives_deprecated.py:30: error: overload def (x: builtins.int) -> builtins.str of function _directives_deprecated_library.foo is deprecated: Only str will be allowed  [deprecated]
+directives_deprecated.py:41: error: function _directives_deprecated_library.Spam.__add__ is deprecated: There is enough spam in the world  [deprecated]
+directives_deprecated.py:42: error: function _directives_deprecated_library.Spam.__add__ is deprecated: There is enough spam in the world  [deprecated]
+directives_deprecated.py:44: error: function _directives_deprecated_library.Spam.greasy is deprecated: All spam will be equally greasy  [deprecated]
+directives_deprecated.py:47: error: function _directives_deprecated_library.Spam.shape is deprecated: Shapes are becoming immutable  [deprecated]
+directives_deprecated.py:48: error: function _directives_deprecated_library.Spam.shape is deprecated: Shapes are becoming immutable  [deprecated]
+directives_deprecated.py:58: error: function directives_deprecated.Invocable.__call__ is deprecated: Deprecated  [deprecated]
+directives_deprecated.py:69: error: function directives_deprecated.lorem is deprecated: Deprecated  [deprecated]
+directives_deprecated.py:98: error: function directives_deprecated.SupportsFoo1.foo is deprecated: Deprecated  [deprecated]
 """
diff --git a/conformance/results/mypy/enums_members.toml b/conformance/results/mypy/enums_members.toml
@@ -1,7 +1,6 @@
 conformant = "Partial"
 notes = """
 Does not treat attribute with annotation and no assignment as non-member.
-Does not reject use of annotation with enum member.
 Does not treat callables as non-members.
 Does not honor `enum.member` as method decorator.
 Does not properly handle aliased enum members.
@@ -10,7 +9,6 @@ Does not treat attributes with private names as non-members.
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 50: Expected 1 errors
 Line 82: Expected 1 errors
 Line 83: Expected 1 errors
 Line 129: Expected 1 errors
@@ -26,6 +24,8 @@ enums_members.py:27: error: Expression is of type "Literal[Pet.genus]", not "str
 enums_members.py:28: error: Expression is of type "Literal[Pet.species]", not "str"  [assert-type]
 enums_members.py:35: error: Expression is of type "Literal[Pet2.genus]", not "str"  [assert-type]
 enums_members.py:36: error: Expression is of type "Literal[Pet2.species]", not "str"  [assert-type]
+enums_members.py:50: error: Enum members must be left unannotated  [misc]
+enums_members.py:50: note: See https://typing.readthedocs.io/en/latest/spec/enums.html#defining-members
 enums_members.py:84: error: Expression is of type "Callable[[Pet4], str]", not "Any"  [assert-type]
 enums_members.py:84: error: Parameter 1 of Literal[...] is invalid  [valid-type]
 enums_members.py:85: error: Expression is of type "Callable[[Pet4], None]", not "Any"  [assert-type]
diff --git a/conformance/results/mypy/generics_basic.toml b/conformance/results/mypy/generics_basic.toml
@@ -1,11 +1,9 @@
-conformant = "Partial"
-notes = """
-Does not reject the use of a constraint parameterized by another type variable.
-"""
+conformant = "Pass"
 output = """
 generics_basic.py:40: error: Value of type variable "AnyStr" of "concat" cannot be "Sequence[object]"  [type-var]
 generics_basic.py:41: error: Value of type variable "AnyStr" of "concat" cannot be "Sequence[object]"  [type-var]
 generics_basic.py:49: error: Type variable must have at least two constrained types  [misc]
+generics_basic.py:55: error: TypeVar constraint type cannot be parametrized by type variables  [misc]
 generics_basic.py:69: error: Value of type variable "AnyStr" of "concat" cannot be "Sequence[object]"  [type-var]
 generics_basic.py:121: error: Duplicate type variables in Generic[...] or Protocol[...]  [misc]
 generics_basic.py:157: error: Invalid index type "int" for "MyMap1[str, int]"; expected type "str"  [index]
@@ -19,7 +17,6 @@ generics_basic.py:208: error: Type variable "generics_basic.T" is unbound  [vali
 generics_basic.py:208: note: (Hint: Use "Generic[T]" or "Protocol[T]" base class to bind "T" inside a class)
 generics_basic.py:208: note: (Hint: Use "T" in function signature to bind "T" inside a function)
 """
-conformance_automated = "Fail"
+conformance_automated = "Pass"
 errors_diff = """
-Line 55: Expected 1 errors
 """
diff --git a/conformance/results/mypy/generics_defaults.toml b/conformance/results/mypy/generics_defaults.toml
@@ -1,4 +1,4 @@
-conformant = "Unsupported"
+conformant = "Partial"
 output = """
 generics_defaults.py:24: error: "T" cannot appear after "DefaultStrT" in type parameter list because it has no default type  [misc]
 generics_defaults.py:30: error: Expression is of type "type[NoNonDefaults[Any, Any]]", not "type[NoNonDefaults[str, int]]"  [assert-type]
diff --git a/conformance/results/mypy/generics_defaults_referential.toml b/conformance/results/mypy/generics_defaults_referential.toml
@@ -1,4 +1,4 @@
-conformant = "Unsupported"
+conformant = "Partial"
 output = """
 generics_defaults_referential.py:23: error: Expression is of type "type[slice[Any, Any, Any]]", not "type[slice[int, int, int | None]]"  [assert-type]
 generics_defaults_referential.py:37: error: Argument 1 to "Foo" has incompatible type "str"; expected "int"  [arg-type]
diff --git a/conformance/results/mypy/generics_defaults_specialization.toml b/conformance/results/mypy/generics_defaults_specialization.toml
@@ -1,4 +1,4 @@
-conformant = "Unsupported"
+conformant = "Partial"
 output = """
 generics_defaults_specialization.py:30: error: Bad number of arguments for type alias, expected between 0 and 1, given 2  [type-arg]
 generics_defaults_specialization.py:45: error: Expression is of type "type[Bar[Any]]", not "type[Bar[str]]"  [assert-type]
diff --git a/conformance/results/mypy/generics_paramspec_basic.toml b/conformance/results/mypy/generics_paramspec_basic.toml
@@ -1,9 +1,8 @@
-conformant = "Partial"
-notes = """
-Does not reject ParamSpec when used "bare" in type alias definition.
-"""
+conformant = "Pass"
 output = """
 generics_paramspec_basic.py:10: error: String argument 1 "NotIt" to ParamSpec(...) does not match variable name "WrongName"  [misc]
+generics_paramspec_basic.py:15: error: Invalid location for ParamSpec "P"  [valid-type]
+generics_paramspec_basic.py:15: note: You can use ParamSpec as the first argument to Callable, e.g., "Callable[P, int]"
 generics_paramspec_basic.py:23: error: Invalid location for ParamSpec "P"  [valid-type]
 generics_paramspec_basic.py:23: note: You can use ParamSpec as the first argument to Callable, e.g., "Callable[P, int]"
 generics_paramspec_basic.py:27: error: Invalid location for Concatenate  [valid-type]
@@ -15,7 +14,6 @@ generics_paramspec_basic.py:35: note: You can use ParamSpec as the first argumen
 generics_paramspec_basic.py:39: error: Invalid location for ParamSpec "P"  [valid-type]
 generics_paramspec_basic.py:39: note: You can use ParamSpec as the first argument to Callable, e.g., "Callable[P, int]"
 """
-conformance_automated = "Fail"
+conformance_automated = "Pass"
 errors_diff = """
-Line 15: Expected 1 errors
 """
diff --git a/conformance/results/mypy/narrowing_typeguard.toml b/conformance/results/mypy/narrowing_typeguard.toml
@@ -4,6 +4,7 @@ narrowing_typeguard.py:102: error: TypeGuard functions must have a positional ar
 narrowing_typeguard.py:107: error: TypeGuard functions must have a positional argument  [valid-type]
 narrowing_typeguard.py:128: error: Argument 1 to "takes_callable_str" has incompatible type "Callable[[object], TypeGuard[int]]"; expected "Callable[[object], str]"  [arg-type]
 narrowing_typeguard.py:148: error: Argument 1 to "takes_callable_str_proto" has incompatible type "Callable[[object], TypeGuard[int]]"; expected "CallableStrProto"  [arg-type]
+narrowing_typeguard.py:148: note: "CallableStrProto.__call__" has type "Callable[[Arg(object, 'val')], str]"
 """
 conformance_automated = "Pass"
 errors_diff = """
diff --git a/conformance/results/mypy/narrowing_typeis.toml b/conformance/results/mypy/narrowing_typeis.toml
@@ -4,6 +4,7 @@ narrowing_typeis.py:105: error: "TypeIs" functions must have a positional argume
 narrowing_typeis.py:110: error: "TypeIs" functions must have a positional argument  [valid-type]
 narrowing_typeis.py:132: error: Argument 1 to "takes_callable_str" has incompatible type "Callable[[object], TypeIs[int]]"; expected "Callable[[object], str]"  [arg-type]
 narrowing_typeis.py:152: error: Argument 1 to "takes_callable_str_proto" has incompatible type "Callable[[object], TypeIs[int]]"; expected "CallableStrProto"  [arg-type]
+narrowing_typeis.py:152: note: "CallableStrProto.__call__" has type "Callable[[Arg(object, 'val')], str]"
 narrowing_typeis.py:169: error: Argument 1 to "takes_typeguard" has incompatible type "Callable[[object], TypeIs[int]]"; expected "Callable[[object], TypeGuard[int]]"  [arg-type]
 narrowing_typeis.py:170: error: Argument 1 to "takes_typeis" has incompatible type "Callable[[object], TypeGuard[int]]"; expected "Callable[[object], TypeIs[int]]"  [arg-type]
 narrowing_typeis.py:191: error: Argument 1 to "takes_int_typeis" has incompatible type "Callable[[object], TypeIs[bool]]"; expected "Callable[[object], TypeIs[int]]"  [arg-type]
diff --git a/conformance/results/mypy/overloads_basic.toml b/conformance/results/mypy/overloads_basic.toml
@@ -3,7 +3,7 @@ output = """
 overloads_basic.py:37: error: No overload variant of "__getitem__" of "Bytes" matches argument type "str"  [call-overload]
 overloads_basic.py:37: note: Possible overload variants:
 overloads_basic.py:37: note:     def __getitem__(self, int, /) -> int
-overloads_basic.py:37: note:     def __getitem__(self, slice, /) -> bytes
+overloads_basic.py:37: note:     def __getitem__(self, slice[Any, Any, Any], /) -> bytes
 overloads_basic.py:62: error: Single overload definition, multiple required  [misc]
 overloads_basic.py:74: error: An overloaded function outside a stub file must have an implementation  [no-overload-impl]
 """
diff --git a/conformance/results/mypy/qualifiers_annotated.toml b/conformance/results/mypy/qualifiers_annotated.toml
@@ -28,8 +28,6 @@ qualifiers_annotated.py:91: error: "<typing special form>" not callable  [operat
 qualifiers_annotated.py:92: error: "<typing special form>" not callable  [operator]
 qualifiers_annotated.py:98: error: Invalid type: ClassVar nested inside other type  [valid-type]
 qualifiers_annotated.py:100: error: Final can be only used as an outermost qualifier in a variable annotation  [valid-type]
-qualifiers_annotated.py:110: error: Required[] can be only used in a TypedDict definition  [valid-type]
-qualifiers_annotated.py:112: error: NotRequired[] can be only used in a TypedDict definition  [valid-type]
 qualifiers_annotated.py:119: error: Cannot redefine "T" as a type variable  [misc]
 qualifiers_annotated.py:119: error: Invalid assignment target  [misc]
 """
@@ -40,7 +38,5 @@ Line 85: Expected 1 errors
 Line 93: Expected 1 errors
 Line 98: Unexpected errors ['qualifiers_annotated.py:98: error: Invalid type: ClassVar nested inside other type  [valid-type]']
 Line 100: Unexpected errors ['qualifiers_annotated.py:100: error: Final can be only used as an outermost qualifier in a variable annotation  [valid-type]']
-Line 110: Unexpected errors ['qualifiers_annotated.py:110: error: Required[] can be only used in a TypedDict definition  [valid-type]']
-Line 112: Unexpected errors ['qualifiers_annotated.py:112: error: NotRequired[] can be only used in a TypedDict definition  [valid-type]']
 Line 119: Unexpected errors ['qualifiers_annotated.py:119: error: Cannot redefine "T" as a type variable  [misc]', 'qualifiers_annotated.py:119: error: Invalid assignment target  [misc]']
 """
diff --git a/conformance/results/mypy/typeddicts_readonly.toml b/conformance/results/mypy/typeddicts_readonly.toml
@@ -1,17 +1,12 @@
-conformant = "Partial"
-notes = """
-Incorrectly rejects NotRequired directive when used in an Annotated annotation.
-"""
+conformant = "Pass"
 output = """
 typeddicts_readonly.py:24: error: ReadOnly TypedDict key "members" TypedDict is mutated  [typeddict-readonly-mutated]
 typeddicts_readonly.py:36: error: ReadOnly TypedDict key "members" TypedDict is mutated  [typeddict-readonly-mutated]
 typeddicts_readonly.py:50: error: ReadOnly TypedDict key "title" TypedDict is mutated  [typeddict-readonly-mutated]
 typeddicts_readonly.py:51: error: ReadOnly TypedDict key "year" TypedDict is mutated  [typeddict-readonly-mutated]
-typeddicts_readonly.py:56: error: NotRequired[] can be only used in a TypedDict definition  [valid-type]
 typeddicts_readonly.py:60: error: ReadOnly TypedDict key "title" TypedDict is mutated  [typeddict-readonly-mutated]
+typeddicts_readonly.py:61: error: ReadOnly TypedDict key "year" TypedDict is mutated  [typeddict-readonly-mutated]
 """
-conformance_automated = "Fail"
+conformance_automated = "Pass"
 errors_diff = """
-Line 61: Expected 1 errors
-Line 56: Unexpected errors ['typeddicts_readonly.py:56: error: NotRequired[] can be only used in a TypedDict definition  [valid-type]']
 """
diff --git a/conformance/results/mypy/typeddicts_readonly_consistency.toml b/conformance/results/mypy/typeddicts_readonly_consistency.toml
@@ -1,18 +1,13 @@
-conformant = "Partial"
-notes = """
-Incorrectly rejects assignment of required item to not-required read-only item.
-"""
+conformant = "Pass"
 output = """
 typeddicts_readonly_consistency.py:37: error: Incompatible types in assignment (expression has type "A1", variable has type "B1")  [assignment]
 typeddicts_readonly_consistency.py:38: error: Incompatible types in assignment (expression has type "C1", variable has type "B1")  [assignment]
 typeddicts_readonly_consistency.py:40: error: Incompatible types in assignment (expression has type "A1", variable has type "C1")  [assignment]
-typeddicts_readonly_consistency.py:79: error: Incompatible types in assignment (expression has type "C2", variable has type "A2")  [assignment]
 typeddicts_readonly_consistency.py:81: error: Incompatible types in assignment (expression has type "A2", variable has type "B2")  [assignment]
 typeddicts_readonly_consistency.py:82: error: Incompatible types in assignment (expression has type "C2", variable has type "B2")  [assignment]
 typeddicts_readonly_consistency.py:84: error: Incompatible types in assignment (expression has type "A2", variable has type "C2")  [assignment]
 typeddicts_readonly_consistency.py:85: error: Incompatible types in assignment (expression has type "B2", variable has type "C2")  [assignment]
 """
-conformance_automated = "Fail"
+conformance_automated = "Pass"
 errors_diff = """
-Line 79: Unexpected errors ['typeddicts_readonly_consistency.py:79: error: Incompatible types in assignment (expression has type "C2", variable has type "A2")  [assignment]']
 """
diff --git a/conformance/results/mypy/typeddicts_required.toml b/conformance/results/mypy/typeddicts_required.toml
@@ -1,17 +1,10 @@
-conformant = "Partial"
-notes = """
-Does not support nesting of `Annotated` and `Required` or `NotRequired`.
-"""
+conformant = "Pass"
 output = """
 typeddicts_required.py:12: error: Required[] can be only used in a TypedDict definition  [valid-type]
 typeddicts_required.py:19: error: NotRequired[] can be only used in a TypedDict definition  [valid-type]
 typeddicts_required.py:62: error: "Required[]" type cannot be nested  [valid-type]
 typeddicts_required.py:63: error: "NotRequired[]" type cannot be nested  [valid-type]
-typeddicts_required.py:68: error: Required[] can be only used in a TypedDict definition  [valid-type]
-typeddicts_required.py:70: error: Required[] can be only used in a TypedDict definition  [valid-type]
 """
-conformance_automated = "Fail"
+conformance_automated = "Pass"
 errors_diff = """
-Line 68: Unexpected errors ['typeddicts_required.py:68: error: Required[] can be only used in a TypedDict definition  [valid-type]']
-Line 70: Unexpected errors ['typeddicts_required.py:70: error: Required[] can be only used in a TypedDict definition  [valid-type]']
 """
diff --git a/conformance/results/mypy/version.toml b/conformance/results/mypy/version.toml
@@ -1,2 +1,2 @@
-version = "mypy 1.13.0"
-test_duration = 2.5
+version = "mypy 1.14.0"
+test_duration = 1.6
diff --git a/conformance/results/pyre/version.toml b/conformance/results/pyre/version.toml
@@ -1,2 +1,2 @@
 version = "pyre 0.9.23"
-test_duration = 4.3
+test_duration = 7.3
diff --git a/conformance/results/pyright/version.toml b/conformance/results/pyright/version.toml
@@ -1,2 +1,2 @@
-version = "pyright 1.1.389"
-test_duration = 2.5
+version = "pyright 1.1.391"
+test_duration = 1.2
diff --git a/conformance/results/pytype/specialtypes_never.toml b/conformance/results/pytype/specialtypes_never.toml
@@ -8,10 +8,11 @@ specialtypes_never.py:11:8: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <mod
 T_co = TypeVar("T_co", covariant=True)
        \u001b[1m\u001b[31m~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\u001b[39m\u001b[0m
 
-specialtypes_never.py:21:5: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in func1: bad return type [bad-return-type]
+specialtypes_never.py:21:8: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in func1: bad return type [bad-return-type]
 
+        sys.exit(1)       \u001b[1m\u001b[31m~~~~~~~~~~~\u001b[39m\u001b[0m
         sys.exit(1)
-    \u001b[1m\u001b[31m~~~~~~~~~~~~~~~\u001b[39m\u001b[0m
+\u001b[1m\u001b[31m~~~~~~~~~~~~~\u001b[39m\u001b[0m
 
 specialtypes_never.py:68:5: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in func6: Type annotation for v1 does not match type of assignment [annotation-type-mismatch]
 
@@ -39,7 +40,7 @@ errors_diff = """
 Line 19: Expected 1 errors
 Line 105: Expected 1 errors
 Line 11: Unexpected errors ['specialtypes_never.py:11:8: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in <module>: argument "covariant" to TypeVar not supported yet [not-supported-yet]']
-Line 21: Unexpected errors ['specialtypes_never.py:21:5: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in func1: bad return type [bad-return-type]']
+Line 21: Unexpected errors ['specialtypes_never.py:21:8: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in func1: bad return type [bad-return-type]']
 Line 68: Unexpected errors ['specialtypes_never.py:68:5: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in func6: Type annotation for v1 does not match type of assignment [annotation-type-mismatch]']
 Line 69: Unexpected errors ['specialtypes_never.py:69:5: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in func6: Type annotation for v2 does not match type of assignment [annotation-type-mismatch]']
 Line 70: Unexpected errors ['specialtypes_never.py:70:5: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in func6: Type annotation for v3 does not match type of assignment [annotation-type-mismatch]']
diff --git a/conformance/results/pytype/version.toml b/conformance/results/pytype/version.toml
@@ -1,2 +1,2 @@
 version = "pytype 2024.10.11"
-test_duration = 45.2
+test_duration = 37.2
diff --git a/conformance/results/results.html b/conformance/results/results.html
diff --git a/conformance/src/type_checker.py b/conformance/src/type_checker.py

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-conformant = "Unsupported"`
	`1`	`+conformant = "Partial"`
`2`	`2`	`output = """`
`3`	`3`	`generics_defaults.py:24: error: "T" cannot appear after "DefaultStrT" in type parameter list because it has no default type [misc]`
`4`	`4`	`generics_defaults.py:30: error: Expression is of type "type[NoNonDefaults[Any, Any]]", not "type[NoNonDefaults[str, int]]" [assert-type]`