From 1e8ea59ac7bc8dd1f4cee48d96f16622dd670a16 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 14 Oct 2025 16:21:34 +0100 Subject: [PATCH 01/18] [WIP] Start working on cache to json exporter --- mypy/exportjson.py | 216 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 mypy/exportjson.py diff --git a/mypy/exportjson.py b/mypy/exportjson.py new file mode 100644 index 000000000000..6d06de5e0333 --- /dev/null +++ b/mypy/exportjson.py @@ -0,0 +1,216 @@ +"""Tool to convert mypy cache file to a JSON format (print to stdout). + +Usage: + python -m mypy.exportjson .mypy_cache/.../my_module.data.ff + +The idea is to make caches introspectable once we've switched to a binary +cache format and removed support for the older JSON cache format. + +This is primarily to support existing use cases that need to inspect +cache files, and to support debugging mypy caching issues. This means that +this doesn't necessarily need to be kept 1:1 up to date with changes in the +binary cache format (to simplify maintenance -- we don't want this to slow +down mypy development). +""" + +import argparse +import pprint +from typing import Any, TypeAlias as _TypeAlias + +from mypy.types import Type +from mypy.nodes import ( + MypyFile, SymbolTable, SymbolTableNode, node_kinds, SymbolNode, FuncDef, TypeInfo, + TypeAlias, TypeVarExpr, Var, OverloadedFuncDef, get_flags, FUNCDEF_FLAGS, + DataclassTransformSpec, FUNCBASE_FLAGS, OverloadPart, Decorator, VAR_FLAGS +) +from librt.internal import Buffer + +JsonDict: _TypeAlias = dict[str, Any] + + +def convert_binary_cache_to_json(data: bytes) -> JsonDict: + tree = MypyFile.read(Buffer(data)) + return convert_mypy_file_to_json(tree) + + +def convert_mypy_file_to_json(self: MypyFile) -> JsonDict: + return { + ".class": "MypyFile", + "_fullname": self._fullname, + "names": convert_symbol_table(self.names, self._fullname), + "is_stub": self.is_stub, + "path": self.path, + "is_partial_stub_package": self.is_partial_stub_package, + "future_import_flags": sorted(self.future_import_flags), + } + + +def convert_symbol_table(self: SymbolTable, fullname: str) -> JsonDict: + data: JsonDict = {".class": "SymbolTable"} + for key, value in self.items(): + # Skip __builtins__: it's a reference to the builtins + # module that gets added to every module by + # SemanticAnalyzerPass2.visit_file(), but it shouldn't be + # accessed by users of the module. + if key == "__builtins__" or value.no_serialize: + continue + data[key] = convert_symbol_table_node(value, fullname, key) + return data + + +def convert_symbol_table_node(self: SymbolTableNode, prefix: str | None, name: str) -> JsonDict: + data: JsonDict = {".class": "SymbolTableNode", "kind": node_kinds[self.kind]} + if self.module_hidden: + data["module_hidden"] = True + if not self.module_public: + data["module_public"] = False + if self.implicit: + data["implicit"] = True + if self.plugin_generated: + data["plugin_generated"] = True + if self.cross_ref: + data["cross_ref"] = self.cross_ref + elif self.node is not None: + data["node"] = convert_symbol_node(self.node) + return data + + +def convert_symbol_node(self: SymbolNode) -> JsonDict: + if isinstance(self, FuncDef): + return convert_func_def(self) + elif isinstance(self, OverloadedFuncDef): + return convert_overloaded_func_def(self) + elif isinstance(self, Decorator): + return convert_decorator(self) + elif isinstance(self, Var): + return convert_var(self) + elif isinstance(self, TypeInfo): + return convert_type_info(self) + elif isinstance(self, TypeAlias): + return convert_type_alias(self) + elif isinstance(self, TypeVarExpr): + return convert_type_var_expr(self) + assert False, type(self) + + +def convert_func_def(self: FuncDef) -> JsonDict: + return { + ".class": "FuncDef", + "name": self._name, + "fullname": self._fullname, + "arg_names": self.arg_names, + "arg_kinds": [int(x.value) for x in self.arg_kinds], + "type": None if self.type is None else convert_type(self.type), + "flags": get_flags(self, FUNCDEF_FLAGS), + "abstract_status": self.abstract_status, + # TODO: Do we need expanded, original_def? + "dataclass_transform_spec": ( + None + if self.dataclass_transform_spec is None + else convert_dataclass_transform_spec(self.dataclass_transform_spec) + ), + "deprecated": self.deprecated, + "original_first_arg": self.original_first_arg, + } + + +def convert_dataclass_transform_spec(self: DataclassTransformSpec) -> JsonDict: + return { + "eq_default": self.eq_default, + "order_default": self.order_default, + "kw_only_default": self.kw_only_default, + "frozen_default": self.frozen_default, + "field_specifiers": list(self.field_specifiers), + } + + +def convert_overloaded_func_def(self: OverloadedFuncDef) -> JsonDict: + return { + ".class": "OverloadedFuncDef", + "items": [convert_overload_part(i) for i in self.items], + "type": None if self.type is None else convert_type(self.type), + "fullname": self._fullname, + "impl": None if self.impl is None else convert_overload_part(self.impl), + "flags": get_flags(self, FUNCBASE_FLAGS), + "deprecated": self.deprecated, + "setter_index": self.setter_index, + } + + +def convert_overload_part(self: OverloadPart) -> JsonDict: + if isinstance(self, FuncDef): + return convert_func_def(self) + else: + return convert_decorator(self) + + +def convert_decorator(self: Decorator) -> JsonDict: + return { + ".class": "Decorator", + "func": convert_func_def(self.func), + "var": convert_var(self.var), + "is_overload": self.is_overload, + } + + +def convert_type_info(self: TypeInfo) -> JsonDict: + return {} + + +def convert_type_alias(self: TypeAlias) -> JsonDict: + data: JsonDict = { + ".class": "TypeAlias", + "fullname": self._fullname, + "module": self.module, + "target": convert_type(self.target), + "alias_tvars": [convert_type(v) for v in self.alias_tvars], + "no_args": self.no_args, + "normalized": self.normalized, + "python_3_12_type_alias": self.python_3_12_type_alias, + } + return data + + +def convert_type_var_expr(self: TypeVarExpr) -> JsonDict: + return { + ".class": "TypeVarExpr", + "name": self._name, + "fullname": self._fullname, + "values": [convert_type(t) for t in self.values], + "upper_bound": convert_type(self.upper_bound), + "default": convert_type(self.default), + "variance": self.variance, + } + + +def convert_var(self: Var) -> JsonDict: + data: JsonDict = { + ".class": "Var", + "name": self._name, + "fullname": self._fullname, + "type": None if self.type is None else convert_type(self.type), + "setter_type": None if self.setter_type is None else convert_type(self.setter_type), + "flags": get_flags(self, VAR_FLAGS), + } + if self.final_value is not None: + data["final_value"] = self.final_value + return data + + +def convert_type(self: Type) -> JsonDict: + return {"XXX": "YYY"} + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("path") + args = parser.parse_args() + fnam: str = args.path + with open(fnam, "rb") as f: + data = f.read() + json_data = convert_binary_cache_to_json(data) + pprint.pprint(json_data) + + +if __name__ == "__main__": + main() From 3f3dc09ea75b6d38ba031e98d7c487cfb4677ebc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 14 Oct 2025 16:53:49 +0100 Subject: [PATCH 02/18] Various improvements --- mypy/exportjson.py | 174 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 155 insertions(+), 19 deletions(-) diff --git a/mypy/exportjson.py b/mypy/exportjson.py index 6d06de5e0333..59297d008976 100644 --- a/mypy/exportjson.py +++ b/mypy/exportjson.py @@ -14,14 +14,18 @@ """ import argparse -import pprint +import json from typing import Any, TypeAlias as _TypeAlias -from mypy.types import Type +from mypy.types import ( + Type, get_proper_type, Instance, AnyType, UnionType, TupleType, CallableType, + Overloaded, TypeVarType, TypeAliasType, LiteralType +) from mypy.nodes import ( MypyFile, SymbolTable, SymbolTableNode, node_kinds, SymbolNode, FuncDef, TypeInfo, TypeAlias, TypeVarExpr, Var, OverloadedFuncDef, get_flags, FUNCDEF_FLAGS, - DataclassTransformSpec, FUNCBASE_FLAGS, OverloadPart, Decorator, VAR_FLAGS + DataclassTransformSpec, FUNCBASE_FLAGS, OverloadPart, Decorator, VAR_FLAGS, + ParamSpecExpr, TypeVarTupleExpr ) from librt.internal import Buffer @@ -90,6 +94,10 @@ def convert_symbol_node(self: SymbolNode) -> JsonDict: return convert_type_alias(self) elif isinstance(self, TypeVarExpr): return convert_type_var_expr(self) + elif isinstance(self, ParamSpecExpr): + return convert_param_spec_expr(self) + elif isinstance(self, TypeVarTupleExpr): + return convert_type_var_tuple_expr(self) assert False, type(self) @@ -153,6 +161,20 @@ def convert_decorator(self: Decorator) -> JsonDict: } +def convert_var(self: Var) -> JsonDict: + data: JsonDict = { + ".class": "Var", + "name": self._name, + "fullname": self._fullname, + "type": None if self.type is None else convert_type(self.type), + "setter_type": None if self.setter_type is None else convert_type(self.setter_type), + "flags": get_flags(self, VAR_FLAGS), + } + if self.final_value is not None: + data["final_value"] = self.final_value + return data + + def convert_type_info(self: TypeInfo) -> JsonDict: return {} @@ -183,33 +205,147 @@ def convert_type_var_expr(self: TypeVarExpr) -> JsonDict: } -def convert_var(self: Var) -> JsonDict: - data: JsonDict = { - ".class": "Var", +def convert_param_spec_expr(self: ParamSpecExpr) -> JsonDict: + return { + ".class": "ParamSpecExpr", "name": self._name, "fullname": self._fullname, - "type": None if self.type is None else convert_type(self.type), - "setter_type": None if self.setter_type is None else convert_type(self.setter_type), - "flags": get_flags(self, VAR_FLAGS), + "upper_bound": convert_type(self.upper_bound), + "default": convert_type(self.default), + "variance": self.variance, + } + + +def convert_type_var_tuple_expr(self: TypeVarTupleExpr) -> JsonDict: + return { + ".class": "TypeVarTupleExpr", + "name": self._name, + "fullname": self._fullname, + "upper_bound": convert_type(self.upper_bound), + "tuple_fallback": convert_type(self.tuple_fallback), + "default": convert_type(self.default), + "variance": self.variance, + } + + +def convert_type(typ: Type) -> JsonDict: + if type(typ) is TypeAliasType: + return convert_type_alias_type(typ) + typ = get_proper_type(typ) + if isinstance(typ, Instance): + return convert_instance(typ) + elif isinstance(typ, AnyType): + return convert_any_type(typ) + elif isinstance(typ, UnionType): + return convert_union_type(typ) + elif isinstance(typ, TupleType): + return convert_tuple_type(typ) + elif isinstance(typ, CallableType): + return convert_callable_type(typ) + elif isinstance(typ, Overloaded): + return convert_overloaded(typ) + elif isinstance(typ, LiteralType): + return convert_literal_type(typ) + elif isinstance(typ, TypeVarType): + return convert_type_var_type(typ) + assert False, type(typ) + + +def convert_instance(self: Instance) -> JsonDict: + data: JsonDict = { + ".class": "Instance", + "type_ref": self.type_ref, + "args": [convert_type(arg) for arg in self.args], + } + return data + + +def convert_type_alias_type(self: TypeAliasType) -> JsonDict: + data: JsonDict = { + ".class": "TypeAliasType", + "type_ref": self.type_ref, + "args": [convert_type(arg) for arg in self.args], } - if self.final_value is not None: - data["final_value"] = self.final_value return data -def convert_type(self: Type) -> JsonDict: - return {"XXX": "YYY"} +def convert_any_type(self: AnyType) -> JsonDict: + return { + ".class": "AnyType", + "type_of_any": self.type_of_any, + "source_any": convert_type(self.source_any) if self.source_any is not None else None, + "missing_import_name": self.missing_import_name, + } + + +def convert_union_type(self: UnionType) -> JsonDict: + return { + ".class": "UnionType", + "items": [convert_type(t) for t in self.items], + "uses_pep604_syntax": self.uses_pep604_syntax, + } + + +def convert_tuple_type(self: TupleType) -> JsonDict: + return { + ".class": "TupleType", + "items": [convert_type(t) for t in self.items], + "partial_fallback": convert_type(self.partial_fallback), + "implicit": self.implicit, + } + + +def convert_literal_type(self: LiteralType) -> JsonDict: + return { + ".class": "LiteralType", + "value": self.value, + "fallback": convert_type(self.fallback), + } + + +def convert_type_var_type(self: TypeVarType) -> JsonDict: + return {} + + +def convert_callable_type(self: CallableType) -> JsonDict: + return { + ".class": "CallableType", + "arg_types": [convert_type(t) for t in self.arg_types], + "arg_kinds": [int(x.value) for x in self.arg_kinds], + "arg_names": self.arg_names, + "ret_type": convert_type(self.ret_type), + "fallback": convert_type(self.fallback), + "name": self.name, + # We don't serialize the definition (only used for error messages). + "variables": [convert_type(v) for v in self.variables], + "is_ellipsis_args": self.is_ellipsis_args, + "implicit": self.implicit, + "is_bound": self.is_bound, + "type_guard": convert_type(self.type_guard) if self.type_guard is not None else None, + "type_is": convert_type(self.type_is) if self.type_is is not None else None, + "from_concatenate": self.from_concatenate, + "imprecise_arg_kinds": self.imprecise_arg_kinds, + "unpack_kwargs": self.unpack_kwargs, + } + + +def convert_overloaded(self: Overloaded) -> JsonDict: + return {} def main() -> None: parser = argparse.ArgumentParser() - parser.add_argument("path") + parser.add_argument("path", nargs="+") args = parser.parse_args() - fnam: str = args.path - with open(fnam, "rb") as f: - data = f.read() - json_data = convert_binary_cache_to_json(data) - pprint.pprint(json_data) + fnams: list[str] = args.path + for fnam in fnams: + with open(fnam, "rb") as f: + data = f.read() + json_data = convert_binary_cache_to_json(data) + new_fnam = fnam + ".json" + with open(new_fnam, "w") as f: + json.dump(json_data, f) + print(f"{fnam} -> {new_fnam}") if __name__ == "__main__": From 6da171834c1f1c313264a3a92385e5271daf3e65 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 14 Oct 2025 17:19:40 +0100 Subject: [PATCH 03/18] Add support for additional types --- mypy/exportjson.py | 89 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 86 insertions(+), 3 deletions(-) diff --git a/mypy/exportjson.py b/mypy/exportjson.py index 59297d008976..b1fb651600d4 100644 --- a/mypy/exportjson.py +++ b/mypy/exportjson.py @@ -19,7 +19,8 @@ from mypy.types import ( Type, get_proper_type, Instance, AnyType, UnionType, TupleType, CallableType, - Overloaded, TypeVarType, TypeAliasType, LiteralType + Overloaded, TypeVarType, TypeAliasType, LiteralType, NoneType, TypeType, + UninhabitedType, UnpackType, TypeVarTupleType, ParamSpecType, Parameters ) from mypy.nodes import ( MypyFile, SymbolTable, SymbolTableNode, node_kinds, SymbolNode, FuncDef, TypeInfo, @@ -236,6 +237,8 @@ def convert_type(typ: Type) -> JsonDict: return convert_instance(typ) elif isinstance(typ, AnyType): return convert_any_type(typ) + elif isinstance(typ, NoneType): + return convert_none_type(typ) elif isinstance(typ, UnionType): return convert_union_type(typ) elif isinstance(typ, TupleType): @@ -248,6 +251,18 @@ def convert_type(typ: Type) -> JsonDict: return convert_literal_type(typ) elif isinstance(typ, TypeVarType): return convert_type_var_type(typ) + elif isinstance(typ, TypeType): + return convert_type_type(typ) + elif isinstance(typ, UninhabitedType): + return convert_uninhabited_type(typ) + elif isinstance(typ, UnpackType): + return convert_unpack_type(typ) + elif isinstance(typ, ParamSpecType): + return convert_param_spec_type(typ) + elif isinstance(typ, TypeVarTupleType): + return convert_type_var_tuple_type(typ) + elif isinstance(typ, Parameters): + return convert_parameters(typ) assert False, type(typ) @@ -278,6 +293,10 @@ def convert_any_type(self: AnyType) -> JsonDict: } +def convert_none_type(self: NoneType) -> JsonDict: + return {".class": "NoneType"} + + def convert_union_type(self: UnionType) -> JsonDict: return { ".class": "UnionType", @@ -304,7 +323,18 @@ def convert_literal_type(self: LiteralType) -> JsonDict: def convert_type_var_type(self: TypeVarType) -> JsonDict: - return {} + assert not self.id.is_meta_var() + return { + ".class": "TypeVarType", + "name": self.name, + "fullname": self.fullname, + "id": self.id.raw_id, + "namespace": self.id.namespace, + "values": [convert_type(v) for v in self.values], + "upper_bound": convert_type(self.upper_bound), + "default": convert_type(self.default), + "variance": self.variance, + } def convert_callable_type(self: CallableType) -> JsonDict: @@ -330,7 +360,60 @@ def convert_callable_type(self: CallableType) -> JsonDict: def convert_overloaded(self: Overloaded) -> JsonDict: - return {} + return {".class": "Overloaded", "items": [convert_type(t) for t in self.items]} + + +def convert_type_type(self: TypeType) -> JsonDict: + return {".class": "TypeType", "item": convert_type(self.item)} + + +def convert_uninhabited_type(self: UninhabitedType) -> JsonDict: + return {".class": "UninhabitedType"} + + +def convert_unpack_type(self: UnpackType) -> JsonDict: + return {".class": "UnpackType", "type": convert_type(self.type)} + + +def convert_param_spec_type(self: ParamSpecType) -> JsonDict: + assert not self.id.is_meta_var() + return { + ".class": "ParamSpecType", + "name": self.name, + "fullname": self.fullname, + "id": self.id.raw_id, + "namespace": self.id.namespace, + "flavor": self.flavor, + "upper_bound": convert_type(self.upper_bound), + "default": convert_type(self.default), + "prefix": convert_type(self.prefix), + } + + +def convert_type_var_tuple_type(self: TypeVarTupleType) -> JsonDict: + assert not self.id.is_meta_var() + return { + ".class": "TypeVarTupleType", + "name": self.name, + "fullname": self.fullname, + "id": self.id.raw_id, + "namespace": self.id.namespace, + "upper_bound": convert_type(self.upper_bound), + "tuple_fallback": convert_type(self.tuple_fallback), + "default": convert_type(self.default), + "min_len": self.min_len, + } + + +def convert_parameters(self: Parameters) -> JsonDict: + return { + ".class": "Parameters", + "arg_types": [convert_type(t) for t in self.arg_types], + "arg_kinds": [int(x.value) for x in self.arg_kinds], + "arg_names": self.arg_names, + "variables": [convert_type(tv) for tv in self.variables], + "imprecise_arg_kinds": self.imprecise_arg_kinds, + } def main() -> None: From cc4219614478aa990a2ad52b882f999e34545baa Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 14 Oct 2025 17:26:49 +0100 Subject: [PATCH 04/18] Add support for additional node types --- mypy/exportjson.py | 56 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/mypy/exportjson.py b/mypy/exportjson.py index b1fb651600d4..66b734e5af97 100644 --- a/mypy/exportjson.py +++ b/mypy/exportjson.py @@ -20,13 +20,14 @@ from mypy.types import ( Type, get_proper_type, Instance, AnyType, UnionType, TupleType, CallableType, Overloaded, TypeVarType, TypeAliasType, LiteralType, NoneType, TypeType, - UninhabitedType, UnpackType, TypeVarTupleType, ParamSpecType, Parameters + UninhabitedType, UnpackType, TypeVarTupleType, ParamSpecType, Parameters, + TypedDictType ) from mypy.nodes import ( MypyFile, SymbolTable, SymbolTableNode, node_kinds, SymbolNode, FuncDef, TypeInfo, TypeAlias, TypeVarExpr, Var, OverloadedFuncDef, get_flags, FUNCDEF_FLAGS, DataclassTransformSpec, FUNCBASE_FLAGS, OverloadPart, Decorator, VAR_FLAGS, - ParamSpecExpr, TypeVarTupleExpr + ParamSpecExpr, TypeVarTupleExpr, ClassDef ) from librt.internal import Buffer @@ -177,6 +178,45 @@ def convert_var(self: Var) -> JsonDict: def convert_type_info(self: TypeInfo) -> JsonDict: + data = { + ".class": "TypeInfo", + "module_name": self.module_name, + "fullname": self.fullname, + "names": convert_symbol_table(self.names, self.fullname), + "defn": convert_class_def(self.defn), + "abstract_attributes": self.abstract_attributes, + "type_vars": self.type_vars, + "has_param_spec_type": self.has_param_spec_type, + "bases": [convert_type(b) for b in self.bases], + "mro": [c.fullname for c in self.mro], + "_promote": [convert_type(p) for p in self._promote], + "alt_promote": None if self.alt_promote is None else convert_type(self.alt_promote), + "declared_metaclass": ( + None if self.declared_metaclass is None else convert_type(self.declared_metaclass) + ), + "metaclass_type": ( + None if self.metaclass_type is None else convert_type(self.metaclass_type) + ), + "tuple_type": None if self.tuple_type is None else convert_type(self.tuple_type), + "typeddict_type": ( + None if self.typeddict_type is None else convert_typeddict_type(self.typeddict_type) + ), + "flags": get_flags(self, TypeInfo.FLAGS), + "metadata": self.metadata, + "slots": sorted(self.slots) if self.slots is not None else None, + "deletable_attributes": self.deletable_attributes, + "self_type": convert_type(self.self_type) if self.self_type is not None else None, + "dataclass_transform_spec": ( + convert_dataclass_transform_spec(self.dataclass_transform_spec) + if self.dataclass_transform_spec is not None + else None + ), + "deprecated": self.deprecated, + } + return data + + +def convert_class_def(self: ClassDef) -> JsonDict: return {} @@ -263,6 +303,8 @@ def convert_type(typ: Type) -> JsonDict: return convert_type_var_tuple_type(typ) elif isinstance(typ, Parameters): return convert_parameters(typ) + elif isinstance(typ, TypedDictType): + return convert_typeddict_type(typ) assert False, type(typ) @@ -416,6 +458,16 @@ def convert_parameters(self: Parameters) -> JsonDict: } +def convert_typeddict_type(self: TypedDictType) -> JsonDict: + return { + ".class": "TypedDictType", + "items": [[n, convert_type(t)] for (n, t) in self.items.items()], + "required_keys": sorted(self.required_keys), + "readonly_keys": sorted(self.readonly_keys), + "fallback": convert_type(self.fallback), + } + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("path", nargs="+") From 4f1d0cdcdba739f208dde9539e4617e5690a9f67 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 14 Oct 2025 17:27:22 +0100 Subject: [PATCH 05/18] Lint --- mypy/exportjson.py | 61 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/mypy/exportjson.py b/mypy/exportjson.py index 66b734e5af97..83090f3dabf8 100644 --- a/mypy/exportjson.py +++ b/mypy/exportjson.py @@ -17,19 +17,52 @@ import json from typing import Any, TypeAlias as _TypeAlias -from mypy.types import ( - Type, get_proper_type, Instance, AnyType, UnionType, TupleType, CallableType, - Overloaded, TypeVarType, TypeAliasType, LiteralType, NoneType, TypeType, - UninhabitedType, UnpackType, TypeVarTupleType, ParamSpecType, Parameters, - TypedDictType -) +from librt.internal import Buffer + from mypy.nodes import ( - MypyFile, SymbolTable, SymbolTableNode, node_kinds, SymbolNode, FuncDef, TypeInfo, - TypeAlias, TypeVarExpr, Var, OverloadedFuncDef, get_flags, FUNCDEF_FLAGS, - DataclassTransformSpec, FUNCBASE_FLAGS, OverloadPart, Decorator, VAR_FLAGS, - ParamSpecExpr, TypeVarTupleExpr, ClassDef + FUNCBASE_FLAGS, + FUNCDEF_FLAGS, + VAR_FLAGS, + ClassDef, + DataclassTransformSpec, + Decorator, + FuncDef, + MypyFile, + OverloadedFuncDef, + OverloadPart, + ParamSpecExpr, + SymbolNode, + SymbolTable, + SymbolTableNode, + TypeAlias, + TypeInfo, + TypeVarExpr, + TypeVarTupleExpr, + Var, + get_flags, + node_kinds, +) +from mypy.types import ( + AnyType, + CallableType, + Instance, + LiteralType, + NoneType, + Overloaded, + Parameters, + ParamSpecType, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeType, + TypeVarTupleType, + TypeVarType, + UninhabitedType, + UnionType, + UnpackType, + get_proper_type, ) -from librt.internal import Buffer JsonDict: _TypeAlias = dict[str, Any] @@ -357,11 +390,7 @@ def convert_tuple_type(self: TupleType) -> JsonDict: def convert_literal_type(self: LiteralType) -> JsonDict: - return { - ".class": "LiteralType", - "value": self.value, - "fallback": convert_type(self.fallback), - } + return {".class": "LiteralType", "value": self.value, "fallback": convert_type(self.fallback)} def convert_type_var_type(self: TypeVarType) -> JsonDict: From 00ae328a44bc23dbbaaabdd605dceafaf1800ef5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 15 Oct 2025 11:22:24 +0100 Subject: [PATCH 06/18] Add test cases --- mypy/exportjson.py | 45 ++++++++---- mypy/test/testexportjson.py | 61 ++++++++++++++++ test-data/unit/exportjson.test | 124 +++++++++++++++++++++++++++++++++ 3 files changed, 217 insertions(+), 13 deletions(-) create mode 100644 mypy/test/testexportjson.py create mode 100644 test-data/unit/exportjson.test diff --git a/mypy/exportjson.py b/mypy/exportjson.py index 83090f3dabf8..83d220383be2 100644 --- a/mypy/exportjson.py +++ b/mypy/exportjson.py @@ -67,16 +67,21 @@ JsonDict: _TypeAlias = dict[str, Any] -def convert_binary_cache_to_json(data: bytes) -> JsonDict: +class Config: + def __init__(self, *, implicit_names: bool = True) -> None: + self.implicit_names = implicit_names + + +def convert_binary_cache_to_json(data: bytes, *, implicit_names: bool = True) -> JsonDict: tree = MypyFile.read(Buffer(data)) - return convert_mypy_file_to_json(tree) + return convert_mypy_file_to_json(tree, Config(implicit_names=implicit_names)) -def convert_mypy_file_to_json(self: MypyFile) -> JsonDict: +def convert_mypy_file_to_json(self: MypyFile, cfg: Config) -> JsonDict: return { ".class": "MypyFile", "_fullname": self._fullname, - "names": convert_symbol_table(self.names, self._fullname), + "names": convert_symbol_table(self.names, cfg), "is_stub": self.is_stub, "path": self.path, "is_partial_stub_package": self.is_partial_stub_package, @@ -84,7 +89,7 @@ def convert_mypy_file_to_json(self: MypyFile) -> JsonDict: } -def convert_symbol_table(self: SymbolTable, fullname: str) -> JsonDict: +def convert_symbol_table(self: SymbolTable, cfg: Config) -> JsonDict: data: JsonDict = {".class": "SymbolTable"} for key, value in self.items(): # Skip __builtins__: it's a reference to the builtins @@ -93,11 +98,20 @@ def convert_symbol_table(self: SymbolTable, fullname: str) -> JsonDict: # accessed by users of the module. if key == "__builtins__" or value.no_serialize: continue - data[key] = convert_symbol_table_node(value, fullname, key) + if not cfg.implicit_names and key in { + "__spec__", + "__package__", + "__file__", + "__doc__", + "__annotations__", + "__name__", + }: + continue + data[key] = convert_symbol_table_node(value, cfg) return data -def convert_symbol_table_node(self: SymbolTableNode, prefix: str | None, name: str) -> JsonDict: +def convert_symbol_table_node(self: SymbolTableNode, cfg: Config) -> JsonDict: data: JsonDict = {".class": "SymbolTableNode", "kind": node_kinds[self.kind]} if self.module_hidden: data["module_hidden"] = True @@ -110,11 +124,11 @@ def convert_symbol_table_node(self: SymbolTableNode, prefix: str | None, name: s if self.cross_ref: data["cross_ref"] = self.cross_ref elif self.node is not None: - data["node"] = convert_symbol_node(self.node) + data["node"] = convert_symbol_node(self.node, cfg) return data -def convert_symbol_node(self: SymbolNode) -> JsonDict: +def convert_symbol_node(self: SymbolNode, cfg: Config) -> JsonDict: if isinstance(self, FuncDef): return convert_func_def(self) elif isinstance(self, OverloadedFuncDef): @@ -124,7 +138,7 @@ def convert_symbol_node(self: SymbolNode) -> JsonDict: elif isinstance(self, Var): return convert_var(self) elif isinstance(self, TypeInfo): - return convert_type_info(self) + return convert_type_info(self, cfg) elif isinstance(self, TypeAlias): return convert_type_alias(self) elif isinstance(self, TypeVarExpr): @@ -210,12 +224,12 @@ def convert_var(self: Var) -> JsonDict: return data -def convert_type_info(self: TypeInfo) -> JsonDict: +def convert_type_info(self: TypeInfo, cfg: Config) -> JsonDict: data = { ".class": "TypeInfo", "module_name": self.module_name, "fullname": self.fullname, - "names": convert_symbol_table(self.names, self.fullname), + "names": convert_symbol_table(self.names, cfg), "defn": convert_class_def(self.defn), "abstract_attributes": self.abstract_attributes, "type_vars": self.type_vars, @@ -250,7 +264,12 @@ def convert_type_info(self: TypeInfo) -> JsonDict: def convert_class_def(self: ClassDef) -> JsonDict: - return {} + return { + ".class": "ClassDef", + "name": self.name, + "fullname": self.fullname, + "type_vars": [convert_type(v) for v in self.type_vars], + } def convert_type_alias(self: TypeAlias) -> JsonDict: diff --git a/mypy/test/testexportjson.py b/mypy/test/testexportjson.py new file mode 100644 index 000000000000..04740e6858f8 --- /dev/null +++ b/mypy/test/testexportjson.py @@ -0,0 +1,61 @@ +"""Test cases for the mypy cache JSON export tool.""" + +from __future__ import annotations + +import os +import json +import re +import pprint +import sys + +from mypy import build +from mypy.errors import CompileError +from mypy.modulefinder import BuildSource +from mypy.options import Options +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase, DataSuite +from mypy.test.helpers import assert_string_arrays_equal +from mypy.exportjson import convert_binary_cache_to_json + + +class TypeExportSuite(DataSuite): + required_out_section = True + files = ["exportjson.test"] + + def run_case(self, testcase: DataDrivenTestCase) -> None: + try: + src = "\n".join(testcase.input) + options = Options() + options.use_builtins_fixtures = True + options.show_traceback = True + options.allow_empty_bodies = True + options.fixed_format_cache = True + result = build.build( + sources=[BuildSource("main", None, src)], + options=options, + alt_lib_path=test_temp_dir, + ) + a = result.errors + + major, minor = sys.version_info[:2] + cache_dir = os.path.join(".mypy_cache", f"{major}.{minor}") + + for module in result.files: + if module in ("builtins", "typing", "_typeshed", "__main__"): + continue + fnam = os.path.join(cache_dir, f"{module}.data.ff") + with open(fnam, "rb") as f: + json_data = convert_binary_cache_to_json(f.read(), implicit_names=False) + for line in json.dumps(json_data, indent=4).splitlines(): + if '"path": ' in line: + # We source file path is unpredictable, so filter it out + line = re.sub(r'"[^"]+\.pyi?"', "...", line) + a.append(line) + print(fnam) + except CompileError as e: + a = e.messages + assert_string_arrays_equal( + testcase.output, + a, + f"Invalid output ({testcase.file}, line {testcase.line})", + ) diff --git a/test-data/unit/exportjson.test b/test-data/unit/exportjson.test new file mode 100644 index 000000000000..c08d1207371e --- /dev/null +++ b/test-data/unit/exportjson.test @@ -0,0 +1,124 @@ +-- Test cases for exporting mypy cache files to JSON (mypy.exportjson). +-- +-- The tool is maintained on a best effort basis so we don't attempt to have +-- full test coverage. + +[case testExportVar] +import m + +[file m.py] +x = 0 + +[out] +{ + ".class": "MypyFile", + "_fullname": "m", + "names": { + ".class": "SymbolTable", + "x": { + ".class": "SymbolTableNode", + "kind": "Gdef", + "node": { + ".class": "Var", + "name": "x", + "fullname": "m.x", + "type": { + ".class": "Instance", + "type_ref": null, + "args": [] + }, + "setter_type": null, + "flags": [ + "is_ready", + "is_inferred", + "has_explicit_value" + ] + } + } + }, + "is_stub": false, + "path": ..., + "is_partial_stub_package": false, + "future_import_flags": [] +} + +[case testExportClass] +import m + +[file m.py] +class C: + x: int + +[out] +{ + ".class": "MypyFile", + "_fullname": "m", + "names": { + ".class": "SymbolTable", + "C": { + ".class": "SymbolTableNode", + "kind": "Gdef", + "node": { + ".class": "TypeInfo", + "module_name": "m", + "fullname": "m.C", + "names": { + ".class": "SymbolTable", + "x": { + ".class": "SymbolTableNode", + "kind": "Mdef", + "node": { + ".class": "Var", + "name": "x", + "fullname": "m.C.x", + "type": { + ".class": "Instance", + "type_ref": null, + "args": [] + }, + "setter_type": null, + "flags": [ + "is_initialized_in_class", + "is_ready" + ] + } + } + }, + "defn": { + ".class": "ClassDef", + "name": "C", + "fullname": "m.C", + "type_vars": [] + }, + "abstract_attributes": [], + "type_vars": [], + "has_param_spec_type": false, + "bases": [ + { + ".class": "Instance", + "type_ref": "builtins.object", + "args": [] + } + ], + "mro": [], + "_promote": [], + "alt_promote": null, + "declared_metaclass": null, + "metaclass_type": null, + "tuple_type": null, + "typeddict_type": null, + "flags": [], + "metadata": {}, + "slots": null, + "deletable_attributes": [], + "self_type": null, + "dataclass_transform_spec": null, + "deprecated": null + } + } + }, + "is_stub": false, + "path": ..., + "is_partial_stub_package": false, + "future_import_flags": [] +} From 484ab81719b6315a558914065d75359cd8cb1ebf Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 15 Oct 2025 11:55:30 +0100 Subject: [PATCH 07/18] Improve tests --- mypy/test/testexportjson.py | 7 +++++-- test-data/unit/exportjson.test | 22 +++++++--------------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/mypy/test/testexportjson.py b/mypy/test/testexportjson.py index 04740e6858f8..93f675b3e715 100644 --- a/mypy/test/testexportjson.py +++ b/mypy/test/testexportjson.py @@ -30,8 +30,11 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: options.show_traceback = True options.allow_empty_bodies = True options.fixed_format_cache = True + fnam = os.path.join(self.base_path, "main.py") + with open(fnam, "w") as f: + f.write(src) result = build.build( - sources=[BuildSource("main", None, src)], + sources=[BuildSource(fnam, "main")], options=options, alt_lib_path=test_temp_dir, ) @@ -41,7 +44,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: cache_dir = os.path.join(".mypy_cache", f"{major}.{minor}") for module in result.files: - if module in ("builtins", "typing", "_typeshed", "__main__"): + if module in ("builtins", "typing", "_typeshed"): continue fnam = os.path.join(cache_dir, f"{module}.data.ff") with open(fnam, "rb") as f: diff --git a/test-data/unit/exportjson.test b/test-data/unit/exportjson.test index c08d1207371e..a77a9ac8856a 100644 --- a/test-data/unit/exportjson.test +++ b/test-data/unit/exportjson.test @@ -4,15 +4,11 @@ -- full test coverage. [case testExportVar] -import m - -[file m.py] x = 0 - [out] { ".class": "MypyFile", - "_fullname": "m", + "_fullname": "main", "names": { ".class": "SymbolTable", "x": { @@ -21,7 +17,7 @@ x = 0 "node": { ".class": "Var", "name": "x", - "fullname": "m.x", + "fullname": "main.x", "type": { ".class": "Instance", "type_ref": null, @@ -43,16 +39,12 @@ x = 0 } [case testExportClass] -import m - -[file m.py] class C: x: int - [out] { ".class": "MypyFile", - "_fullname": "m", + "_fullname": "main", "names": { ".class": "SymbolTable", "C": { @@ -60,8 +52,8 @@ class C: "kind": "Gdef", "node": { ".class": "TypeInfo", - "module_name": "m", - "fullname": "m.C", + "module_name": "main", + "fullname": "main.C", "names": { ".class": "SymbolTable", "x": { @@ -70,7 +62,7 @@ class C: "node": { ".class": "Var", "name": "x", - "fullname": "m.C.x", + "fullname": "main.C.x", "type": { ".class": "Instance", "type_ref": null, @@ -87,7 +79,7 @@ class C: "defn": { ".class": "ClassDef", "name": "C", - "fullname": "m.C", + "fullname": "main.C", "type_vars": [] }, "abstract_attributes": [], From 919045fa898ebe7c70ed25a1abf563b087e72928 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 15 Oct 2025 12:07:17 +0100 Subject: [PATCH 08/18] Fix and add test --- mypy/exportjson.py | 13 +++++++++++++ mypy/test/testexportjson.py | 33 +++++++++++++++++++-------------- test-data/unit/exportjson.test | 20 ++++++++++++++++++++ 3 files changed, 52 insertions(+), 14 deletions(-) diff --git a/mypy/exportjson.py b/mypy/exportjson.py index 83d220383be2..90a0e8114c22 100644 --- a/mypy/exportjson.py +++ b/mypy/exportjson.py @@ -58,6 +58,7 @@ TypeType, TypeVarTupleType, TypeVarType, + UnboundType, UninhabitedType, UnionType, UnpackType, @@ -357,6 +358,8 @@ def convert_type(typ: Type) -> JsonDict: return convert_parameters(typ) elif isinstance(typ, TypedDictType): return convert_typeddict_type(typ) + elif isinstance(typ, UnboundType): + return convert_unbound_type(typ) assert False, type(typ) @@ -516,6 +519,16 @@ def convert_typeddict_type(self: TypedDictType) -> JsonDict: } +def convert_unbound_type(self: UnboundType) -> JsonDict: + return { + ".class": "UnboundType", + "name": self.name, + "args": [convert_type(a) for a in self.args], + "expr": self.original_str_expr, + "expr_fallback": self.original_str_fallback, + } + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("path", nargs="+") diff --git a/mypy/test/testexportjson.py b/mypy/test/testexportjson.py index 93f675b3e715..8e1d5b901276 100644 --- a/mypy/test/testexportjson.py +++ b/mypy/test/testexportjson.py @@ -2,20 +2,19 @@ from __future__ import annotations -import os import json +import os import re -import pprint import sys from mypy import build from mypy.errors import CompileError +from mypy.exportjson import convert_binary_cache_to_json from mypy.modulefinder import BuildSource from mypy.options import Options from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase, DataSuite from mypy.test.helpers import assert_string_arrays_equal -from mypy.exportjson import convert_binary_cache_to_json class TypeExportSuite(DataSuite): @@ -23,8 +22,9 @@ class TypeExportSuite(DataSuite): files = ["exportjson.test"] def run_case(self, testcase: DataDrivenTestCase) -> None: + error = False + src = "\n".join(testcase.input) try: - src = "\n".join(testcase.input) options = Options() options.use_builtins_fixtures = True options.show_traceback = True @@ -34,17 +34,23 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: with open(fnam, "w") as f: f.write(src) result = build.build( - sources=[BuildSource(fnam, "main")], - options=options, - alt_lib_path=test_temp_dir, + sources=[BuildSource(fnam, "main")], options=options, alt_lib_path=test_temp_dir ) a = result.errors + error = bool(a) major, minor = sys.version_info[:2] cache_dir = os.path.join(".mypy_cache", f"{major}.{minor}") for module in result.files: - if module in ("builtins", "typing", "_typeshed"): + if module in ( + "builtins", + "typing", + "_typeshed", + "__future__", + "typing_extensions", + "sys", + ): continue fnam = os.path.join(cache_dir, f"{module}.data.ff") with open(fnam, "rb") as f: @@ -54,11 +60,10 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: # We source file path is unpredictable, so filter it out line = re.sub(r'"[^"]+\.pyi?"', "...", line) a.append(line) - print(fnam) except CompileError as e: a = e.messages - assert_string_arrays_equal( - testcase.output, - a, - f"Invalid output ({testcase.file}, line {testcase.line})", - ) + error = True + if error or "\n".join(testcase.output).strip() != "": + assert_string_arrays_equal( + testcase.output, a, f"Invalid output ({testcase.file}, line {testcase.line})" + ) diff --git a/test-data/unit/exportjson.test b/test-data/unit/exportjson.test index a77a9ac8856a..f0bad93c7a68 100644 --- a/test-data/unit/exportjson.test +++ b/test-data/unit/exportjson.test @@ -2,6 +2,9 @@ -- -- The tool is maintained on a best effort basis so we don't attempt to have -- full test coverage. +-- +-- Some tests only ensure that *some* JSON is generated successfully. These +-- have as [out]. [case testExportVar] x = 0 @@ -114,3 +117,20 @@ class C: "is_partial_stub_package": false, "future_import_flags": [] } + +[case testExportDifferentTypes] +from __future__ import annotations + +from typing import Callable, Any, Literal + +list_ann: list[int] +any_ann: Any +tuple_ann: tuple[int, str] +union_ann: int | None +callable_ann: Callable[[int], str] +type_type_ann: type[int] +literal_ann: Literal['x', 5, False] + +[builtins fixtures/tuple.pyi] +[out] + From 59d8550306f6347217984e98d7e7f947c4cf23c3 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 15 Oct 2025 12:09:16 +0100 Subject: [PATCH 09/18] Add tests --- test-data/unit/exportjson.test | 89 ++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/test-data/unit/exportjson.test b/test-data/unit/exportjson.test index f0bad93c7a68..17804da79418 100644 --- a/test-data/unit/exportjson.test +++ b/test-data/unit/exportjson.test @@ -118,6 +118,95 @@ class C: "future_import_flags": [] } +[case testExportCrossRef] +from typing import Any +[out] +{ + ".class": "MypyFile", + "_fullname": "main", + "names": { + ".class": "SymbolTable", + "Any": { + ".class": "SymbolTableNode", + "kind": "Gdef", + "cross_ref": "typing.Any" + } + }, + "is_stub": false, + "path": ..., + "is_partial_stub_package": false, + "future_import_flags": [] +} + +[case testExportFuncDef] +def foo(a: int) -> None: ... +[out] +{ + ".class": "MypyFile", + "_fullname": "main", + "names": { + ".class": "SymbolTable", + "foo": { + ".class": "SymbolTableNode", + "kind": "Gdef", + "node": { + ".class": "FuncDef", + "name": "foo", + "fullname": "main.foo", + "arg_names": [ + "a" + ], + "arg_kinds": [ + 0 + ], + "type": { + ".class": "CallableType", + "arg_types": [ + { + ".class": "Instance", + "type_ref": null, + "args": [] + } + ], + "arg_kinds": [ + 0 + ], + "arg_names": [ + "a" + ], + "ret_type": { + ".class": "NoneType" + }, + "fallback": { + ".class": "Instance", + "type_ref": null, + "args": [] + }, + "name": "foo", + "variables": [], + "is_ellipsis_args": false, + "implicit": false, + "is_bound": false, + "type_guard": null, + "type_is": null, + "from_concatenate": false, + "imprecise_arg_kinds": false, + "unpack_kwargs": false + }, + "flags": [], + "abstract_status": 0, + "dataclass_transform_spec": null, + "deprecated": null, + "original_first_arg": "a" + } + } + }, + "is_stub": false, + "path": ..., + "is_partial_stub_package": false, + "future_import_flags": [] +} + [case testExportDifferentTypes] from __future__ import annotations From 2bcfac98af484f22e37d1ce08c616223e89aab48 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 15 Oct 2025 12:53:03 +0100 Subject: [PATCH 10/18] Add test case --- test-data/unit/exportjson.test | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test-data/unit/exportjson.test b/test-data/unit/exportjson.test index 17804da79418..7b4b35f62688 100644 --- a/test-data/unit/exportjson.test +++ b/test-data/unit/exportjson.test @@ -223,3 +223,33 @@ literal_ann: Literal['x', 5, False] [builtins fixtures/tuple.pyi] [out] + +[case testExportDifferentNodes] +import typing + +from typing import overload, TypeVar +from typing_extensions import TypeVarTuple, ParamSpec + +@overload +def f(x: int) -> int: ... +@overload +def f(x: str) -> str: ... +def f(x: int | str) -> int | str: ... + +T = TypeVar("T") + +def deco(f: T) -> T: + return f + +@deco +def foo(x: int) -> int: ... + +X = int +x: X = 2 + +Ts = TypeVarTuple("Ts") +P = ParamSpec("P") + +[builtins fixtures/tuple.pyi] +[out] + From 8e3bd76794a2a10fa19ce4248b7989c1c6218f05 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 15 Oct 2025 13:21:41 +0100 Subject: [PATCH 11/18] Test more --- test-data/unit/exportjson.test | 40 +++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/test-data/unit/exportjson.test b/test-data/unit/exportjson.test index 7b4b35f62688..69a6cc1dcb30 100644 --- a/test-data/unit/exportjson.test +++ b/test-data/unit/exportjson.test @@ -210,7 +210,7 @@ def foo(a: int) -> None: ... [case testExportDifferentTypes] from __future__ import annotations -from typing import Callable, Any, Literal +from typing import Callable, Any, Literal, NoReturn list_ann: list[int] any_ann: Any @@ -220,6 +220,40 @@ callable_ann: Callable[[int], str] type_type_ann: type[int] literal_ann: Literal['x', 5, False] +def f() -> NoReturn: + assert False + +BadType = 1 +x: BadType # type: ignore + +[builtins fixtures/tuple.pyi] +[out] + + +[case testExportGenericTypes] +from __future__ import annotations + +from typing import TypeVar, Callable +from typing_extensions import TypeVarTuple, ParamSpec, Unpack, Concatenate + +T = TypeVar("T") + +def ident(x: T) -> T: + return x + +Ts = TypeVarTuple("Ts") + +def ts(t: tuple[Unpack[Ts]]) -> tuple[Unpack[Ts]]: + return t + +P = ParamSpec("P") + +def pspec(f: Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None: + f(*args, **kwargs) + +def concat(f: Callable[Concatenate[int, P], None], *args: P.args, **kwargs: P.kwargs) -> None: + f(1, *args, **kwargs) + [builtins fixtures/tuple.pyi] [out] @@ -228,7 +262,6 @@ literal_ann: Literal['x', 5, False] import typing from typing import overload, TypeVar -from typing_extensions import TypeVarTuple, ParamSpec @overload def f(x: int) -> int: ... @@ -247,9 +280,6 @@ def foo(x: int) -> int: ... X = int x: X = 2 -Ts = TypeVarTuple("Ts") -P = ParamSpec("P") - [builtins fixtures/tuple.pyi] [out] From 95887036173179f5baae45107ec610a5f165acb1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 15 Oct 2025 13:38:02 +0100 Subject: [PATCH 12/18] Fix Instance export --- mypy/exportjson.py | 106 +++++++++++++++++++-------------- test-data/unit/exportjson.test | 30 ++-------- 2 files changed, 67 insertions(+), 69 deletions(-) diff --git a/mypy/exportjson.py b/mypy/exportjson.py index 90a0e8114c22..d8a118911c24 100644 --- a/mypy/exportjson.py +++ b/mypy/exportjson.py @@ -43,8 +43,10 @@ node_kinds, ) from mypy.types import ( + NOT_READY, AnyType, CallableType, + ExtraAttrs, Instance, LiteralType, NoneType, @@ -65,7 +67,7 @@ get_proper_type, ) -JsonDict: _TypeAlias = dict[str, Any] +Json: _TypeAlias = dict[str, Any] | str class Config: @@ -73,12 +75,12 @@ def __init__(self, *, implicit_names: bool = True) -> None: self.implicit_names = implicit_names -def convert_binary_cache_to_json(data: bytes, *, implicit_names: bool = True) -> JsonDict: +def convert_binary_cache_to_json(data: bytes, *, implicit_names: bool = True) -> Json: tree = MypyFile.read(Buffer(data)) return convert_mypy_file_to_json(tree, Config(implicit_names=implicit_names)) -def convert_mypy_file_to_json(self: MypyFile, cfg: Config) -> JsonDict: +def convert_mypy_file_to_json(self: MypyFile, cfg: Config) -> Json: return { ".class": "MypyFile", "_fullname": self._fullname, @@ -90,8 +92,8 @@ def convert_mypy_file_to_json(self: MypyFile, cfg: Config) -> JsonDict: } -def convert_symbol_table(self: SymbolTable, cfg: Config) -> JsonDict: - data: JsonDict = {".class": "SymbolTable"} +def convert_symbol_table(self: SymbolTable, cfg: Config) -> Json: + data: Json = {".class": "SymbolTable"} for key, value in self.items(): # Skip __builtins__: it's a reference to the builtins # module that gets added to every module by @@ -112,8 +114,8 @@ def convert_symbol_table(self: SymbolTable, cfg: Config) -> JsonDict: return data -def convert_symbol_table_node(self: SymbolTableNode, cfg: Config) -> JsonDict: - data: JsonDict = {".class": "SymbolTableNode", "kind": node_kinds[self.kind]} +def convert_symbol_table_node(self: SymbolTableNode, cfg: Config) -> Json: + data: Json = {".class": "SymbolTableNode", "kind": node_kinds[self.kind]} if self.module_hidden: data["module_hidden"] = True if not self.module_public: @@ -129,7 +131,7 @@ def convert_symbol_table_node(self: SymbolTableNode, cfg: Config) -> JsonDict: return data -def convert_symbol_node(self: SymbolNode, cfg: Config) -> JsonDict: +def convert_symbol_node(self: SymbolNode, cfg: Config) -> Json: if isinstance(self, FuncDef): return convert_func_def(self) elif isinstance(self, OverloadedFuncDef): @@ -151,7 +153,7 @@ def convert_symbol_node(self: SymbolNode, cfg: Config) -> JsonDict: assert False, type(self) -def convert_func_def(self: FuncDef) -> JsonDict: +def convert_func_def(self: FuncDef) -> Json: return { ".class": "FuncDef", "name": self._name, @@ -172,7 +174,7 @@ def convert_func_def(self: FuncDef) -> JsonDict: } -def convert_dataclass_transform_spec(self: DataclassTransformSpec) -> JsonDict: +def convert_dataclass_transform_spec(self: DataclassTransformSpec) -> Json: return { "eq_default": self.eq_default, "order_default": self.order_default, @@ -182,7 +184,7 @@ def convert_dataclass_transform_spec(self: DataclassTransformSpec) -> JsonDict: } -def convert_overloaded_func_def(self: OverloadedFuncDef) -> JsonDict: +def convert_overloaded_func_def(self: OverloadedFuncDef) -> Json: return { ".class": "OverloadedFuncDef", "items": [convert_overload_part(i) for i in self.items], @@ -195,14 +197,14 @@ def convert_overloaded_func_def(self: OverloadedFuncDef) -> JsonDict: } -def convert_overload_part(self: OverloadPart) -> JsonDict: +def convert_overload_part(self: OverloadPart) -> Json: if isinstance(self, FuncDef): return convert_func_def(self) else: return convert_decorator(self) -def convert_decorator(self: Decorator) -> JsonDict: +def convert_decorator(self: Decorator) -> Json: return { ".class": "Decorator", "func": convert_func_def(self.func), @@ -211,8 +213,8 @@ def convert_decorator(self: Decorator) -> JsonDict: } -def convert_var(self: Var) -> JsonDict: - data: JsonDict = { +def convert_var(self: Var) -> Json: + data: Json = { ".class": "Var", "name": self._name, "fullname": self._fullname, @@ -225,7 +227,7 @@ def convert_var(self: Var) -> JsonDict: return data -def convert_type_info(self: TypeInfo, cfg: Config) -> JsonDict: +def convert_type_info(self: TypeInfo, cfg: Config) -> Json: data = { ".class": "TypeInfo", "module_name": self.module_name, @@ -264,7 +266,7 @@ def convert_type_info(self: TypeInfo, cfg: Config) -> JsonDict: return data -def convert_class_def(self: ClassDef) -> JsonDict: +def convert_class_def(self: ClassDef) -> Json: return { ".class": "ClassDef", "name": self.name, @@ -273,8 +275,8 @@ def convert_class_def(self: ClassDef) -> JsonDict: } -def convert_type_alias(self: TypeAlias) -> JsonDict: - data: JsonDict = { +def convert_type_alias(self: TypeAlias) -> Json: + data: Json = { ".class": "TypeAlias", "fullname": self._fullname, "module": self.module, @@ -287,7 +289,7 @@ def convert_type_alias(self: TypeAlias) -> JsonDict: return data -def convert_type_var_expr(self: TypeVarExpr) -> JsonDict: +def convert_type_var_expr(self: TypeVarExpr) -> Json: return { ".class": "TypeVarExpr", "name": self._name, @@ -299,7 +301,7 @@ def convert_type_var_expr(self: TypeVarExpr) -> JsonDict: } -def convert_param_spec_expr(self: ParamSpecExpr) -> JsonDict: +def convert_param_spec_expr(self: ParamSpecExpr) -> Json: return { ".class": "ParamSpecExpr", "name": self._name, @@ -310,7 +312,7 @@ def convert_param_spec_expr(self: ParamSpecExpr) -> JsonDict: } -def convert_type_var_tuple_expr(self: TypeVarTupleExpr) -> JsonDict: +def convert_type_var_tuple_expr(self: TypeVarTupleExpr) -> Json: return { ".class": "TypeVarTupleExpr", "name": self._name, @@ -322,7 +324,7 @@ def convert_type_var_tuple_expr(self: TypeVarTupleExpr) -> JsonDict: } -def convert_type(typ: Type) -> JsonDict: +def convert_type(typ: Type) -> Json: if type(typ) is TypeAliasType: return convert_type_alias_type(typ) typ = get_proper_type(typ) @@ -363,17 +365,33 @@ def convert_type(typ: Type) -> JsonDict: assert False, type(typ) -def convert_instance(self: Instance) -> JsonDict: - data: JsonDict = { +def convert_instance(self: Instance) -> Json: + ready = self.type is not NOT_READY + if not self.args and not self.last_known_value and not self.extra_attrs: + return self.type.fullname if ready else self.type_ref + + data: Json = { ".class": "Instance", - "type_ref": self.type_ref, + "type_ref": self.type.fullname if ready else self.type_ref, "args": [convert_type(arg) for arg in self.args], } + if self.last_known_value is not None: + data["last_known_value"] = convert_type(self.last_known_value) + data["extra_attrs"] = convert_extra_attrs(self.extra_attrs) if self.extra_attrs else None return data -def convert_type_alias_type(self: TypeAliasType) -> JsonDict: - data: JsonDict = { +def convert_extra_attrs(self: ExtraAttrs) -> Json: + return { + ".class": "ExtraAttrs", + "attrs": {k: convert_type(v) for k, v in self.attrs.items()}, + "immutable": sorted(self.immutable), + "mod_name": self.mod_name, + } + + +def convert_type_alias_type(self: TypeAliasType) -> Json: + data: Json = { ".class": "TypeAliasType", "type_ref": self.type_ref, "args": [convert_type(arg) for arg in self.args], @@ -381,7 +399,7 @@ def convert_type_alias_type(self: TypeAliasType) -> JsonDict: return data -def convert_any_type(self: AnyType) -> JsonDict: +def convert_any_type(self: AnyType) -> Json: return { ".class": "AnyType", "type_of_any": self.type_of_any, @@ -390,11 +408,11 @@ def convert_any_type(self: AnyType) -> JsonDict: } -def convert_none_type(self: NoneType) -> JsonDict: +def convert_none_type(self: NoneType) -> Json: return {".class": "NoneType"} -def convert_union_type(self: UnionType) -> JsonDict: +def convert_union_type(self: UnionType) -> Json: return { ".class": "UnionType", "items": [convert_type(t) for t in self.items], @@ -402,7 +420,7 @@ def convert_union_type(self: UnionType) -> JsonDict: } -def convert_tuple_type(self: TupleType) -> JsonDict: +def convert_tuple_type(self: TupleType) -> Json: return { ".class": "TupleType", "items": [convert_type(t) for t in self.items], @@ -411,11 +429,11 @@ def convert_tuple_type(self: TupleType) -> JsonDict: } -def convert_literal_type(self: LiteralType) -> JsonDict: +def convert_literal_type(self: LiteralType) -> Json: return {".class": "LiteralType", "value": self.value, "fallback": convert_type(self.fallback)} -def convert_type_var_type(self: TypeVarType) -> JsonDict: +def convert_type_var_type(self: TypeVarType) -> Json: assert not self.id.is_meta_var() return { ".class": "TypeVarType", @@ -430,7 +448,7 @@ def convert_type_var_type(self: TypeVarType) -> JsonDict: } -def convert_callable_type(self: CallableType) -> JsonDict: +def convert_callable_type(self: CallableType) -> Json: return { ".class": "CallableType", "arg_types": [convert_type(t) for t in self.arg_types], @@ -452,23 +470,23 @@ def convert_callable_type(self: CallableType) -> JsonDict: } -def convert_overloaded(self: Overloaded) -> JsonDict: +def convert_overloaded(self: Overloaded) -> Json: return {".class": "Overloaded", "items": [convert_type(t) for t in self.items]} -def convert_type_type(self: TypeType) -> JsonDict: +def convert_type_type(self: TypeType) -> Json: return {".class": "TypeType", "item": convert_type(self.item)} -def convert_uninhabited_type(self: UninhabitedType) -> JsonDict: +def convert_uninhabited_type(self: UninhabitedType) -> Json: return {".class": "UninhabitedType"} -def convert_unpack_type(self: UnpackType) -> JsonDict: +def convert_unpack_type(self: UnpackType) -> Json: return {".class": "UnpackType", "type": convert_type(self.type)} -def convert_param_spec_type(self: ParamSpecType) -> JsonDict: +def convert_param_spec_type(self: ParamSpecType) -> Json: assert not self.id.is_meta_var() return { ".class": "ParamSpecType", @@ -483,7 +501,7 @@ def convert_param_spec_type(self: ParamSpecType) -> JsonDict: } -def convert_type_var_tuple_type(self: TypeVarTupleType) -> JsonDict: +def convert_type_var_tuple_type(self: TypeVarTupleType) -> Json: assert not self.id.is_meta_var() return { ".class": "TypeVarTupleType", @@ -498,7 +516,7 @@ def convert_type_var_tuple_type(self: TypeVarTupleType) -> JsonDict: } -def convert_parameters(self: Parameters) -> JsonDict: +def convert_parameters(self: Parameters) -> Json: return { ".class": "Parameters", "arg_types": [convert_type(t) for t in self.arg_types], @@ -509,7 +527,7 @@ def convert_parameters(self: Parameters) -> JsonDict: } -def convert_typeddict_type(self: TypedDictType) -> JsonDict: +def convert_typeddict_type(self: TypedDictType) -> Json: return { ".class": "TypedDictType", "items": [[n, convert_type(t)] for (n, t) in self.items.items()], @@ -519,7 +537,7 @@ def convert_typeddict_type(self: TypedDictType) -> JsonDict: } -def convert_unbound_type(self: UnboundType) -> JsonDict: +def convert_unbound_type(self: UnboundType) -> Json: return { ".class": "UnboundType", "name": self.name, diff --git a/test-data/unit/exportjson.test b/test-data/unit/exportjson.test index 69a6cc1dcb30..2020aed40ab1 100644 --- a/test-data/unit/exportjson.test +++ b/test-data/unit/exportjson.test @@ -21,11 +21,7 @@ x = 0 ".class": "Var", "name": "x", "fullname": "main.x", - "type": { - ".class": "Instance", - "type_ref": null, - "args": [] - }, + "type": "builtins.int", "setter_type": null, "flags": [ "is_ready", @@ -66,11 +62,7 @@ class C: ".class": "Var", "name": "x", "fullname": "main.C.x", - "type": { - ".class": "Instance", - "type_ref": null, - "args": [] - }, + "type": "builtins.int", "setter_type": null, "flags": [ "is_initialized_in_class", @@ -89,11 +81,7 @@ class C: "type_vars": [], "has_param_spec_type": false, "bases": [ - { - ".class": "Instance", - "type_ref": "builtins.object", - "args": [] - } + "builtins.object" ], "mro": [], "_promote": [], @@ -162,11 +150,7 @@ def foo(a: int) -> None: ... "type": { ".class": "CallableType", "arg_types": [ - { - ".class": "Instance", - "type_ref": null, - "args": [] - } + "builtins.int" ], "arg_kinds": [ 0 @@ -177,11 +161,7 @@ def foo(a: int) -> None: ... "ret_type": { ".class": "NoneType" }, - "fallback": { - ".class": "Instance", - "type_ref": null, - "args": [] - }, + "fallback": "builtins.function", "name": "foo", "variables": [], "is_ellipsis_args": false, From 95a04d9e6412c41a2d641417db74a6619d7412f2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 15 Oct 2025 13:54:12 +0100 Subject: [PATCH 13/18] Fix self check --- mypy/exportjson.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/mypy/exportjson.py b/mypy/exportjson.py index d8a118911c24..d04a049407d4 100644 --- a/mypy/exportjson.py +++ b/mypy/exportjson.py @@ -15,7 +15,8 @@ import argparse import json -from typing import Any, TypeAlias as _TypeAlias +from typing import Any, Union +from typing_extensions import TypeAlias as _TypeAlias from librt.internal import Buffer @@ -67,7 +68,7 @@ get_proper_type, ) -Json: _TypeAlias = dict[str, Any] | str +Json: _TypeAlias = Union[dict[str, Any], str] class Config: @@ -93,7 +94,7 @@ def convert_mypy_file_to_json(self: MypyFile, cfg: Config) -> Json: def convert_symbol_table(self: SymbolTable, cfg: Config) -> Json: - data: Json = {".class": "SymbolTable"} + data: dict[str, Any] = {".class": "SymbolTable"} for key, value in self.items(): # Skip __builtins__: it's a reference to the builtins # module that gets added to every module by @@ -115,7 +116,7 @@ def convert_symbol_table(self: SymbolTable, cfg: Config) -> Json: def convert_symbol_table_node(self: SymbolTableNode, cfg: Config) -> Json: - data: Json = {".class": "SymbolTableNode", "kind": node_kinds[self.kind]} + data: dict[str, Any] = {".class": "SymbolTableNode", "kind": node_kinds[self.kind]} if self.module_hidden: data["module_hidden"] = True if not self.module_public: @@ -214,7 +215,7 @@ def convert_decorator(self: Decorator) -> Json: def convert_var(self: Var) -> Json: - data: Json = { + data: dict[str, Any] = { ".class": "Var", "name": self._name, "fullname": self._fullname, @@ -368,9 +369,12 @@ def convert_type(typ: Type) -> Json: def convert_instance(self: Instance) -> Json: ready = self.type is not NOT_READY if not self.args and not self.last_known_value and not self.extra_attrs: - return self.type.fullname if ready else self.type_ref + if ready: + return self.type.fullname + elif self.type_ref: + return self.type_ref - data: Json = { + data: dict[str, Any] = { ".class": "Instance", "type_ref": self.type.fullname if ready else self.type_ref, "args": [convert_type(arg) for arg in self.args], From c15e7c688cebd0e9cb1379db94f2fe02b557f866 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 15 Oct 2025 14:54:53 +0100 Subject: [PATCH 14/18] Fix mro export --- mypy/exportjson.py | 2 +- test-data/unit/exportjson.test | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mypy/exportjson.py b/mypy/exportjson.py index d04a049407d4..17c30ac04efd 100644 --- a/mypy/exportjson.py +++ b/mypy/exportjson.py @@ -239,7 +239,7 @@ def convert_type_info(self: TypeInfo, cfg: Config) -> Json: "type_vars": self.type_vars, "has_param_spec_type": self.has_param_spec_type, "bases": [convert_type(b) for b in self.bases], - "mro": [c.fullname for c in self.mro], + "mro": self._mro_refs, "_promote": [convert_type(p) for p in self._promote], "alt_promote": None if self.alt_promote is None else convert_type(self.alt_promote), "declared_metaclass": ( diff --git a/test-data/unit/exportjson.test b/test-data/unit/exportjson.test index 2020aed40ab1..b1a19460a7a8 100644 --- a/test-data/unit/exportjson.test +++ b/test-data/unit/exportjson.test @@ -83,7 +83,10 @@ class C: "bases": [ "builtins.object" ], - "mro": [], + "mro": [ + "main.C", + "builtins.object" + ], "_promote": [], "alt_promote": null, "declared_metaclass": null, From f30b7b269eb050d959d4a64543482a25dab1ca98 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 15 Oct 2025 15:11:37 +0100 Subject: [PATCH 15/18] Don't compile, since it needs to be runnable --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 1d093ec3b9e2..0037624f9bbc 100644 --- a/setup.py +++ b/setup.py @@ -81,6 +81,7 @@ def run(self) -> None: "__main__.py", "pyinfo.py", os.path.join("dmypy", "__main__.py"), + "exportjson.py", # Uses __getattr__/__setattr__ "split_namespace.py", # Lies to mypy about code reachability From 222a9fb103d7917ee7b4194b4ab35295e4238a46 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 15 Oct 2025 15:25:21 +0100 Subject: [PATCH 16/18] Improve command line interface --- mypy/exportjson.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mypy/exportjson.py b/mypy/exportjson.py index 17c30ac04efd..75b809996348 100644 --- a/mypy/exportjson.py +++ b/mypy/exportjson.py @@ -15,6 +15,7 @@ import argparse import json +import sys from typing import Any, Union from typing_extensions import TypeAlias as _TypeAlias @@ -552,11 +553,18 @@ def convert_unbound_type(self: UnboundType) -> Json: def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("path", nargs="+") + parser = argparse.ArgumentParser( + description="Convert binary cache files to JSON. " + "Create files in the same directory with extra .json extension." + ) + parser.add_argument( + "path", nargs="+", help="mypy cache data file to convert (.data.ff extension)" + ) args = parser.parse_args() fnams: list[str] = args.path for fnam in fnams: + if not fnam.endswith(".data.ff"): + sys.exit(f"error: Expected .data.ff extension, but got {fnam}") with open(fnam, "rb") as f: data = f.read() json_data = convert_binary_cache_to_json(data) From fb9134db3b540a56f8d113688ae42477f02c7411 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 15 Oct 2025 15:32:35 +0100 Subject: [PATCH 17/18] Succeed even if there are unsupported node types, but report error in output JSON Also add tests. --- mypy/exportjson.py | 4 ++-- mypy/test/testexportjson.py | 1 + test-data/unit/exportjson.test | 12 +++++++++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/mypy/exportjson.py b/mypy/exportjson.py index 75b809996348..09945f0ef28f 100644 --- a/mypy/exportjson.py +++ b/mypy/exportjson.py @@ -152,7 +152,7 @@ def convert_symbol_node(self: SymbolNode, cfg: Config) -> Json: return convert_param_spec_expr(self) elif isinstance(self, TypeVarTupleExpr): return convert_type_var_tuple_expr(self) - assert False, type(self) + return {"ERROR": f"{type(self)!r} unrecognized"} def convert_func_def(self: FuncDef) -> Json: @@ -364,7 +364,7 @@ def convert_type(typ: Type) -> Json: return convert_typeddict_type(typ) elif isinstance(typ, UnboundType): return convert_unbound_type(typ) - assert False, type(typ) + return {"ERROR": f"{type(typ)!r} unrecognized"} def convert_instance(self: Instance) -> Json: diff --git a/mypy/test/testexportjson.py b/mypy/test/testexportjson.py index 8e1d5b901276..13bd96d06642 100644 --- a/mypy/test/testexportjson.py +++ b/mypy/test/testexportjson.py @@ -59,6 +59,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: if '"path": ' in line: # We source file path is unpredictable, so filter it out line = re.sub(r'"[^"]+\.pyi?"', "...", line) + assert "ERROR" not in line, line a.append(line) except CompileError as e: a = e.messages diff --git a/test-data/unit/exportjson.test b/test-data/unit/exportjson.test index b1a19460a7a8..689c0fb5523e 100644 --- a/test-data/unit/exportjson.test +++ b/test-data/unit/exportjson.test @@ -193,7 +193,7 @@ def foo(a: int) -> None: ... [case testExportDifferentTypes] from __future__ import annotations -from typing import Callable, Any, Literal, NoReturn +from typing import Callable, Any, Literal, NoReturn, TypedDict, NamedTuple list_ann: list[int] any_ann: Any @@ -209,7 +209,17 @@ def f() -> NoReturn: BadType = 1 x: BadType # type: ignore +class TD(TypedDict): + x: int + +td = TD(x=1) + +NT = NamedTuple("NT", [("x", int)]) + +nt = NT(x=1) + [builtins fixtures/tuple.pyi] +[typing fixtures/typing-medium.pyi] [out] From 72085d91169085b3ccdcf891f15f6f6fe62aa9f0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 15 Oct 2025 16:50:14 +0100 Subject: [PATCH 18/18] Fix test on 3.9 --- test-data/unit/exportjson.test | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test-data/unit/exportjson.test b/test-data/unit/exportjson.test index 689c0fb5523e..14295281a48f 100644 --- a/test-data/unit/exportjson.test +++ b/test-data/unit/exportjson.test @@ -252,6 +252,8 @@ def concat(f: Callable[Concatenate[int, P], None], *args: P.args, **kwargs: P.kw [case testExportDifferentNodes] +from __future__ import annotations + import typing from typing import overload, TypeVar