SONARPY-874 Serialize merged symbols for various Python version (#944)

Author: guillaume-dequenne

python-frontend/typeshed_serializer/serializer/symbols.py

@@ -1,5 +1,6 @@
 import os
 from enum import Enum
+from typing import List, Union
 
 import mypy.types as mpt
 import mypy.nodes as mpn
@@ -158,6 +159,9 @@ def __init__(self, overloaded_func_def: mpn.OverloadedFuncDef):
             if isinstance(item, mpn.Decorator):
                 self.definitions.append(FunctionSymbol(item.func, decorators=item.original_decorators))
 
+    def __eq__(self, other):
+        return isinstance(other, OverloadedFunctionSymbol) and self.to_proto() == other.to_proto()
+
     def to_proto(self) -> symbols_pb2.OverloadedFunctionSymbol:
         pb_overloaded_func = symbols_pb2.OverloadedFunctionSymbol()
         pb_overloaded_func.name = self.name
@@ -188,6 +192,9 @@ def __init__(self, func_def: mpn.FuncDef, decorators=None):
                 if decorator_name is not None:
                     self.resolved_decorator_names.append(decorator_name)
 
+    def __eq__(self, other):
+        return isinstance(other, FunctionSymbol) and self.to_proto() == other.to_proto()
+
     def to_proto(self) -> symbols_pb2.FunctionSymbol:
         pb_func = symbols_pb2.FunctionSymbol()
         pb_func.name = self.name
@@ -246,6 +253,19 @@ def __init__(self, type_info: mpn.TypeInfo):
                 self.metaclass_name = class_def.metaclass.fullname
         self.has_decorators = len(class_def.decorators) > 0
 
+    def __eq__(self, other):
+        if not isinstance(other, ClassSymbol):
+            return False
+        return (self.name == other.name
+                and self.fullname == other.fullname
+                and self.super_classes == other.super_classes
+                and self.mro == other.mro
+                and self.is_enum == other.is_enum
+                and self.is_generic == other.is_generic
+                and self.is_protocol == other.is_protocol
+                and self.metaclass_name == other.metaclass_name
+                and self.has_decorators == other.has_decorators)
+
     def to_proto(self) -> symbols_pb2.ClassSymbol:
         pb_class = symbols_pb2.ClassSymbol()
         pb_class.name = self.name
@@ -266,6 +286,86 @@ def to_proto(self) -> symbols_pb2.ClassSymbol:
         return pb_class
 
 
+class MergedFunctionSymbol:
+    def __init__(self, function_symbol: FunctionSymbol, valid_for: List[str]):
+        self.function_symbol = function_symbol
+        self.valid_for = valid_for
+
+    def to_proto(self) -> symbols_pb2.FunctionSymbol:
+        pb_func = self.function_symbol.to_proto()
+        for elem in self.valid_for:
+            pb_func.valid_for.append(elem)
+        return pb_func
+
+
+class MergedOverloadedFunctionSymbol:
+    def __init__(self, overloaded_function_symbol: OverloadedFunctionSymbol, valid_for: List[str]):
+        self.overloaded_function_symbol = overloaded_function_symbol
+        self.valid_for = valid_for
+
+    def to_proto(self) -> symbols_pb2.FunctionSymbol:
+        pb_func = self.overloaded_function_symbol.to_proto()
+        for elem in self.valid_for:
+            pb_func.valid_for.append(elem)
+        return pb_func
+
+
+class MergedClassSymbol:
+    def __init__(self, reference_class_symbols: ClassSymbol, merged_methods, merged_overloaded_methods,
+                 valid_for: List[str]):
+        # nested class symbols functions are not relevant anymore
+        self.class_symbol = reference_class_symbols
+        self.methods = merged_methods
+        self.overloaded_methods = merged_overloaded_methods
+        self.valid_for = valid_for
+
+    def to_proto(self) -> symbols_pb2.ClassSymbol:
+        pb_class = symbols_pb2.ClassSymbol()
+        pb_class.name = self.class_symbol.name
+        pb_class.fully_qualified_name = self.class_symbol.fullname
+        pb_class.super_classes.extend(self.class_symbol.super_classes)
+        pb_class.mro.extend(self.class_symbol.mro)
+        pb_class.has_decorators = self.class_symbol.has_decorators
+        pb_class.has_metaclass = self.class_symbol.has_metaclass
+        pb_class.is_enum = self.class_symbol.is_enum
+        pb_class.is_generic = self.class_symbol.is_generic
+        pb_class.is_protocol = self.class_symbol.is_protocol
+        if self.class_symbol.metaclass_name is not None:
+            pb_class.metaclass_name = self.class_symbol.metaclass_name
+        for method in self.methods:
+            for elem in self.methods[method]:
+                pb_class.methods.append(elem.to_proto())
+        for overloaded_func in self.overloaded_methods:
+            for elem in self.overloaded_methods[overloaded_func]:
+                pb_class.overloaded_methods.append(elem.to_proto())
+        for elem in self.valid_for:
+            pb_class.valid_for.append(elem)
+        return pb_class
+
+
+class MergedModuleSymbol:
+    def __init__(self, fullname, classes, functions, overloaded_functions):
+        self.fullname = fullname
+        self.classes = classes
+        self.functions = functions
+        self.overloaded_functions = overloaded_functions
+
+    def to_proto(self):
+        pb_module = symbols_pb2.ModuleSymbol()
+        pb_module.name = self.fullname  # FIXME: is it even useful to have name?
+        pb_module.fully_qualified_name = self.fullname
+        for cls in self.classes:
+            for elem in self.classes[cls]:
+                pb_module.classes.append(elem.to_proto())
+        for func in self.functions:
+            for elem in self.functions[func]:
+                pb_module.functions.append(elem.to_proto())
+        for overloaded_func in self.overloaded_functions:
+            for elem in self.overloaded_functions[overloaded_func]:
+                pb_module.overloaded_functions.append(elem.to_proto())
+        return pb_module
+
+
 class ModuleSymbol:
     def __init__(self, mypy_file: mpn.MypyFile):
         self.name = mypy_file.name
@@ -338,15 +438,25 @@ def extract_return_type(func_def: mpn.FuncDef):
     return TypeDescriptor(func_type.ret_type)
 
 
-def save_module(mypy_file: mpn.MypyFile, save_as_text=True, output_dir_name="output",
-                save_location="../../src/main/resources/org/sonar/python/types"):
-    ms = ModuleSymbol(mypy_file)
+def save_module(ms: Union[ModuleSymbol, MergedModuleSymbol], is_debug=False, debug_dir="output"):
     ms_pb = ms.to_proto()
-    save_dir = f"{save_location}/{output_dir_name}"
-    save_string = str(ms_pb) if save_as_text else ms_pb.SerializeToString()
-    open_mode = "w" if save_as_text else "wb"
+    save_dir = "../../src/main/resources/org/sonar/python/types/protobuf" if not is_debug else f"../{debug_dir}"
+    save_string = ms_pb.SerializeToString() if not is_debug else str(ms_pb)
+    open_mode = "wb" if not is_debug else "w"
     save_dir_path = os.path.join(CURRENT_PATH, save_dir)
     if not os.path.exists(save_dir_path):
         os.makedirs(save_dir_path)
-    with open(f"{save_dir_path}/{ms.fullname}.protobuf", open_mode) as f:
+    save_name = ms.fullname if not is_python_2_only_exception(ms) else f"2@{ms.fullname}"
+    with open(f"{save_dir_path}/{save_name}.protobuf", open_mode) as f:
         f.write(save_string)
+
+
+def is_python_2_only_exception(ms) -> bool:
+    """ This methods aims to flag some Python 2 modules whose name differ from their Python 3 counterpart
+    by capitalization only. This is done to avoid conflicts in the saved file for OS which are not case sensitive
+    (e.g Windows and macOS)
+    """
+    if (not isinstance(ms, MergedModuleSymbol)
+            or ms.fullname not in ['ConfigParser', 'Queue', 'SocketServer']):
+        return False
+    return True

python-frontend/typeshed_serializer/serializer/symbols_merger.py

@@ -0,0 +1,114 @@
+from typing import Dict, Set, List
+
+from serializer.symbols import ModuleSymbol, MergedFunctionSymbol, MergedClassSymbol, MergedOverloadedFunctionSymbol, \
+    MergedModuleSymbol
+from serializer import typeshed_serializer as ts
+
+SUPPORTED_PYTHON_VERSIONS = ((2, 7), (3, 5), (3, 6), (3, 7), (3, 8), (3, 9))
+
+
+def build_multiple_python_version() -> Dict[str, Dict[str, ModuleSymbol]]:
+    model_by_version: Dict[str, Dict[str, ModuleSymbol]] = {}
+    for major, minor in SUPPORTED_PYTHON_VERSIONS:
+        build_result = ts.walk_typeshed_stdlib(ts.get_options((major, minor)))
+        modules = {}
+        for file in build_result.files:
+            ms = ModuleSymbol(build_result.files.get(file))
+            modules[ms.fullname] = ms
+        model_by_version[f"{major}{minor}"] = modules
+    return model_by_version
+
+
+def merge_multiple_python_versions():
+    model_by_version = build_multiple_python_version()
+    all_python_modules: Set[str] = set()
+    for version in model_by_version:
+        model = model_by_version[version]
+        for module_fqn in model:
+            mod: ModuleSymbol = model[module_fqn]
+            all_python_modules.add(mod.fullname)
+    merged_modules = merge_modules(all_python_modules, model_by_version)
+    return merged_modules
+
+
+def merge_modules(all_python_modules: Set[str], model_by_version: Dict[str, Dict[str, ModuleSymbol]]):
+    merged_modules: Dict[str, MergedModuleSymbol] = {}
+    for python_mod in all_python_modules:
+        handled_classes: Dict[str, List[MergedClassSymbol]] = {}
+        handled_funcs: Dict[str, List[MergedFunctionSymbol]] = {}
+        handled_overloaded_functions: Dict[str, List[MergedOverloadedFunctionSymbol]] = {}
+        merged_modules[python_mod] = MergedModuleSymbol(python_mod, handled_classes,
+                                                        handled_funcs, handled_overloaded_functions)
+        for version in model_by_version:
+            model = model_by_version[version]
+            # get current module
+            if python_mod not in model:
+                continue
+            current_module = model[python_mod]
+            merge_classes(current_module, handled_classes, version)
+            merge_functions(current_module, handled_funcs, version)
+            merge_overloaded_functions(current_module, handled_overloaded_functions, version)
+    return merged_modules
+
+
+def merge_classes(current_module, handled_classes, version):
+    for mod_class in current_module.classes:
+        if mod_class.fullname not in handled_classes:
+            functions = {}
+            overloaded_functions = {}
+            merge_functions(mod_class, functions, version)
+            merge_overloaded_functions(mod_class, overloaded_functions, version)
+            handled_classes[mod_class.fullname] = [MergedClassSymbol(mod_class, functions,
+                                                                     overloaded_functions, [version])]
+        else:
+            # merge
+            compared = handled_classes[mod_class.fullname]
+            for elem in compared:
+                if elem.class_symbol == mod_class:
+                    functions = elem.methods
+                    overloaded_functions = elem.overloaded_methods
+                    merge_functions(mod_class, functions, version)
+                    merge_overloaded_functions(mod_class, overloaded_functions, version)
+                    elem.valid_for.append(version)
+                    break
+            else:
+                functions = {}
+                overloaded_functions = {}
+                merge_functions(mod_class, functions, version)
+                merge_overloaded_functions(mod_class, overloaded_functions, version)
+                compared.append(MergedClassSymbol(mod_class, functions, overloaded_functions, [version]))
+
+
+def merge_overloaded_functions(module_or_class, handled_overloaded_funcs, version):
+    functions = (module_or_class.overloaded_functions
+                 if isinstance(module_or_class, ModuleSymbol) else module_or_class.overloaded_methods)
+    for func in functions:
+        if func.fullname not in handled_overloaded_funcs:
+            # doesn't exist: we add it
+            handled_overloaded_funcs[func.fullname] = [MergedOverloadedFunctionSymbol(func, [version])]
+        else:
+            compared = handled_overloaded_funcs[func.fullname]
+            for elem in compared:
+                if elem.overloaded_function_symbol == func:
+                    elem.valid_for.append(version)
+                    break
+            else:
+                # no equivalent yet in the variations: add a new one
+                handled_overloaded_funcs[func.fullname].append(MergedOverloadedFunctionSymbol(func, [version]))
+
+
+def merge_functions(module_or_class, handled_funcs, version):
+    functions = module_or_class.functions if isinstance(module_or_class, ModuleSymbol) else module_or_class.methods
+    for func in functions:
+        if func.fullname not in handled_funcs:
+            # doesn't exist: we add it
+            handled_funcs[func.fullname] = [MergedFunctionSymbol(func, [version])]
+        else:
+            compared = handled_funcs[func.fullname]
+            for elem in compared:
+                if elem.function_symbol == func:
+                    elem.valid_for.append(version)
+                    break
+            else:
+                # no equivalent yet in the variations: add a new one
+                handled_funcs[func.fullname].append(MergedFunctionSymbol(func, [version]))

python-frontend/typeshed_serializer/serializer/typeshed_serializer.py

@@ -1,25 +1,23 @@
 import os
-import sys
-import collections
+
 from mypy import build, options
 
-from serializer.symbols import save_module
+from serializer import symbols_merger, symbols
 
-VersionInfoTuple = collections.namedtuple('version_info', ['major', 'minor', 'micro', 'releaselevel', 'serial'])
 STDLIB_PATH = "../resources/typeshed/stdlib"
 CURRENT_PATH = os.path.dirname(__file__)
 
 
-def get_options(python_version=None):
+def get_options(python_version=(3, 8)):
     opt = options.Options()
     # Setting incremental to false to avoid issues with mypy caching
     opt.incremental = False
-    if python_version is not None:
-        opt.python_version = python_version
+    opt.python_version = python_version
     return opt
 
 
-def build_single_module(module_fqn: str, category="stdlib", opt=get_options()):
+def build_single_module(module_fqn: str, category="stdlib", python_version=(3, 8)):
+    opt = get_options(python_version)
     module_source = load_single_module(module_fqn, category)
     build_result = build.build([module_source], opt)
     built_file = build_result.files.get(module_fqn)
@@ -66,24 +64,31 @@ def walk_typeshed_stdlib(opt: options.Options = get_options()):
     return build_result
 
 
-def serialize_typeshed_stdlib(output_dir_name="output", python_version=(3, 8)):
+def serialize_typeshed_stdlib(output_dir_name="output", python_version=(3, 8), is_debug=False):
     """ Serialize semantic model for Python standard library
     :param output_dir_name: Optional output directory name
     :param python_version: Optional version of Python to use for serialization
+    :param is_debug: debug flag
     """
     output_dir_name = output_dir_name if python_version >= (3, 0) else f"{output_dir_name}@python2"
     opt = get_options(python_version)
     build_result = walk_typeshed_stdlib(opt)
     for file in build_result.files:
-        save_module(build_result.files.get(file), save_as_text=True, output_dir_name=output_dir_name)
+        module_symbol = symbols.ModuleSymbol(build_result.files.get(file))
+        symbols.save_module(module_symbol, is_debug=is_debug, debug_dir=output_dir_name)
 
 
 def serialize_typeshed_stdlib_multiple_python_version():
     """ Serialize semantic model for Python stdlib versions from 3.5 to 3.9
     """
     for minor in range(5, 10):
-        sys.version_info = VersionInfoTuple(3, minor, 0, 'final', 0)
-        serialize_typeshed_stdlib(f"output3{minor}", (3, minor))
+        serialize_typeshed_stdlib(f"output3{minor}", (3, minor), is_debug=True)
+
+
+def save_merged_symbols(is_debug=False):
+    merged_modules = symbols_merger.merge_multiple_python_versions()
+    for mod in merged_modules:
+        symbols.save_module(merged_modules[mod], is_debug=is_debug, debug_dir="output_merge")
 
 
 def main():

python-frontend/typeshed_serializer/tests/conftest.py

@@ -0,0 +1,21 @@
+import os
+from unittest.mock import Mock
+
+import pytest
+from mypy import build
+
+from serializer import typeshed_serializer
+
+
+@pytest.fixture(scope="session")
+def typeshed_stdlib():
+    return typeshed_serializer.walk_typeshed_stdlib()
+
+
+@pytest.fixture(scope="session")
+def fake_module_36_38():
+    fake_module_path = os.path.join(os.path.dirname(__file__), "resources/fakemodule.pyi")
+    typeshed_serializer.load_single_module = Mock(return_value=build.BuildSource(fake_module_path, "fakemodule"))
+    fake_module_36 = typeshed_serializer.build_single_module('fakemodule', python_version=(3, 6))
+    fake_module_38 = typeshed_serializer.build_single_module('fakemodule', python_version=(3, 8))
+    return [fake_module_36, fake_module_38]