SONARPY-944 Use precomputed Typeshed symbols for third-party libraries in the Python analyzer

Author: andreaguarino

python-frontend/src/main/resources/org/sonar/python/types/protobuf/annoy.protobuf

@@ -91,9 +91,13 @@ def __init__(self, _type: mpt.Type):
         elif isinstance(_type, mpt.TupleType):
             self.kind = TypeKind.TUPLE
             items = [TypeDescriptor(t) for t in _type.items]
-            item_names = [i.pretty_printed_name for i in items]
-            self.args.extend(items)
-            self.pretty_printed_name = f"Tuple[{','.join(item_names)}]"
+            if any(item.is_unknown for item in items):
+                self.kind = None
+                self.is_unknown = True
+            else:
+                item_names = [i.pretty_printed_name for i in items]
+                self.args.extend(items)
+                self.pretty_printed_name = f"Tuple[{','.join(item_names)}]"
         elif isinstance(_type, mpt.TypeVarType):
             self.kind = TypeKind.TYPE_VAR
             self.pretty_printed_name = _type.fullname
@@ -524,9 +528,10 @@ def extract_return_type(func_def: mpn.FuncDef):
     return TypeDescriptor(func_type.ret_type)
 
 
-def save_module(ms: Union[ModuleSymbol, MergedModuleSymbol], is_debug=False, debug_dir="output"):
+def save_module(ms: Union[ModuleSymbol, MergedModuleSymbol], is_debug=False, debug_dir="output", is_stdlib=True):
     ms_pb = ms.to_proto()
     save_dir = "../../src/main/resources/org/sonar/python/types/protobuf" if not is_debug else f"../{debug_dir}"
+    save_dir = save_dir if is_stdlib else os.path.join(save_dir, "stubs")
     save_string = ms_pb.SerializeToString() if not is_debug else str(ms_pb)
     open_mode = "wb" if not is_debug else "w"
     save_dir_path = os.path.join(CURRENT_PATH, save_dir)

python-frontend/typeshed_serializer/serializer/symbols.py

@@ -23,25 +23,29 @@
 from serializer.symbols import ModuleSymbol, MergedFunctionSymbol, MergedClassSymbol, MergedOverloadedFunctionSymbol, \
     MergedModuleSymbol, MergedVarSymbol
 from serializer import typeshed_serializer as ts
-from serializer.proto_out import symbols_pb2
 
 SUPPORTED_PYTHON_VERSIONS = ((2, 7), (3, 5), (3, 6), (3, 7), (3, 8), (3, 9), (3, 10))
 
 
-def build_multiple_python_version() -> Dict[str, Dict[str, ModuleSymbol]]:
+def build_multiple_python_version(is_third_parties=False) -> Dict[str, Dict[str, ModuleSymbol]]:
     model_by_version: Dict[str, Dict[str, ModuleSymbol]] = {}
     for major, minor in SUPPORTED_PYTHON_VERSIONS:
-        build_result = ts.walk_typeshed_stdlib(ts.get_options((major, minor)))
+        opt = ts.get_options((major, minor))
+        build_result, source_paths = ts.walk_typeshed_third_parties(opt) if is_third_parties else ts.walk_typeshed_stdlib(opt)
         modules = {}
         for file in build_result.files:
+            path = build_result.files[file].path
+            if is_third_parties and path not in source_paths:
+                # build_result contains more modules from stdlib unrelated to third_parties
+                continue
             ms = ModuleSymbol(build_result.files.get(file))
             modules[ms.fullname] = ms
         model_by_version[f"{major}{minor}"] = modules
     return model_by_version
 
 
-def merge_multiple_python_versions():
-    model_by_version = build_multiple_python_version()
+def merge_multiple_python_versions(is_third_parties=False):
+    model_by_version = build_multiple_python_version(is_third_parties)
     all_python_modules: Set[str] = set()
     for version in model_by_version:
         model = model_by_version[version]

python-frontend/typeshed_serializer/serializer/symbols_merger.py

@@ -25,7 +25,9 @@
 from serializer import symbols_merger, symbols
 
 STDLIB_PATH = "../resources/typeshed/stdlib"
+STUBS_PATH = "../resources/typeshed/stubs"
 CURRENT_PATH = os.path.dirname(__file__)
+THIRD_PARTIES_STUBS = os.listdir(os.path.join(CURRENT_PATH, STUBS_PATH))
 
 
 def get_options(python_version=(3, 8)):
@@ -60,13 +62,34 @@ def load_single_module(module_fqn: str, category="stdlib"):
 
 
 def walk_typeshed_stdlib(opt: options.Options = get_options()):
-    source_list = []
     generate_python2_stdlib = opt.python_version < (3, 0)
     relative_path = STDLIB_PATH if not generate_python2_stdlib else f"{STDLIB_PATH}/@python2"
+    source_list, source_paths = get_sources(relative_path, generate_python2_stdlib)
+    build_result = build.build(source_list, opt)
+    return build_result, source_paths
+
+
+def walk_typeshed_third_parties(opt: options.Options = get_options()):
+    source_list = []
+    source_paths = set()
+    generate_python2 = opt.python_version < (3, 0)
+    for third_party_stub in THIRD_PARTIES_STUBS:
+        stub_path = os.path.join(STUBS_PATH, third_party_stub)
+        relative_path = stub_path if not generate_python2 else f"{stub_path}/@python2"
+        src_list, src_paths = get_sources(relative_path, generate_python2)
+        source_list.extend(src_list)
+        source_paths = source_paths.union(src_paths)
+    build_result = build.build(source_list, opt)
+    return build_result, source_paths
+
+
+def get_sources(relative_path: str, generate_python2: bool):
+    source_list = []
+    source_paths = set()
     path = os.path.join(CURRENT_PATH, relative_path)
     for root, dirs, files in os.walk(path):
         package_name = root.replace(path, "").replace("\\", ".").replace("/", ".").lstrip(".")
-        if not generate_python2_stdlib and "python2" in package_name:
+        if not generate_python2 and "python2" in package_name:
             # Avoid python2 stubs
             continue
         for file in files:
@@ -80,8 +103,8 @@ def walk_typeshed_stdlib(opt: options.Options = get_options()):
             file_path = f"{root}/{file}"
             source = build.BuildSource(file_path, module=fq_module_name)
             source_list.append(source)
-    build_result = build.build(source_list, opt)
-    return build_result
+            source_paths.add(source.path)
+    return source_list, source_paths
 
 
 def serialize_typeshed_stdlib(output_dir_name="output", python_version=(3, 8), is_debug=False):
@@ -92,7 +115,7 @@ def serialize_typeshed_stdlib(output_dir_name="output", python_version=(3, 8), i
     """
     output_dir_name = output_dir_name if python_version >= (3, 0) else f"{output_dir_name}@python2"
     opt = get_options(python_version)
-    build_result = walk_typeshed_stdlib(opt)
+    build_result, _ = walk_typeshed_stdlib(opt)
     for file in build_result.files:
         module_symbol = symbols.ModuleSymbol(build_result.files.get(file))
         symbols.save_module(module_symbol, is_debug=is_debug, debug_dir=output_dir_name)
@@ -105,14 +128,15 @@ def serialize_typeshed_stdlib_multiple_python_version():
         serialize_typeshed_stdlib(f"output3{minor}", (3, minor), is_debug=True)
 
 
-def save_merged_symbols(is_debug=False):
-    merged_modules = symbols_merger.merge_multiple_python_versions()
+def save_merged_symbols(is_debug=False, is_third_parties=False):
+    merged_modules = symbols_merger.merge_multiple_python_versions(is_third_parties)
     for mod in merged_modules:
-        symbols.save_module(merged_modules[mod], is_debug=is_debug, debug_dir="output_merge")
+        symbols.save_module(merged_modules[mod], is_debug=is_debug, debug_dir="output_merge", is_stdlib=not is_third_parties)
 
 
 def main():
     save_merged_symbols()
+    save_merged_symbols(is_third_parties=True)
 
 
 if __name__ == '__main__':

python-frontend/typeshed_serializer/serializer/typeshed_serializer.py

@@ -24,12 +24,13 @@
 import pytest
 from mypy import build
 
-from serializer import typeshed_serializer
+from serializer import typeshed_serializer, symbols_merger
 
 
 @pytest.fixture(scope="session")
 def typeshed_stdlib():
-    return typeshed_serializer.walk_typeshed_stdlib()
+    build_result, _ = typeshed_serializer.walk_typeshed_stdlib()
+    return build_result
 
 
 @pytest.fixture(scope="session")
@@ -39,3 +40,8 @@ def fake_module_36_38():
     fake_module_36 = typeshed_serializer.build_single_module('fakemodule', python_version=(3, 6))
     fake_module_38 = typeshed_serializer.build_single_module('fakemodule', python_version=(3, 8))
     return [fake_module_36, fake_module_38]
+
+
+@pytest.fixture(scope="session")
+def typeshed_third_parties():
+    return symbols_merger.merge_multiple_python_versions(is_third_parties=True)