SONARPY-938 Serialize information about typeshed imported modules into protobuf

Author: andreaguarino

python-frontend/src/main/protobuf/symbols.proto

@@ -95,5 +95,6 @@ message VarSymbol {
   string name = 1;
   string fully_qualified_name = 2;
   optional Type type_annotation = 3;
-  repeated string valid_for = 4;
+  bool is_imported_module = 4;
+  repeated string valid_for = 5;
 }

python-frontend/typeshed_serializer/serializer/symbols.py

@@ -320,11 +320,15 @@ def to_proto(self) -> symbols_pb2.ClassSymbol:
 
 
 class VarSymbol:
-    def __init__(self, var: mpn.Var):
-        self.name = var.name
-        self.fullname = var.fullname
-        if var.type:
-            self.type = TypeDescriptor(var.type)
+    def __init__(self, name: str, fullname: str, is_imported_module=False, type_descriptor: TypeDescriptor = None):
+        self.name = name
+        self.fullname = fullname
+        self.is_imported_module = is_imported_module
+        self.type = type_descriptor
+
+    @classmethod
+    def from_var(cls, var: mpn.Var):
+        return cls(var.name, var.fullname, type_descriptor=TypeDescriptor(var.type) if var.type else None)
 
     def __eq__(self, other):
         return isinstance(other, VarSymbol) and self.to_proto() == other.to_proto()
@@ -333,7 +337,9 @@ def to_proto(self) -> symbols_pb2.VarSymbol:
         pb_var = symbols_pb2.VarSymbol()
         pb_var.name = self.name
         pb_var.fully_qualified_name = self.fullname
-        pb_var.type_annotation.CopyFrom(self.type.to_proto())
+        if self.type is not None:
+            pb_var.type_annotation.CopyFrom(self.type.to_proto())
+        pb_var.is_imported_module = self.is_imported_module
         return pb_var
 
 
@@ -354,7 +360,11 @@ def __init__(self, mypy_file: mpn.MypyFile):
             if isinstance(symbol_table_node, mpn.TypeInfo):
                 self.classes.append(ClassSymbol(symbol_table_node))
             if isinstance(symbol_table_node, mpn.Var) and symbol_table_node.name not in DEFAULT_EXPORTED_VARS:
-                self.vars.append(VarSymbol(symbol_table_node))
+                self.vars.append(VarSymbol.from_var(symbol_table_node))
+            if isinstance(symbol_table_node, mpn.MypyFile):
+                module_name = symbol_table_node.fullname
+                if module_name != "builtins":
+                    self.vars.append(VarSymbol(key, module_name, is_imported_module=True))
 
     def to_proto(self) -> symbols_pb2.ModuleSymbol:
         pb_module = symbols_pb2.ModuleSymbol()

python-frontend/typeshed_serializer/tests/test_symbols.py

@@ -37,6 +37,17 @@ def test_module_symbol(typeshed_stdlib):
     assert pb_module.fully_qualified_name == "abc"
     assert len(pb_module.classes) == 5
     assert len(pb_module.functions) == 4
+    imported_modules = [imported_module for imported_module in pb_module.vars if imported_module.is_imported_module is True]
+    assert len(imported_modules) == 0
+
+    os_module = typeshed_stdlib.files.get("os")
+    pb_module = symbols.ModuleSymbol(os_module).to_proto()
+    imported_modules = [imported_module for imported_module in pb_module.vars if imported_module.is_imported_module is True]
+    assert len(imported_modules) == 3
+    imported_modules = map(lambda m: (m.fully_qualified_name, m.name), imported_modules)
+    assert ("sys", "sys") in imported_modules
+    assert ("os.path", "_path") in imported_modules
+    assert ("os.path", "path") in imported_modules
 
 
 def test_class_symbol(typeshed_stdlib):

python-frontend/typeshed_serializer/tests/test_symbols_merger.py

@@ -205,14 +205,15 @@ def test_actual_module_merge(fake_module_36_38):
     assert len(fakemodule_proto.overloaded_functions) == len(flattened_overloaded_funcs)
 
     all_vars = merged_fakemodule_module.vars
-    assert len(all_vars) == 5
+    assert len(all_vars) == 6
     common_var = all_vars['fakemodule.common_var']
     assert len(common_var) == 1
     assert common_var[0].valid_for == ["36", "38"]
     var_symbol = common_var[0].var_symbol
     assert var_symbol.name == "common_var"
     assert var_symbol.fullname == "fakemodule.common_var"
     assert var_symbol.type.fully_qualified_name == "builtins.bool"
+    assert var_symbol.is_imported_module is False
 
     unique_var_36 = all_vars['fakemodule.unique_var_36']
     assert len(unique_var_36) == 1
@@ -237,6 +238,10 @@ def test_actual_module_merge(fake_module_36_38):
     assert alias_symbol.type.fully_qualified_name is None
     assert alias_symbol.type.is_unknown is True
 
+    imported_sys = all_vars['sys']
+    assert len(imported_sys) == 1
+    assert imported_sys[0].var_symbol.is_imported_module is True
+
 
 def assert_merged_class_symbol_to_proto(merged_classes_proto, merged_classes):
     assert len(merged_classes_proto) == len(merged_classes)