SONARPY-970 Serialize stub class members to Protobuf

Author: guillaume-dequenne

python-frontend/src/main/java/org/sonar/python/semantic/ClassSymbolImpl.java

@@ -137,22 +137,25 @@ public ClassSymbolImpl(SymbolsProtos.ClassSymbol classSymbolProto, String module
     hasMetaClass = classSymbolProto.getHasMetaclass();
     metaclassFQN = classSymbolProto.getMetaclassName();
     supportsGenerics = classSymbolProto.getIsGeneric();
-    Set<Symbol> methods = new HashSet<>();
+    Set<Symbol> classMembers = new HashSet<>();
     Map<String, Set<Object>> descriptorsByFqn = new HashMap<>();
     classSymbolProto.getMethodsList().stream()
       .filter(d -> isValidForProjectPythonVersion(d.getValidForList()))
       .forEach(proto -> descriptorsByFqn.computeIfAbsent(proto.getFullyQualifiedName(), d -> new HashSet<>()).add(proto));
     classSymbolProto.getOverloadedMethodsList().stream()
       .filter(d -> isValidForProjectPythonVersion(d.getValidForList()))
       .forEach(proto -> descriptorsByFqn.computeIfAbsent(proto.getFullname(), d -> new HashSet<>()).add(proto));
+    classSymbolProto.getAttributesList().stream()
+      .filter(d -> isValidForProjectPythonVersion(d.getValidForList()))
+      .forEach(proto -> descriptorsByFqn.computeIfAbsent(proto.getFullyQualifiedName(), d -> new HashSet<>()).add(proto));
 
     inlineInheritedMethodsFromPrivateClass(classSymbolProto.getSuperClassesList(), descriptorsByFqn);
 
     for (Map.Entry<String, Set<Object>> entry : descriptorsByFqn.entrySet()) {
       Set<Symbol> symbols = symbolsFromProtobufDescriptors(entry.getValue(), fullyQualifiedName, moduleName);
-      methods.add(symbols.size() > 1 ? AmbiguousSymbolImpl.create(symbols) : symbols.iterator().next());
+      classMembers.add(symbols.size() > 1 ? AmbiguousSymbolImpl.create(symbols) : symbols.iterator().next());
     }
-    addMembers(methods);
+    addMembers(classMembers);
     superClassesFqns.addAll(classSymbolProto.getSuperClassesList().stream().map(TypeShed::normalizedFqn).collect(Collectors.toList()));
     superClassesFqns.removeAll(inlinedSuperClassFqn);
     validForPythonVersions = new HashSet<>(classSymbolProto.getValidForList());

python-frontend/src/main/protobuf/symbols.proto

@@ -81,6 +81,7 @@ message ClassSymbol {
   bool is_protocol = 11;
   optional string metaclass_name = 12;
   repeated string valid_for = 13;
+  repeated VarSymbol attributes = 14;
 }
 
 message ModuleSymbol {

python-frontend/typeshed_serializer/serializer/symbols.py

@@ -264,6 +264,7 @@ def __init__(self, type_info: mpn.TypeInfo):
         self.super_classes = []
         self.methods = []
         self.overloaded_methods = []
+        self.vars = []
         self.is_enum = type_info.is_enum
         self.is_generic = type_info.is_generic()
         self.is_protocol = type_info.is_protocol
@@ -276,10 +277,12 @@ def __init__(self, type_info: mpn.TypeInfo):
             node = name.node
             if isinstance(node, mpn.FuncDef):
                 self.methods.append(FunctionSymbol(node))
-            if isinstance(node, mpn.Decorator):
+            elif isinstance(node, mpn.Decorator):
                 self.methods.append(FunctionSymbol(node.func, decorators=node.original_decorators))
-            if isinstance(node, mpn.OverloadedFuncDef):
+            elif isinstance(node, mpn.OverloadedFuncDef):
                 self.overloaded_methods.append(OverloadedFunctionSymbol(node))
+            elif isinstance(node, mpn.Var) and node.name not in DEFAULT_EXPORTED_VARS:
+                self.vars.append(VarSymbol.from_var(node))
         class_def = type_info.defn
         self.has_metaclass = class_def.metaclass is not None
         if class_def.metaclass is not None:
@@ -316,6 +319,8 @@ def to_proto(self) -> symbols_pb2.ClassSymbol:
             pb_class.methods.append(method.to_proto())
         for overloaded_method in self.overloaded_methods:
             pb_class.overloaded_methods.append(overloaded_method.to_proto())
+        for var in self.vars:
+            pb_class.attributes.append(var.to_proto())
         return pb_class
 
 
@@ -406,11 +411,12 @@ def to_proto(self) -> symbols_pb2.FunctionSymbol:
 
 class MergedClassSymbol:
     def __init__(self, reference_class_symbols: ClassSymbol, merged_methods, merged_overloaded_methods,
-                 valid_for: List[str]):
+                 merged_attributes, valid_for: List[str]):
         # nested class symbols functions are not relevant anymore
         self.class_symbol = reference_class_symbols
         self.methods = merged_methods
         self.overloaded_methods = merged_overloaded_methods
+        self.vars = merged_attributes
         self.valid_for = valid_for
 
     def to_proto(self) -> symbols_pb2.ClassSymbol:
@@ -431,6 +437,9 @@ def to_proto(self) -> symbols_pb2.ClassSymbol:
         for overloaded_func in self.overloaded_methods:
             for elem in self.overloaded_methods[overloaded_func]:
                 pb_class.overloaded_methods.append(elem.to_proto())
+        for var in self.vars:
+            for elem in self.vars[var]:
+                pb_class.attributes.append(elem.to_proto())
         for elem in self.valid_for:
             pb_class.valid_for.append(elem)
         return pb_class

python-frontend/typeshed_serializer/serializer/symbols_merger.py

@@ -74,8 +74,8 @@ def merge_modules(all_python_modules: Set[str], model_by_version: Dict[str, Dict
     return merged_modules
 
 
-def merge_vars(current_module, handled_vars, version):
-    for var in current_module.vars:
+def merge_vars(module_or_class, handled_vars, version):
+    for var in module_or_class.vars:
         if var.fullname not in handled_vars:
             # doesn't exist: we add it
             handled_vars[var.fullname] = [MergedVarSymbol(var, [version])]
@@ -95,27 +95,33 @@ def merge_classes(current_module, handled_classes, version):
         if mod_class.fullname not in handled_classes:
             functions = {}
             overloaded_functions = {}
+            variables = {}
             merge_functions(mod_class, functions, version)
             merge_overloaded_functions(mod_class, overloaded_functions, version)
+            merge_vars(mod_class, variables, version)
             handled_classes[mod_class.fullname] = [MergedClassSymbol(mod_class, functions,
-                                                                     overloaded_functions, [version])]
+                                                                     overloaded_functions, variables, [version])]
         else:
             # merge
             compared = handled_classes[mod_class.fullname]
             for elem in compared:
                 if elem.class_symbol == mod_class:
                     functions = elem.methods
                     overloaded_functions = elem.overloaded_methods
+                    variables = elem.vars
                     merge_functions(mod_class, functions, version)
                     merge_overloaded_functions(mod_class, overloaded_functions, version)
+                    merge_vars(mod_class, variables, version)
                     elem.valid_for.append(version)
                     break
             else:
                 functions = {}
                 overloaded_functions = {}
+                variables = {}
                 merge_functions(mod_class, functions, version)
                 merge_overloaded_functions(mod_class, overloaded_functions, version)
-                compared.append(MergedClassSymbol(mod_class, functions, overloaded_functions, [version]))
+                merge_vars(mod_class, variables, version)
+                compared.append(MergedClassSymbol(mod_class, functions, overloaded_functions, variables, [version]))
 
 
 def merge_overloaded_functions(module_or_class, handled_overloaded_funcs, version):

python-frontend/typeshed_serializer/tests/resources/fakemodule.pyi

@@ -103,3 +103,12 @@ else:
 common_var: bool
 
 alias = common_overloaded_function
+
+class ClassWithFields:
+    common_field: str
+    if sys.version_info >= (3, 8):
+        field_unique_38: int
+        field_multiple_defs: str
+    else:
+        field_unique_36: int
+        field_multiple_defs: int

python-frontend/typeshed_serializer/tests/test_symbols.py

@@ -37,12 +37,14 @@ def test_module_symbol(typeshed_stdlib):
     assert pb_module.fully_qualified_name == "abc"
     assert len(pb_module.classes) == 5
     assert len(pb_module.functions) == 4
-    imported_modules = [imported_module for imported_module in pb_module.vars if imported_module.is_imported_module is True]
+    imported_modules = [imported_module for imported_module in pb_module.vars if
+                        imported_module.is_imported_module is True]
     assert len(imported_modules) == 0
 
     os_module = typeshed_stdlib.files.get("os")
     pb_module = symbols.ModuleSymbol(os_module).to_proto()
-    imported_modules = [imported_module for imported_module in pb_module.vars if imported_module.is_imported_module is True]
+    imported_modules = [imported_module for imported_module in pb_module.vars if
+                        imported_module.is_imported_module is True]
     assert len(imported_modules) == 3
     imported_modules = map(lambda m: (m.fully_qualified_name, m.name), imported_modules)
     assert ("sys", "sys") in imported_modules
@@ -57,6 +59,8 @@ def test_class_symbol(typeshed_stdlib):
     assert cmd_class_symbol.fullname == "cmd.Cmd"
     assert cmd_class_symbol.name == "Cmd"
     assert cmd_class_symbol.super_classes == ["builtins.object"]
+    assert len(cmd_class_symbol.methods) == 18
+    assert len(cmd_class_symbol.vars) == 17
 
     pb_class_symbol = cmd_class_symbol.to_proto()
     assert pb_class_symbol.name == "Cmd"

python-frontend/typeshed_serializer/tests/test_symbols_merger.py

@@ -139,7 +139,7 @@ def test_actual_module_merge(fake_module_36_38):
     merged_fakemodule_module = merged_modules['fakemodule']
     classes_dict = merged_fakemodule_module.classes
 
-    assert len(classes_dict) == 4
+    assert len(classes_dict) == 5
 
     # Class unique to Python 3.6 is present
     fakemodule_someclassunique36_symbols = classes_dict['fakemodule.SomeClassUnique36']
@@ -242,6 +242,18 @@ def test_actual_module_merge(fake_module_36_38):
     assert len(imported_sys) == 1
     assert imported_sys[0].var_symbol.is_imported_module is True
 
+    fakemodule_class_with_fields_symbols = classes_dict['fakemodule.ClassWithFields']
+    assert len(fakemodule_class_with_fields_symbols) == 1
+    fakemodule_class_symbol = fakemodule_class_with_fields_symbols[0]
+    # Some fields are common
+    assert fakemodule_class_symbol.vars['fakemodule.ClassWithFields.common_field'][0].valid_for == ["36", "38"]
+    # Some fields exist only in a given Python version
+    assert fakemodule_class_symbol.vars['fakemodule.ClassWithFields.field_unique_36'][0].valid_for == ["36"]
+    assert fakemodule_class_symbol.vars['fakemodule.ClassWithFields.field_unique_38'][0].valid_for == ["38"]
+    # Some fields have different definitions depending on the Python version
+    assert fakemodule_class_symbol.vars['fakemodule.ClassWithFields.field_multiple_defs'][0].valid_for == ["36"]
+    assert fakemodule_class_symbol.vars['fakemodule.ClassWithFields.field_multiple_defs'][1].valid_for == ["38"]
+
 
 def assert_merged_class_symbol_to_proto(merged_classes_proto, merged_classes):
     assert len(merged_classes_proto) == len(merged_classes)