SONARPY-942 Serialize unanalyzed overloaded items when regular ones are missing (#1032)

Author: guillaume-dequenne

python-frontend/src/main/java/org/sonar/python/semantic/ClassSymbolImpl.java

@@ -146,10 +146,6 @@ public ClassSymbolImpl(SymbolsProtos.ClassSymbol classSymbolProto) {
       .forEach(proto -> descriptorsByFqn.computeIfAbsent(proto.getFullname(), d -> new HashSet<>()).add(proto));
     for (Map.Entry<String, Set<Object>> entry : descriptorsByFqn.entrySet()) {
       Set<Symbol> symbols = symbolsFromDescriptor(entry.getValue(), true);
-      // FIXME: SONARPY-942
-      if (symbols.isEmpty()) {
-        continue;
-      }
       methods.add(symbols.size() > 1 ? AmbiguousSymbolImpl.create(symbols) : symbols.iterator().next());
     }
     addMembers(methods);

python-frontend/src/main/java/org/sonar/python/types/TypeShed.java

@@ -453,10 +453,8 @@ public static Set<Symbol> symbolsFromDescriptor(Set<Object> descriptors, boolean
         symbols.add(new FunctionSymbolImpl(((SymbolsProtos.FunctionSymbol) descriptor), isInsideClass));
       }
       if (descriptor instanceof OverloadedFunctionSymbol) {
-        // FIXME: SONARPY-942
         if (((OverloadedFunctionSymbol) descriptor).getDefinitionsList().size() < 2) {
-          LOG.error("Overloaded function symbols should have at least two definitions");
-          continue;
+          throw new IllegalStateException("Overloaded function symbols should have at least two definitions.");
         }
         symbols.add(fromOverloadedFunction(((OverloadedFunctionSymbol) descriptor), isInsideClass));
       }

python-frontend/src/main/resources/org/sonar/python/types/protobuf/__builtin__.protobuf

@@ -857,7 +857,7 @@ ValuesViewtyping.ValuesView"typing.MappingView"typing.Iterable*
 	__trunc__$__builtin__._SupportsTrunc.__trunc__""
 __builtin__.int"__builtin__.int*B
 self
8
-__builtin__._SupportsTrunc"__builtin__._SupportsTruncz27X
j27�
+__builtin__._SupportsTrunc"__builtin__._SupportsTruncz27X
j27�
 object__builtin__.object"builtins.object*i
 __init____builtin__.object.__init__"
 None*2
@@ -940,8 +940,20 @@ __reduce____builtin__.object.__reduce__"
 self
(
 __builtin__.object"__builtin__.object*0
 protocol
"
-__builtin__.int"__builtin__.intz272-
-	__class____builtin__.object.__class__"27j27�
+__builtin__.int"__builtin__.intz272�
+	__class____builtin__.object.__class__�

+	__class____builtin__.object.__class__",
+Type[__builtin__._T]
+__builtin__._T*
+self

+__builtin__._T0
:property�

+	__class____builtin__.object.__class__"
+None*2
+self
(
+__builtin__.object"__builtin__.object*P
+__typeF
+Type[__builtin__.object](
+__builtin__.object"__builtin__.object0
:__class__.setter"27j27�
 staticmethod__builtin__.staticmethod"__builtin__.object*�

 __init__!__builtin__.staticmethod.__init__"
 None*>

python-frontend/typeshed_serializer/serializer/symbols.py

@@ -18,6 +18,7 @@
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
 
+import logging
 import os
 from enum import Enum
 from typing import List, Union
@@ -29,6 +30,8 @@
 
 CURRENT_PATH = os.path.dirname(__file__)
 
+logger = logging.getLogger(__name__)
+
 
 class ParamKind(Enum):
     POSITIONAL_ONLY = 0
@@ -173,11 +176,23 @@ def __init__(self, overloaded_func_def: mpn.OverloadedFuncDef):
         self.fullname = overloaded_func_def.fullname
         self.definitions = []
         for item in overloaded_func_def.items:
-            if isinstance(item, mpn.FuncDef):
-                # Should not happen?
-                self.definitions.append(FunctionSymbol(item))
-            if isinstance(item, mpn.Decorator):
-                self.definitions.append(FunctionSymbol(item.func, decorators=item.original_decorators))
+            self.add_overloaded_func_definition(item)
+        if len(self.definitions) < 2:
+            # Consider unanalyzed items if analyzed definitions are missing
+            if len(overloaded_func_def.unanalyzed_items) > 0:
+                logger.warning(f'Overloaded function definitions of '
+                               f'"{overloaded_func_def.fullname}" are missing: falling back on unanalyzed items.')
+            for item in overloaded_func_def.unanalyzed_items:
+                self.add_overloaded_func_definition(item)
+        if len(self.definitions) < 2:
+            raise RuntimeError("Overloaded function symbol should contain at least 2 definitions.")
+
+    def add_overloaded_func_definition(self, item):
+        if isinstance(item, mpn.FuncDef):
+            # Should not happen?
+            self.definitions.append(FunctionSymbol(item))
+        if isinstance(item, mpn.Decorator):
+            self.definitions.append(FunctionSymbol(item.func, decorators=item.original_decorators))
 
     def __eq__(self, other):
         return isinstance(other, OverloadedFunctionSymbol) and self.to_proto() == other.to_proto()

python-frontend/typeshed_serializer/tests/test_symbols.py

@@ -20,6 +20,8 @@
 
 from unittest import mock
 
+import pytest
+
 from serializer import symbols
 import mypy.nodes as mpn
 
@@ -112,3 +114,35 @@ def test_python2_exception():
     other_symbol = symbols.MergedModuleSymbol("other", {}, {}, {})
     assert symbols.is_python_2_only_exception(queue_symbol) is True
     assert symbols.is_python_2_only_exception(other_symbol) is False
+
+
+def mock_add_definition(self, arg):
+    assert isinstance(arg, mpn.Decorator)
+    self.definitions.append(arg)
+
+
+def test_fallback_unanalyzed_items_when_items_are_missing():
+    overloaded_func_mock = mock.Mock(mpn.OverloadedFuncDef)
+    overloaded_func_mock.items = []
+    overloaded_func_mock.fullname = "overloaded_func_mock_fullname"
+    func_def_mock1 = mock.Mock(mpn.Decorator)
+    func_def_mock2 = mock.Mock(mpn.Decorator)
+    overloaded_func_mock.unanalyzed_items = [func_def_mock1, func_def_mock2]
+
+    with mock.patch('serializer.symbols.OverloadedFunctionSymbol.add_overloaded_func_definition', mock_add_definition), \
+            mock.patch('logging.Logger.warning') as log_mock:
+        overloaded_func_symbol = symbols.OverloadedFunctionSymbol(overloaded_func_mock)
+        assert len(overloaded_func_symbol.definitions) == 2
+        log_mock.assert_called_with('Overloaded function definitions of "overloaded_func_mock_fullname" are missing: '
+                                    'falling back on unanalyzed items.')
+
+
+def test_error_when_overloaded_definitions_are_missing():
+    overloaded_func_mock = mock.Mock(mpn.OverloadedFuncDef)
+    overloaded_func_mock.items = []
+    overloaded_func_mock.unanalyzed_items = []
+
+    with mock.patch('serializer.symbols.OverloadedFunctionSymbol.add_overloaded_func_definition', mock_add_definition):
+        with pytest.raises(RuntimeError) as raised:
+            symbols.OverloadedFunctionSymbol(overloaded_func_mock)
+        assert raised.value.args[0] == 'Overloaded function symbol should contain at least 2 definitions.'