SONARPY-2069 Fix usage of V1 TypeShed during resolution of V2 types (#1924)

Author: guillaume-dequenne

python-frontend/src/main/java/org/sonar/python/semantic/FunctionSymbolImpl.java

@@ -321,6 +321,10 @@ public InferredType declaredReturnType() {
     }
     return declaredReturnType;
   }
+
+  public SymbolsProtos.Type protobufReturnType() {
+    return protobufReturnType;
+  }
 
   static class ParameterType {
     InferredType inferredType;

python-frontend/src/main/java/org/sonar/python/semantic/v2/SymbolsModuleTypeProvider.java

@@ -35,11 +35,12 @@
 import org.sonar.plugins.python.api.symbols.ClassSymbol;
 import org.sonar.plugins.python.api.symbols.FunctionSymbol;
 import org.sonar.plugins.python.api.symbols.Symbol;
-import org.sonar.plugins.python.api.types.InferredType;
 import org.sonar.python.semantic.ClassSymbolImpl;
 import org.sonar.python.semantic.FunctionSymbolImpl;
 import org.sonar.python.semantic.ProjectLevelSymbolTable;
 import org.sonar.python.semantic.SymbolImpl;
+import org.sonar.python.types.InferredTypes;
+import org.sonar.python.types.protobuf.SymbolsProtos;
 import org.sonar.python.types.v2.ClassType;
 import org.sonar.python.types.v2.FunctionType;
 import org.sonar.python.types.v2.LazyType;
@@ -120,9 +121,11 @@ private PythonType convertToFunctionType(FunctionSymbol symbol, Map<Symbol, Pyth
       .map(SymbolsModuleTypeProvider::convertParameter)
       .toList();
 
-    InferredType inferredType = ((FunctionSymbolImpl) symbol).declaredReturnType();
-    ClassSymbol classSymbol = inferredType.runtimeTypeSymbol();
-    var returnType = resolveReturnType(createdTypesBySymbol, classSymbol);
+    var returnType = PythonType.UNKNOWN;
+    var protoReturnType = ((FunctionSymbolImpl) symbol).protobufReturnType();
+    if (protoReturnType != null) {
+      returnType = convertProtobufType(protoReturnType);
+    }
     TypeOrigin typeOrigin = symbol.isStub() ? TypeOrigin.STUB : TypeOrigin.LOCAL;
 
     FunctionTypeBuilder functionTypeBuilder =
@@ -144,20 +147,6 @@ private PythonType convertToFunctionType(FunctionSymbol symbol, Map<Symbol, Pyth
     return functionType;
   }
 
-  private PythonType resolveReturnType(Map<Symbol, PythonType> createdTypesBySymbol, @Nullable ClassSymbol classSymbol) {
-    if (classSymbol == null) {
-      return PythonType.UNKNOWN;
-    }
-    if (createdTypesBySymbol.containsKey(classSymbol)) {
-      return createdTypesBySymbol.get(classSymbol);
-    }
-    String fullyQualifiedName = classSymbol.fullyQualifiedName();
-    if (fullyQualifiedName == null) {
-      return convertToType(classSymbol, createdTypesBySymbol);
-    }
-    return resolvePossibleLazyType(fullyQualifiedName);
-  }
-
   PythonType resolvePossibleLazyType(String fullyQualifiedName) {
     if (rootModule == null) {
       // If root module has not been created yet, return lazy type
@@ -245,4 +234,43 @@ private PythonType convertToType(Symbol symbol, Map<Symbol, PythonType> createdT
       case OTHER -> PythonType.UNKNOWN;
     };
   }
+
+  public PythonType convertProtobufType(SymbolsProtos.Type type) {
+    switch (type.getKind()) {
+      case INSTANCE:
+        String typeName = type.getFullyQualifiedName();
+        // _SpecialForm is the type used for some special types, like Callable, Union, TypeVar, ...
+        // It comes from CPython impl: https://github.com/python/cpython/blob/e39ae6bef2c357a88e232dcab2e4b4c0f367544b/Lib/typing.py#L439
+        // This doesn't seem to be very precisely specified in typeshed, because it has special semantic.
+        // To avoid FPs, we treat it as ANY
+        if ("typing._SpecialForm".equals(typeName)) {
+          return PythonType.UNKNOWN;
+        }
+        typeName = typeName.replaceFirst("^builtins\\.", "");
+        return typeName.isEmpty() ? PythonType.UNKNOWN : resolvePossibleLazyType(typeName);
+      case TYPE:
+        return resolvePossibleLazyType("type");
+      case TYPE_ALIAS:
+        return convertProtobufType(type.getArgs(0));
+      case CALLABLE:
+        // this should be handled as a function type - see SONARPY-953
+        return PythonType.UNKNOWN;
+      case UNION:
+        return UnionType.or(type.getArgsList().stream().map(this::convertProtobufType).collect(Collectors.toSet()));
+      case TUPLE:
+        return resolvePossibleLazyType("tuple");
+      case NONE:
+        return resolvePossibleLazyType("NoneType");
+      case TYPED_DICT:
+        return resolvePossibleLazyType("dict");
+      case TYPE_VAR:
+        return Optional.of(type)
+          .filter(InferredTypes::filterTypeVar)
+          .map(SymbolsProtos.Type::getFullyQualifiedName)
+          .map(this::resolvePossibleLazyType)
+          .orElse(PythonType.UNKNOWN);
+      default:
+        return PythonType.UNKNOWN;
+    }
+  }
 }

python-frontend/src/main/java/org/sonar/python/semantic/v2/TypeShed.java

@@ -91,7 +91,7 @@ public class TypeShed {
   public TypeShed(ProjectLevelSymbolTable projectLevelSymbolTable) {
     // workaround to initialize supported python versions used in ClassSymbolImpl
     // TODO: remove once v2 types model will be populated from TypeShed bypassing conversion to symbols
-    builtins = org.sonar.python.types.TypeShed.builtinSymbols();
+    org.sonar.python.types.TypeShed.builtinSymbols();
     typeShedSymbols = new HashMap<>();
     modulesInProgress = new HashSet<>();
     this.projectLevelSymbolTable = projectLevelSymbolTable;

python-frontend/src/main/java/org/sonar/python/types/InferredTypes.java

@@ -245,7 +245,7 @@ public static InferredType fromTypeshedProtobuf(SymbolsProtos.Type type) {
     "^builtins\\.object$",
     "^_ctypes\\._CanCastTo$"); // ref: SONARPY-1477
 
-  private static boolean filterTypeVar(SymbolsProtos.Type type) {
+  public static boolean filterTypeVar(SymbolsProtos.Type type) {
     return Optional.of(type)
       // Filtering self returning methods until the SONARPY-1472 will be solved
       .filter(Predicate.not(t -> t.getPrettyPrintedName().endsWith(".Self")))

sonar-python-plugin/src/main/java/org/sonar/plugins/python/SymbolVisitor.java

@@ -24,17 +24,16 @@
 import java.util.List;
 import org.sonar.api.batch.sensor.symbol.NewSymbol;
 import org.sonar.api.batch.sensor.symbol.NewSymbolTable;
+import org.sonar.plugins.python.api.symbols.Symbol;
+import org.sonar.plugins.python.api.symbols.Usage;
+import org.sonar.plugins.python.api.tree.BaseTreeVisitor;
 import org.sonar.plugins.python.api.tree.ClassDef;
 import org.sonar.plugins.python.api.tree.ComprehensionExpression;
 import org.sonar.plugins.python.api.tree.DictCompExpression;
 import org.sonar.plugins.python.api.tree.FileInput;
 import org.sonar.plugins.python.api.tree.FunctionDef;
 import org.sonar.plugins.python.api.tree.LambdaExpression;
 import org.sonar.plugins.python.api.tree.Tree;
-import org.sonar.plugins.python.api.symbols.Symbol;
-import org.sonar.plugins.python.api.symbols.Usage;
-import org.sonar.plugins.python.api.tree.BaseTreeVisitor;
-import org.sonar.python.tree.DictCompExpressionImpl;
 
 public class SymbolVisitor extends BaseTreeVisitor {