SONARPY-2170 Support detailed return types in Descriptor model (#2028)

Author: guillaume-dequenne

python-checks/src/test/resources/checks/nonCallableCalled.py

@@ -388,3 +388,14 @@ def fp_typed_dict():
     # TypedDict is defined as "TypedDict: object" in typing_extensions.pyi
     # Despite actually being a function
     x = TypedDict('x', {'a': int, 'b': str}) # Noncompliant
+
+
+def function_type_is_callable():
+    import unittest
+    # unittest.skip() returns a Callable
+    unittest.skip("reason")() # OK
+
+
+def object_typevar():
+    scheduled = []
+    scheduled.pop()()  # OK

python-checks/src/test/resources/checks/use_of_empty_return_value.py

@@ -67,3 +67,13 @@ def import_in_different_branch():
         import fcntl
     def lock():
         ret = fcntl.flock(..., ...)  # Noncompliant
+
+
+
+def smth():
+    import sys
+    options = trial.Options()
+    options.parseOptions(["--coverage"])
+    self.addCleanup(sys.settrace, sys.gettrace())
+    self.assertEqual(sys.gettrace(), options.tracer.globaltrace)
+

python-frontend/src/main/java/org/sonar/python/index/FunctionDescriptor.java

@@ -39,9 +39,18 @@ public class FunctionDescriptor implements Descriptor {
   private final LocationInFile definitionLocation;
   @Nullable
   private final String annotatedReturnTypeName;
+  @Nullable
+  private final TypeAnnotationDescriptor typeAnnotationDescriptor;
+
 
   public FunctionDescriptor(String name, @Nullable String fullyQualifiedName, List<Parameter> parameters, boolean isAsynchronous,
     boolean isInstanceMethod, List<String> decorators, boolean hasDecorators, @Nullable LocationInFile definitionLocation, @Nullable String annotatedReturnTypeName) {
+    this(name, fullyQualifiedName, parameters, isAsynchronous, isInstanceMethod, decorators, hasDecorators, definitionLocation, annotatedReturnTypeName, null);
+  }
+
+  public FunctionDescriptor(String name, @Nullable String fullyQualifiedName, List<Parameter> parameters, boolean isAsynchronous,
+    boolean isInstanceMethod, List<String> decorators, boolean hasDecorators, @Nullable LocationInFile definitionLocation,
+    @Nullable String annotatedReturnTypeName, @Nullable TypeAnnotationDescriptor typeAnnotationDescriptor) {
 
     this.name = name;
     this.fullyQualifiedName = fullyQualifiedName;
@@ -52,6 +61,7 @@ public FunctionDescriptor(String name, @Nullable String fullyQualifiedName, List
     this.hasDecorators = hasDecorators;
     this.definitionLocation = definitionLocation;
     this.annotatedReturnTypeName = annotatedReturnTypeName;
+    this.typeAnnotationDescriptor = typeAnnotationDescriptor;
   }
 
   @Override
@@ -99,6 +109,11 @@ public String annotatedReturnTypeName() {
     return annotatedReturnTypeName;
   }
 
+  @CheckForNull
+  public TypeAnnotationDescriptor typeAnnotationDescriptor() {
+    return typeAnnotationDescriptor;
+  }
+
   public static class Parameter  {
 
     private final String name;
@@ -172,6 +187,7 @@ public static class FunctionDescriptorBuilder {
     private boolean hasDecorators = false;
     private LocationInFile definitionLocation = null;
     private String annotatedReturnTypeName = null;
+    private TypeAnnotationDescriptor typeAnnotationDescriptor = null;
 
     public FunctionDescriptorBuilder withName(String name) {
       this.name = name;
@@ -218,9 +234,14 @@ public FunctionDescriptorBuilder withAnnotatedReturnTypeName(@Nullable String an
       return this;
     }
 
+    public FunctionDescriptorBuilder withTypeAnnotationDescriptor(@Nullable TypeAnnotationDescriptor typeAnnotationDescriptor) {
+      this.typeAnnotationDescriptor = typeAnnotationDescriptor;
+      return this;
+    }
+
     public FunctionDescriptor build() {
       return new FunctionDescriptor(name, fullyQualifiedName, parameters, isAsynchronous, isInstanceMethod, decorators,
-        hasDecorators, definitionLocation, annotatedReturnTypeName);
+        hasDecorators, definitionLocation, annotatedReturnTypeName, typeAnnotationDescriptor);
     }
   }
 }

python-frontend/src/main/java/org/sonar/python/index/TypeAnnotationDescriptor.java

@@ -0,0 +1,72 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2011-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.python.index;
+
+import java.util.List;
+import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
+
+public class TypeAnnotationDescriptor {
+
+  String prettyPrintedName;
+  TypeKind kind;
+  List<TypeAnnotationDescriptor> args;
+  String fullyQualifiedName;
+
+  public TypeAnnotationDescriptor(String prettyPrintedName, TypeKind kind, List<TypeAnnotationDescriptor> args, @Nullable String fullyQualifiedName) {
+    this.prettyPrintedName = prettyPrintedName;
+    this.kind = kind;
+    this.args = args;
+    this.fullyQualifiedName = fullyQualifiedName;
+  }
+
+  public String prettyPrintedName() {
+    return prettyPrintedName;
+  }
+
+  public TypeKind kind() {
+    return kind;
+  }
+
+  public List<TypeAnnotationDescriptor> args() {
+    return args;
+  }
+
+  @CheckForNull
+  public String fullyQualifiedName() {
+    return fullyQualifiedName;
+  }
+
+  public enum TypeKind {
+    INSTANCE,
+    UNION,
+    TYPE,
+    TUPLE,
+    TYPE_VAR,
+    ANY,
+    NONE,
+    TYPE_ALIAS,
+    CALLABLE,
+    LITERAL,
+    UNINHABITED,
+    UNBOUND,
+    TYPED_DICT
+  }
+}

python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/FunctionDescriptorToPythonTypeConverter.java

@@ -32,9 +32,11 @@
 public class FunctionDescriptorToPythonTypeConverter implements DescriptorToPythonTypeConverter {
 
   private final ParameterConverter parameterConverter;
+  private final TypeAnnotationToPythonTypeConverter typeAnnotationConverter;
 
   public FunctionDescriptorToPythonTypeConverter() {
     parameterConverter = new ParameterConverter();
+    typeAnnotationConverter = new TypeAnnotationToPythonTypeConverter();
   }
 
   public PythonType convert(ConversionContext ctx, FunctionDescriptor from) {
@@ -43,8 +45,8 @@ public PythonType convert(ConversionContext ctx, FunctionDescriptor from) {
       .map(parameter -> parameterConverter.convert(ctx, parameter))
       .toList();
 
-    var returnType = Optional.ofNullable(from.annotatedReturnTypeName())
-      .map(fqn -> (PythonType) ctx.lazyTypesContext().getOrCreateLazyType(fqn))
+    PythonType returnType = Optional.ofNullable(from.typeAnnotationDescriptor())
+      .map(typeAnnotation -> typeAnnotationConverter.convert(ctx, typeAnnotation))
       .map(TypeUtils::ensureWrappedObjectType)
       .orElse(PythonType.UNKNOWN);
 

python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/TypeAnnotationToPythonTypeConverter.java

@@ -0,0 +1,91 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2011-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.python.semantic.v2.converter;
+
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import org.sonar.python.index.TypeAnnotationDescriptor;
+import org.sonar.python.types.v2.LazyUnionType;
+import org.sonar.python.types.v2.PythonType;
+
+public class TypeAnnotationToPythonTypeConverter {
+
+  public PythonType convert(ConversionContext context, TypeAnnotationDescriptor type) {
+    switch (type.kind()) {
+      case INSTANCE:
+        String fullyQualifiedName = type.fullyQualifiedName();
+        if (fullyQualifiedName == null) {
+          return PythonType.UNKNOWN;
+        }
+        // _SpecialForm is the type used for some special types, like Callable, Union, TypeVar, ...
+        // It comes from CPython impl: https://github.com/python/cpython/blob/e39ae6bef2c357a88e232dcab2e4b4c0f367544b/Lib/typing.py#L439
+        // This doesn't seem to be very precisely specified in typeshed, because it has special semantic.
+        // To avoid FPs, we treat it as ANY
+        if ("typing._SpecialForm".equals(fullyQualifiedName)) {
+          return PythonType.UNKNOWN;
+        }
+        return fullyQualifiedName.isEmpty() ? PythonType.UNKNOWN : context.lazyTypesContext().getOrCreateLazyType(fullyQualifiedName);
+      case TYPE:
+        return context.lazyTypesContext().getOrCreateLazyType("type");
+      case TYPE_ALIAS:
+        return convert(context, type.args().get(0));
+      case CALLABLE:
+        // this should be handled as a function type - see SONARPY-953
+        return context.lazyTypesContext().getOrCreateLazyType("function");
+      case UNION:
+        return new LazyUnionType(type.args().stream().map(t -> convert(context, t)).collect(Collectors.toSet()));
+      case TUPLE:
+        return context.lazyTypesContext().getOrCreateLazyType("tuple");
+      case NONE:
+        return context.lazyTypesContext().getOrCreateLazyType("NoneType");
+      case TYPED_DICT:
+        // SONARPY-2179: This case only makes sense for parameter types, which are not supported yet
+        return context.lazyTypesContext().getOrCreateLazyType("dict");
+      case TYPE_VAR:
+        return Optional.of(type)
+          .filter(TypeAnnotationToPythonTypeConverter::filterTypeVar)
+          .map(TypeAnnotationDescriptor::fullyQualifiedName)
+          .map(context.lazyTypesContext()::getOrCreateLazyType)
+          .map(PythonType.class::cast)
+          .orElse(PythonType.UNKNOWN);
+      default:
+        return PythonType.UNKNOWN;
+    }
+  }
+
+  private static final Set<String> EXCLUDING_TYPE_VAR_FQN_PATTERNS = Set.of(
+    "object",
+    "^builtins\\.object$",
+    // ref: SONARPY-1477
+    "^_ctypes\\._CanCastTo$");
+
+  public static boolean filterTypeVar(TypeAnnotationDescriptor type) {
+    return Optional.of(type)
+      // Filtering self returning methods until the SONARPY-1472 will be solved
+      .filter(Predicate.not(t -> t.prettyPrintedName().endsWith(".Self")))
+      .map(TypeAnnotationDescriptor::fullyQualifiedName)
+      .filter(Predicate.not(String::isEmpty))
+      // We ignore TypeVar referencing "builtins.object" or "object" to avoid false positives
+      .filter(fqn -> EXCLUDING_TYPE_VAR_FQN_PATTERNS.stream().noneMatch(fqn::matches))
+      .isPresent();
+  }
+}

python-frontend/src/main/java/org/sonar/python/semantic/v2/typeshed/FunctionSymbolToDescriptorConverter.java

@@ -22,14 +22,17 @@
 import java.util.Collection;
 import java.util.Optional;
 import org.sonar.python.index.FunctionDescriptor;
+import org.sonar.python.index.TypeAnnotationDescriptor;
 import org.sonar.python.types.protobuf.SymbolsProtos;
 
 public class FunctionSymbolToDescriptorConverter {
 
   private final ParameterSymbolToDescriptorConverter parameterConverter;
+  private final TypeSymbolToDescriptorConverter typeConverter;
 
   public FunctionSymbolToDescriptorConverter() {
     parameterConverter = new ParameterSymbolToDescriptorConverter();
+    typeConverter = new TypeSymbolToDescriptorConverter();
   }
 
   public FunctionDescriptor convert(SymbolsProtos.FunctionSymbol functionSymbol) {
@@ -38,7 +41,12 @@ public FunctionDescriptor convert(SymbolsProtos.FunctionSymbol functionSymbol) {
 
   public FunctionDescriptor convert(SymbolsProtos.FunctionSymbol functionSymbol, boolean isParentIsAClass) {
     var fullyQualifiedName = TypeShedUtils.normalizedFqn(functionSymbol.getFullyQualifiedName());
-    var returnType = TypeShedUtils.getTypesNormalizedFqn(functionSymbol.getReturnAnnotation());
+    TypeAnnotationDescriptor typeAnnotationDescriptor = null;
+    if (functionSymbol.hasReturnAnnotation()) {
+      SymbolsProtos.Type returnAnnotation = functionSymbol.getReturnAnnotation();
+      typeAnnotationDescriptor = typeConverter.convert(returnAnnotation);
+    }
+    String returnType = TypeShedUtils.getTypesNormalizedFqn(functionSymbol.getReturnAnnotation());
     var decorators = Optional.of(functionSymbol)
       .map(SymbolsProtos.FunctionSymbol::getResolvedDecoratorNamesList)
       .stream()
@@ -56,9 +64,9 @@ public FunctionDescriptor convert(SymbolsProtos.FunctionSymbol functionSymbol, b
       .withIsInstanceMethod(isInstanceMethod)
       .withHasDecorators(functionSymbol.getHasDecorators())
       .withAnnotatedReturnTypeName(returnType)
+      .withTypeAnnotationDescriptor(typeAnnotationDescriptor)
       .withDecorators(decorators)
       .withParameters(parameters)
       .build();
   }
-
 }

python-frontend/src/main/java/org/sonar/python/semantic/v2/typeshed/TypeSymbolToDescriptorConverter.java

@@ -0,0 +1,40 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2011-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.python.semantic.v2.typeshed;
+
+import java.util.List;
+import org.sonar.python.index.TypeAnnotationDescriptor;
+import org.sonar.python.types.protobuf.SymbolsProtos;
+
+public class TypeSymbolToDescriptorConverter {
+
+  TypeAnnotationDescriptor convert(SymbolsProtos.Type type) {
+    List<TypeAnnotationDescriptor> args = type.getArgsList().stream()
+      .map(this::convert)
+      .toList();
+    TypeAnnotationDescriptor.TypeKind kind = TypeAnnotationDescriptor.TypeKind.valueOf(type.getKind().name());
+    String normalizedFqn = TypeShedUtils.normalizedFqn(type.getFullyQualifiedName());
+    return new TypeAnnotationDescriptor(
+      type.getPrettyPrintedName(),
+      kind,
+      args,
+      normalizedFqn);
+  }
+}

python-frontend/src/main/java/org/sonar/python/types/v2/ClassType.java

@@ -188,6 +188,10 @@ public TriBool instancesHaveMember(String memberName) {
       // TODO: instances of NamedTuple are type
       return TriBool.TRUE;
     }
+    if ("function".equals(this.name) && "__call__".equals(memberName)) {
+      // __call__ is not formally defined on function, but is present
+      return TriBool.TRUE;
+    }
     return resolveMember(memberName).isPresent() ? TriBool.TRUE : TriBool.FALSE;
   }
 

python-frontend/src/main/java/org/sonar/python/types/v2/LazyType.java

@@ -26,7 +26,7 @@
 import org.sonar.plugins.python.api.LocationInFile;
 import org.sonar.python.semantic.v2.LazyTypesContext;
 
-public class LazyType implements PythonType {
+public class LazyType implements PythonType, ResolvableType {
 
   String fullyQualifiedName;
   private final Queue<Consumer<PythonType>> consumers;