SONARPY-1869 Provide basic support for generics in type inference V2

guillaume-dequenne · guillaume-dequenne · commit 79f3f29350a6 · 2024-11-21T17:56:56.000+01:00
diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/ClassTypeBuilder.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/ClassTypeBuilder.java
@@ -40,11 +40,12 @@ public class ClassTypeBuilder implements TypeBuilder<ClassType> {
   List<TypeWrapper> superClasses = new ArrayList<>();
   List<PythonType> metaClasses = new ArrayList<>();
   boolean hasDecorators = false;
+  boolean isGeneric = false;
   LocationInFile definitionLocation;
 
   @Override
   public ClassType build() {
-    return new ClassType(name, fullyQualifiedName, members, attributes, superClasses, metaClasses, hasDecorators, definitionLocation);
+    return new ClassType(name, fullyQualifiedName, members, attributes, superClasses, metaClasses, hasDecorators, isGeneric, definitionLocation);
   }
 
   public ClassTypeBuilder(String name, String fullyQualifiedName) {
@@ -57,6 +58,11 @@ public ClassTypeBuilder withHasDecorators(boolean hasDecorators) {
     return this;
   }
 
+  public ClassTypeBuilder withIsGeneric(boolean isGeneric) {
+    this.isGeneric = isGeneric;
+    return this;
+  }
+
   @Override
   public ClassTypeBuilder withDefinitionLocation(@Nullable LocationInFile definitionLocation) {
     this.definitionLocation = definitionLocation;
diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/ClassDescriptorToPythonTypeConverter.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/ClassDescriptorToPythonTypeConverter.java
@@ -30,6 +30,7 @@ public class ClassDescriptorToPythonTypeConverter implements DescriptorToPythonT
 
   private static PythonType convert(ConversionContext ctx, ClassDescriptor from) {
     var typeBuilder = new ClassTypeBuilder(from.name(), from.fullyQualifiedName())
+      .withIsGeneric(from.supportsGenerics())
       .withDefinitionLocation(from.definitionLocation());
 
     from.superClasses().stream()
diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java
@@ -152,7 +152,7 @@ private static Descriptor convert(String moduleFqn, String parentFqn, String sym
       hasSuperClassWithoutDescriptor,
       type.hasMetaClass(),
       metaclassFQN,
-      false
+      type.isGeneric()
     );
   }
 
diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/VariableDescriptorToPythonTypeConverter.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/VariableDescriptorToPythonTypeConverter.java
@@ -19,25 +19,30 @@
  */
 package org.sonar.python.semantic.v2.converter;
 
-import java.util.Optional;
 import org.sonar.python.index.Descriptor;
 import org.sonar.python.index.VariableDescriptor;
 import org.sonar.python.types.v2.ObjectType;
 import org.sonar.python.types.v2.PythonType;
+import org.sonar.python.types.v2.SpecialFormType;
+import org.sonar.python.types.v2.TypeWrapper;
 
 public class VariableDescriptorToPythonTypeConverter implements DescriptorToPythonTypeConverter {
 
   public PythonType convert(ConversionContext ctx, VariableDescriptor from) {
-    if (from.isImportedModule()) {
-      var fqn = from.fullyQualifiedName();
-      if (fqn != null) {
-        return ctx.lazyTypesContext().getOrCreateLazyType(fqn);
+    String fullyQualifiedName = from.fullyQualifiedName();
+    if (from.isImportedModule() && fullyQualifiedName != null) {
+      return ctx.lazyTypesContext().getOrCreateLazyType(fullyQualifiedName);
+    }
+    String annotatedType = from.annotatedType();
+    if (annotatedType != null) {
+      if ("typing._SpecialForm".equals(annotatedType) && fullyQualifiedName != null) {
+        // Defensive null check on fullyQualifiedName: it should never be null for SpecialForm
+        return new SpecialFormType(fullyQualifiedName);
       }
+      TypeWrapper typeWrapper = ctx.lazyTypesContext().getOrCreateLazyTypeWrapper(annotatedType);
+      return new ObjectType(typeWrapper);
     }
-    return Optional.ofNullable(from.annotatedType())
-      .map(fqn -> ctx.lazyTypesContext().getOrCreateLazyTypeWrapper(fqn))
-      .map(t -> (PythonType) new ObjectType(t))
-      .orElse(PythonType.UNKNOWN);
+    return PythonType.UNKNOWN;
   }
 
   @Override
diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/types/TrivialTypeInferenceVisitor.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/types/TrivialTypeInferenceVisitor.java
@@ -81,6 +81,7 @@
 import org.sonar.python.tree.NumericLiteralImpl;
 import org.sonar.python.tree.SetLiteralImpl;
 import org.sonar.python.tree.StringLiteralImpl;
+import org.sonar.python.tree.SubscriptionExpressionImpl;
 import org.sonar.python.tree.TupleImpl;
 import org.sonar.python.tree.UnaryExpressionImpl;
 import org.sonar.python.types.v2.ClassType;
@@ -89,8 +90,10 @@
 import org.sonar.python.types.v2.ModuleType;
 import org.sonar.python.types.v2.ObjectType;
 import org.sonar.python.types.v2.PythonType;
+import org.sonar.python.types.v2.SpecialFormType;
 import org.sonar.python.types.v2.TriBool;
 import org.sonar.python.types.v2.TypeCheckBuilder;
+import org.sonar.python.types.v2.TypeChecker;
 import org.sonar.python.types.v2.TypeOrigin;
 import org.sonar.python.types.v2.TypeSource;
 import org.sonar.python.types.v2.TypeWrapper;
@@ -109,6 +112,7 @@ public class TrivialTypeInferenceVisitor extends BaseTreeVisitor {
 
   private final Deque<Scope> typeStack = new ArrayDeque<>();
   private final Set<String> importedModulesFQN = new HashSet<>();
+  private final TypeChecker typeChecker;
 
   private final Map<String, TypeWrapper> wildcardImportedTypes = new HashMap<>();
 
@@ -120,6 +124,7 @@ public TrivialTypeInferenceVisitor(TypeTable projectLevelTypeTable, PythonFile p
     this.moduleName = pythonFile.fileName();
     this.fileId = path != null ? path.toString() : pythonFile.toString();
     this.fullyQualifiedModuleName = fullyQualifiedModuleName;
+    this.typeChecker = new TypeChecker(projectLevelTypeTable);
   }
 
   public Set<String> importedModulesFQN() {
@@ -273,6 +278,9 @@ private ClassType buildClassType(ClassDef classDef) {
       .withHasDecorators(!classDef.decorators().isEmpty())
       .withDefinitionLocation(locationInFile(className, fileId));
     resolveTypeHierarchy(classDef, classTypeBuilder);
+    if (classDef.typeParams() != null) {
+      classTypeBuilder.withIsGeneric(true);
+    }
     ClassType classType = classTypeBuilder.build();
 
     if (currentType() instanceof ClassType ownerClass) {
@@ -285,22 +293,22 @@ private ClassType buildClassType(ClassDef classDef) {
     return classType;
   }
 
-  static void resolveTypeHierarchy(ClassDef classDef, ClassTypeBuilder classTypeBuilder) {
+  void resolveTypeHierarchy(ClassDef classDef, ClassTypeBuilder classTypeBuilder) {
     Optional.of(classDef)
       .map(ClassDef::args)
       .map(ArgList::arguments)
       .stream()
       .flatMap(Collection::stream)
       .forEach(argument -> {
         if (argument instanceof RegularArgument regularArgument) {
-          addParentClass(classTypeBuilder, regularArgument);
+          this.addParentClass(classTypeBuilder, regularArgument);
         } else {
           classTypeBuilder.addSuperClass(PythonType.UNKNOWN);
         }
       });
   }
 
-  private static void addParentClass(ClassTypeBuilder classTypeBuilder, RegularArgument regularArgument) {
+  private void addParentClass(ClassTypeBuilder classTypeBuilder, RegularArgument regularArgument) {
     Name keyword = regularArgument.keywordArgument();
     // TODO: SONARPY-1871 store names if not resolved properly
     if (keyword != null) {
@@ -312,7 +320,10 @@ private static void addParentClass(ClassTypeBuilder classTypeBuilder, RegularArg
     }
     PythonType argumentType = getTypeV2FromArgument(regularArgument);
     classTypeBuilder.addSuperClass(argumentType);
-    // TODO: SONARPY-1869 handle generics
+    if (typeChecker.typeCheckBuilder().isGeneric().check(argumentType) == TriBool.TRUE && regularArgument.expression().is(Tree.Kind.SUBSCRIPTION)) {
+      // SONARPY-2356: checking that we have a subscription only is too naive (e.g. specialized classes)
+      classTypeBuilder.withIsGeneric(true);
+    }
   }
 
   private static PythonType getTypeV2FromArgument(RegularArgument regularArgument) {
@@ -561,6 +572,14 @@ public void visitQualifiedExpression(QualifiedExpression qualifiedExpression) {
     }
   }
 
+  @Override
+  public void visitSubscriptionExpression(SubscriptionExpression subscriptionExpression) {
+    super.visitSubscriptionExpression(subscriptionExpression);
+    PythonType pythonType = subscriptionExpression.object().typeV2();
+    if (typeChecker.typeCheckBuilder().isGeneric().check(pythonType) == TriBool.TRUE) {
+      ((SubscriptionExpressionImpl) subscriptionExpression).typeV2(pythonType);
+    }
+  }
 
   @Override
   public void visitName(Name name) {
@@ -611,7 +630,8 @@ private static boolean shouldTypeBeEagerlyPropagated(PythonType t) {
     return (t instanceof ClassType)
            || (t instanceof FunctionType)
            || (t instanceof ModuleType)
-           || (t instanceof UnknownType.UnresolvedImportType);
+           || (t instanceof UnknownType.UnresolvedImportType)
+           || (t instanceof SpecialFormType);
   }
 
   private PythonType currentType() {
diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/typeshed/ClassSymbolToDescriptorConverter.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/typeshed/ClassSymbolToDescriptorConverter.java
@@ -72,6 +72,7 @@ public ClassDescriptor convert(SymbolsProtos.ClassSymbol classSymbol) {
       .withMetaclassFQN(metaclassName)
       .withHasMetaClass(classSymbol.getHasMetaclass())
       .withHasDecorators(classSymbol.getHasDecorators())
+      .withSupportsGenerics(classSymbol.getIsGeneric())
       .withMembers(members)
       .build();
   }
diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/typeshed/TypeShedUtils.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/typeshed/TypeShedUtils.java
@@ -55,13 +55,6 @@ private static String getTypesFqn(SymbolsProtos.Type type) {
     switch (type.getKind()) {
       case INSTANCE:
         String typeName = type.getFullyQualifiedName();
-        // _SpecialForm is the type used for some special types, like Callable, Union, TypeVar, ...
-        // It comes from CPython impl: https://github.com/python/cpython/blob/e39ae6bef2c357a88e232dcab2e4b4c0f367544b/Lib/typing.py#L439
-        // This doesn't seem to be very precisely specified in typeshed, because it has special semantic.
-        // To avoid FPs, we treat it as ANY
-        if ("typing._SpecialForm".equals(typeName)) {
-          return null;
-        }
         return typeName.isEmpty() ? null : typeName;
       case TYPE_ALIAS:
         return getTypesFqn(type.getArgs(0));
diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/typeshed/VarSymbolToDescriptorConverter.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/typeshed/VarSymbolToDescriptorConverter.java
@@ -19,6 +19,7 @@
  */
 package org.sonar.python.semantic.v2.typeshed;
 
+import javax.annotation.Nullable;
 import org.sonar.python.index.Descriptor;
 import org.sonar.python.index.VariableDescriptor;
 import org.sonar.python.types.protobuf.SymbolsProtos;
@@ -27,15 +28,16 @@ public class VarSymbolToDescriptorConverter {
 
   public Descriptor convert(SymbolsProtos.VarSymbol varSymbol) {
     var fullyQualifiedName = TypeShedUtils.normalizedFqn(varSymbol.getFullyQualifiedName());
-    var typeAnnotation = TypeShedUtils.getTypesNormalizedFqn(varSymbol.getTypeAnnotation());
+    SymbolsProtos.Type protoTypeAnnotation = varSymbol.getTypeAnnotation();
     var isImportedModule = varSymbol.getIsImportedModule();
+    var typeAnnotation = TypeShedUtils.getTypesNormalizedFqn(protoTypeAnnotation);
     if (isTypeAnnotationKnownToBeIncorrect(fullyQualifiedName)) {
       return new VariableDescriptor(varSymbol.getName(), fullyQualifiedName, null, isImportedModule);
     }
     return new VariableDescriptor(varSymbol.getName(), fullyQualifiedName, typeAnnotation, isImportedModule);
   }
 
-  private static boolean isTypeAnnotationKnownToBeIncorrect(String fullyQualifiedName) {
+  private static boolean isTypeAnnotationKnownToBeIncorrect(@Nullable String fullyQualifiedName) {
     // TypedDict is defined to have type "object" in Typeshed, which is incorrect and leads to FPs
     return "typing.TypedDict".equals(fullyQualifiedName) || "typing_extensions.TypedDict".equals(fullyQualifiedName);
   }
diff --git a/python-frontend/src/main/java/org/sonar/python/tree/SubscriptionExpressionImpl.java b/python-frontend/src/main/java/org/sonar/python/tree/SubscriptionExpressionImpl.java
@@ -28,13 +28,15 @@
 import org.sonar.plugins.python.api.tree.Token;
 import org.sonar.plugins.python.api.tree.Tree;
 import org.sonar.plugins.python.api.tree.TreeVisitor;
+import org.sonar.python.types.v2.PythonType;
 
 public class SubscriptionExpressionImpl extends PyTree implements SubscriptionExpression {
 
   private final Expression object;
   private final Token lBracket;
   private final ExpressionList subscripts;
   private final Token rBracket;
+  private PythonType pythonType = PythonType.UNKNOWN;
 
   public SubscriptionExpressionImpl(Expression object, Token lBracket, ExpressionList subscripts, Token rBracket) {
     this.object = object;
@@ -63,6 +65,16 @@ public Token rightBracket() {
     return rBracket;
   }
 
+  @Override
+  public PythonType typeV2() {
+    return pythonType;
+  }
+
+  public SubscriptionExpression typeV2(PythonType pythonType) {
+    this.pythonType = pythonType;
+    return this;
+  }
+
   @Override
   public void accept(TreeVisitor visitor) {
     visitor.visitSubscriptionExpression(this);
diff --git a/python-frontend/src/main/java/org/sonar/python/types/v2/ClassType.java b/python-frontend/src/main/java/org/sonar/python/types/v2/ClassType.java
@@ -44,6 +44,7 @@ public final class ClassType implements PythonType {
   private final List<TypeWrapper> superClasses;
   private final List<PythonType> metaClasses;
   private final boolean hasDecorators;
+  private final boolean isGeneric;
   private final LocationInFile locationInFile;
 
   public ClassType(
@@ -54,6 +55,7 @@ public ClassType(
     List<TypeWrapper> superClasses,
     List<PythonType> metaClasses,
     boolean hasDecorators,
+    boolean isGeneric,
     @Nullable LocationInFile locationInFile) {
     this.name = name;
     this.fullyQualifiedName = fullyQualifiedName;
@@ -62,11 +64,12 @@ public ClassType(
     this.superClasses = superClasses;
     this.metaClasses = metaClasses;
     this.hasDecorators = hasDecorators;
+    this.isGeneric = isGeneric;
     this.locationInFile = locationInFile;
   }
 
   public ClassType(String name, String fullyQualifiedName) {
-    this(name, fullyQualifiedName, new HashSet<>(), new ArrayList<>(), new ArrayList<>(), new ArrayList<>(), false, null);
+    this(name, fullyQualifiedName, new HashSet<>(), new ArrayList<>(), new ArrayList<>(), new ArrayList<>(), false, false, null);
   }
 
   @Override
@@ -232,4 +235,8 @@ public List<PythonType> metaClasses() {
   public boolean hasDecorators() {
     return hasDecorators;
   }
+
+  public boolean isGeneric() {
+    return isGeneric;
+  }
 }
diff --git a/python-frontend/src/main/java/org/sonar/python/types/v2/SpecialFormType.java b/python-frontend/src/main/java/org/sonar/python/types/v2/SpecialFormType.java
@@ -0,0 +1,33 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2011-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.python.types.v2;
+
+public class SpecialFormType implements PythonType {
+
+  private final String fullyQualifiedName;
+
+  public SpecialFormType(String fullyQualifiedName) {
+    this.fullyQualifiedName = fullyQualifiedName;
+  }
+
+  public String fullyQualifiedName() {
+    return fullyQualifiedName;
+  }
+}
diff --git a/python-frontend/src/main/java/org/sonar/python/types/v2/TypeCheckBuilder.java b/python-frontend/src/main/java/org/sonar/python/types/v2/TypeCheckBuilder.java
@@ -66,6 +66,11 @@ public TypeCheckBuilder isInstance() {
     return this;
   }
 
+  public TypeCheckBuilder isGeneric() {
+    predicates.add(new IsGenericPredicate());
+    return this;
+  }
+
   public TriBool check(PythonType pythonType) {
     TriBool result = TriBool.TRUE;
     for (TypePredicate predicate : predicates) {
@@ -269,6 +274,27 @@ public TriBool test(PythonType pythonType) {
     }
   }
 
+  record IsGenericPredicate() implements TypePredicate {
+
+    @Override
+    public TriBool test(PythonType pythonType) {
+      return isGeneric(pythonType);
+    }
+
+    private static TriBool isGeneric(PythonType pythonType) {
+      if (pythonType instanceof ClassType classType && classType.isGeneric()) {
+        return TriBool.TRUE;
+      }
+      if (pythonType instanceof SpecialFormType specialFormType) {
+        return "typing.Generic".equals(specialFormType.fullyQualifiedName()) ? TriBool.TRUE : TriBool.UNKNOWN;
+      }
+      if (pythonType instanceof UnknownType.UnresolvedImportType unresolvedImportType && "typing.Generic".equals(unresolvedImportType.importPath())) {
+        return TriBool.TRUE;
+      }
+      return TriBool.UNKNOWN;
+    }
+  }
+
   private static boolean containsUnknown(Set<PythonType> types) {
     return types.stream().anyMatch(UnknownType.class::isInstance);
   }
diff --git a/python-frontend/src/test/java/org/sonar/python/index/ClassDescriptorTest.java b/python-frontend/src/test/java/org/sonar/python/index/ClassDescriptorTest.java
@@ -109,8 +109,7 @@ void classDescriptorGenerics() {
     ClassDescriptor classDescriptor = lastClassDescriptor(
       "from typing import Generic",
       "class A(Generic[str]): ...");
-    // SONARPY-2313: supportsGenerics should be true here
-    assertThat(classDescriptor.supportsGenerics()).isFalse();
+    assertThat(classDescriptor.supportsGenerics()).isTrue();
     assertDescriptorToProtobuf(classDescriptor);
   }
 
diff --git a/python-frontend/src/test/java/org/sonar/python/semantic/v2/TypeInferenceV2Test.java b/python-frontend/src/test/java/org/sonar/python/semantic/v2/TypeInferenceV2Test.java
diff --git a/python-frontend/src/test/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverterTest.java b/python-frontend/src/test/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverterTest.java
diff --git a/python-frontend/src/test/java/org/sonar/python/semantic/v2/typeshed/TypeShedUtilsTest.java b/python-frontend/src/test/java/org/sonar/python/semantic/v2/typeshed/TypeShedUtilsTest.java
diff --git a/python-frontend/src/test/java/org/sonar/python/types/TypeInferenceTest.java b/python-frontend/src/test/java/org/sonar/python/types/TypeInferenceTest.java
diff --git a/python-frontend/src/test/java/org/sonar/python/types/v2/TypeCheckerTest.java b/python-frontend/src/test/java/org/sonar/python/types/v2/TypeCheckerTest.java
diff --git a/python-frontend/src/test/java/org/sonar/python/types/v2/UnionTypeTest.java b/python-frontend/src/test/java/org/sonar/python/types/v2/UnionTypeTest.java

Original file line number	Diff line number	Diff line change
`@@ -152,7 +152,7 @@ private static Descriptor convert(String moduleFqn, String parentFqn, String sym`
`152`	`152`	`hasSuperClassWithoutDescriptor,`
`153`	`153`	`type.hasMetaClass(),`
`154`	`154`	`metaclassFQN,`
`155`		`- false`
	`155`	`+ type.isGeneric()`
`156`	`156`	`);`
`157`	`157`	`}`
`158`	`158`
Original file line number	Diff line number	Diff line change
`@@ -72,6 +72,7 @@ public ClassDescriptor convert(SymbolsProtos.ClassSymbol classSymbol) {`
`72`	`72`	`.withMetaclassFQN(metaclassName)`
`73`	`73`	`.withHasMetaClass(classSymbol.getHasMetaclass())`
`74`	`74`	`.withHasDecorators(classSymbol.getHasDecorators())`
	`75`	`+ .withSupportsGenerics(classSymbol.getIsGeneric())`
`75`	`76`	`.withMembers(members)`
`76`	`77`	`.build();`
`77`	`78`	`}`