SONARPY-2010 Avoid creating duplicate types when the corresponding symbols are duplicated (#1882)

Author: guillaume-dequenne

python-checks/src/test/resources/checks/datetime_constructor_use_timezone_aware.py

@@ -36,8 +36,5 @@ def aliased_utcnow():
     # FN because we lose the FQN in the assignment
     reassigned()
 
-    from datetime.datetime import utcnow as aliased
-    aliased() # Noncompliant
-
-    from datetime.datetime import utcnow
-    utcnow() # Noncompliant
+    from datetime import datetime as aliased
+    aliased.utcnow() # Noncompliant

python-frontend/src/main/java/org/sonar/python/semantic/v2/LazyTypesContext.java

@@ -0,0 +1,51 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2011-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.python.semantic.v2;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.sonar.python.types.v2.LazyType;
+import org.sonar.python.types.v2.PythonType;
+
+public class LazyTypesContext {
+  private final Map<String, LazyType> lazyTypes;
+  private final ProjectLevelTypeTable projectLevelTypeTable;
+
+  public LazyTypesContext(ProjectLevelTypeTable projectLevelTypeTable) {
+    this.lazyTypes = new HashMap<>();
+    this.projectLevelTypeTable = projectLevelTypeTable;
+  }
+
+  public LazyType getOrCreateLazyType(String fullyQualifiedName) {
+    if (lazyTypes.containsKey(fullyQualifiedName)) {
+      return lazyTypes.get(fullyQualifiedName);
+    }
+    var lazyType = new LazyType(fullyQualifiedName, this);
+    lazyTypes.put(fullyQualifiedName, lazyType);
+    return lazyType;
+  }
+
+  public PythonType resolveLazyType(LazyType lazyType) {
+    PythonType resolved = projectLevelTypeTable.getType(lazyType.fullyQualifiedName());
+    lazyType.resolve(resolved);
+    lazyTypes.remove(lazyType.fullyQualifiedName());
+    return resolved;
+  }
+}

python-frontend/src/main/java/org/sonar/python/semantic/v2/ProjectLevelTypeTable.java

@@ -20,47 +20,30 @@
 package org.sonar.python.semantic.v2;
 
 import java.util.List;
+import java.util.Optional;
 import java.util.stream.IntStream;
 import org.sonar.python.semantic.ProjectLevelSymbolTable;
 import org.sonar.python.types.v2.ModuleType;
 import org.sonar.python.types.v2.PythonType;
-import org.sonar.python.types.v2.TriBool;
 
 public class ProjectLevelTypeTable {
 
   private final SymbolsModuleTypeProvider symbolsModuleTypeProvider;
   private final ModuleType rootModule;
+  private final LazyTypesContext lazyTypesContext;
 
   public ProjectLevelTypeTable(ProjectLevelSymbolTable projectLevelSymbolTable) {
     this(projectLevelSymbolTable, new TypeShed(projectLevelSymbolTable));
   }
 
   public ProjectLevelTypeTable(ProjectLevelSymbolTable projectLevelSymbolTable, TypeShed typeShed) {
-    this.symbolsModuleTypeProvider = new SymbolsModuleTypeProvider(projectLevelSymbolTable, typeShed);
+    this.lazyTypesContext = new LazyTypesContext(this);
+    this.symbolsModuleTypeProvider = new SymbolsModuleTypeProvider(projectLevelSymbolTable, typeShed, lazyTypesContext);
     this.rootModule = this.symbolsModuleTypeProvider.createBuiltinModule();
   }
 
-  public ModuleType getModule(String... moduleName) {
-    return getModule(List.of(moduleName));
-  }
-
-  public ModuleType getModule(List<String> moduleNameParts) {
-    var parent = rootModule;
-    for (int i = 0; i < moduleNameParts.size(); i++) {
-      var existing = parent.resolveMember(moduleNameParts.get(i)).orElse(PythonType.UNKNOWN);
-
-      if (existing instanceof ModuleType existingModule) {
-        parent = existingModule;
-        continue;
-      }
-
-      var moduleFqn = IntStream.rangeClosed(0, i)
-        .mapToObj(moduleNameParts::get)
-        .toList();
-
-      parent = symbolsModuleTypeProvider.createModuleType(moduleFqn, parent);
-    }
-    return parent;
+  public ModuleType getBuiltinsModule() {
+    return rootModule;
   }
 
   public PythonType getType(String typeFqn) {
@@ -75,17 +58,22 @@ public PythonType getType(List<String> typeFqnParts) {
     var parent = (PythonType) rootModule;
     for (int i = 0; i < typeFqnParts.size(); i++) {
       var part = typeFqnParts.get(i);
-      if (parent.hasMember(part) == TriBool.TRUE) {
-        parent = parent.resolveMember(part).orElse(PythonType.UNKNOWN);
+      Optional<PythonType> resolvedMember = parent.resolveMember(part);
+      if (resolvedMember.isPresent()) {
+        parent = resolvedMember.get();
       } else if (parent instanceof ModuleType module) {
         var moduleFqn = IntStream.rangeClosed(0, i)
           .mapToObj(typeFqnParts::get)
           .toList();
-        parent = symbolsModuleTypeProvider.createModuleType(moduleFqn, module);
+        parent = symbolsModuleTypeProvider.convertModuleType(moduleFqn, module);
       } else {
         return PythonType.UNKNOWN;
       }
     }
     return parent;
   }
+
+  public LazyTypesContext lazyTypesContext() {
+    return lazyTypesContext;
+  }
 }

python-frontend/src/main/java/org/sonar/python/semantic/v2/SymbolsModuleTypeProvider.java

@@ -19,6 +19,8 @@
  */
 package org.sonar.python.semantic.v2;
 
+import java.util.ArrayDeque;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
@@ -40,6 +42,7 @@
 import org.sonar.python.semantic.SymbolImpl;
 import org.sonar.python.types.v2.ClassType;
 import org.sonar.python.types.v2.FunctionType;
+import org.sonar.python.types.v2.LazyType;
 import org.sonar.python.types.v2.Member;
 import org.sonar.python.types.v2.ModuleType;
 import org.sonar.python.types.v2.ParameterV2;
@@ -50,23 +53,28 @@ public class SymbolsModuleTypeProvider {
   private final ProjectLevelSymbolTable projectLevelSymbolTable;
   private final TypeShed typeShed;
   private ModuleType rootModule;
+  private LazyTypesContext lazyTypesContext;
 
-  public SymbolsModuleTypeProvider(ProjectLevelSymbolTable projectLevelSymbolTable, TypeShed typeShed) {
+  public SymbolsModuleTypeProvider(ProjectLevelSymbolTable projectLevelSymbolTable, TypeShed typeShed, LazyTypesContext lazyTypeContext) {
     this.projectLevelSymbolTable = projectLevelSymbolTable;
     this.typeShed = typeShed;
+    this.lazyTypesContext = lazyTypeContext;
     this.rootModule = createModuleFromSymbols(null, null, typeShed.builtinSymbols().values());
   }
 
   public ModuleType createBuiltinModule() {
     return rootModule;
   }
 
-  public ModuleType createModuleType(List<String> moduleFqn, ModuleType parent) {
+  public PythonType convertModuleType(List<String> moduleFqn, ModuleType parent) {
     var moduleName = moduleFqn.get(moduleFqn.size() - 1);
     var moduleFqnString = getModuleFqnString(moduleFqn);
-    return createModuleTypeFromProjectLevelSymbolTable(moduleName, moduleFqnString, parent)
-      .or(() -> createModuleTypeFromTypeShed(moduleName, moduleFqnString, parent))
-      .orElseGet(() -> createEmptyModule(moduleName, parent));
+    Optional<ModuleType> result =  createModuleTypeFromProjectLevelSymbolTable(moduleName, moduleFqnString, parent)
+      .or(() -> createModuleTypeFromTypeShed(moduleName, moduleFqnString, parent));
+    if (result.isEmpty()) {
+      return PythonType.UNKNOWN;
+    }
+    return result.get();
   }
 
   private static String getModuleFqnString(List<String> moduleFqn) {
@@ -84,12 +92,6 @@ private Optional<ModuleType> createModuleTypeFromTypeShed(String moduleName, Str
       .map(typeShedModuleSymbols -> createModuleFromSymbols(moduleName, parent, typeShedModuleSymbols.values()));
   }
 
-  private static ModuleType createEmptyModule(String moduleName, ModuleType parent) {
-    var emptyModule = new ModuleType(moduleName, parent);
-    parent.members().put(moduleName, emptyModule);
-    return emptyModule;
-  }
-
   private ModuleType createModuleFromSymbols(@Nullable String name, @Nullable ModuleType parent, Collection<Symbol> symbols) {
     var members = new HashMap<String, PythonType>();
     Map<Symbol, PythonType> createdTypesBySymbol = new HashMap<>();
@@ -132,6 +134,9 @@ private PythonType convertToFunctionType(FunctionSymbol symbol, Map<Symbol, Pyth
         .withHasVariadicParameter(symbol.hasVariadicParameter())
         .withDefinitionLocation(symbol.definitionLocation());
     FunctionType functionType = functionTypeBuilder.build();
+    if (returnType instanceof LazyType lazyType) {
+      lazyType.addConsumer(functionType::resolveLazyReturnType);
+    }
     createdTypesBySymbol.put(symbol, functionType);
     return functionType;
   }
@@ -140,16 +145,41 @@ private PythonType resolveReturnType(Map<Symbol, PythonType> createdTypesBySymbo
     if (classSymbol == null) {
       return PythonType.UNKNOWN;
     }
-    if (rootModule == null) {
+    if (createdTypesBySymbol.containsKey(classSymbol)) {
+      return createdTypesBySymbol.get(classSymbol);
+    }
+    String fullyQualifiedName = classSymbol.fullyQualifiedName();
+    if (fullyQualifiedName == null) {
       return convertToType(classSymbol, createdTypesBySymbol);
     }
-    // If the symbol is a built-in symbol and the root module already exists, we search for it instead of recreating the symbol
-    // This is necessary to avoid duplicated symbols in the original ProjectSymbolTable to translate into duplicated PythonType
-    // TODO: SONARPY-2010 to fix this
-    return Optional.ofNullable(classSymbol.fullyQualifiedName())
-      .filter(fqn -> !fqn.contains("."))
-      .flatMap(f -> rootModule.resolveMember(f))
-      .orElseGet(() -> convertToType(classSymbol, createdTypesBySymbol));
+    return resolvePossibleLazyType(fullyQualifiedName);
+  }
+
+  PythonType resolvePossibleLazyType(String fullyQualifiedName) {
+    if (rootModule == null) {
+      // If root module has not been created yet, return lazy type
+      return lazyTypesContext.getOrCreateLazyType(fullyQualifiedName);
+    }
+    PythonType currentType = rootModule;
+    String[] fqnParts = fullyQualifiedName.split("\\.");
+    var fqnPartsQueue = new ArrayDeque<>(Arrays.asList(fqnParts));
+    while (!fqnPartsQueue.isEmpty()) {
+      var memberName = fqnPartsQueue.poll();
+      var memberOpt = currentType.resolveMember(memberName);
+      if (memberOpt.isEmpty()) {
+        if (currentType instanceof ModuleType) {
+          // The type is part of an unresolved submodule
+          // Create a lazy type for it
+          return lazyTypesContext.getOrCreateLazyType(fullyQualifiedName);
+        } else {
+          // The type is an unknown member of an already resolved type
+          // Default to UNKNOWN
+          return PythonType.UNKNOWN;
+        }
+      }
+      currentType = memberOpt.get();
+    }
+    return currentType;
   }
 
   private PythonType convertToClassType(ClassSymbol symbol, Map<Symbol, PythonType> createdTypesBySymbol) {

python-frontend/src/main/java/org/sonar/python/semantic/v2/types/TrivialTypeInferenceVisitor.java

@@ -107,7 +107,7 @@ public void visitFileInput(FileInput fileInput) {
 
   @Override
   public void visitStringLiteral(StringLiteral stringLiteral) {
-    ModuleType builtins = this.projectLevelTypeTable.getModule();
+    ModuleType builtins = this.projectLevelTypeTable.getBuiltinsModule();
     // TODO: SONARPY-1867 multiple object types to represent str instance?
     PythonType strType = builtins.resolveMember("str").orElse(PythonType.UNKNOWN);
     ((StringLiteralImpl) stringLiteral).typeV2(new ObjectType(strType, new ArrayList<>(), new ArrayList<>()));
@@ -121,30 +121,30 @@ public void visitTuple(Tuple tuple) {
     if (contentTypes.size() == 1 && !contentTypes.get(0).equals(PythonType.UNKNOWN)) {
       attributes = contentTypes;
     }
-    ModuleType builtins = this.projectLevelTypeTable.getModule();
+    ModuleType builtins = this.projectLevelTypeTable.getBuiltinsModule();
     PythonType tupleType = builtins.resolveMember("tuple").orElse(PythonType.UNKNOWN);
     ((TupleImpl) tuple).typeV2(new ObjectType(tupleType, attributes, new ArrayList<>()));
   }
 
   @Override
   public void visitDictionaryLiteral(DictionaryLiteral dictionaryLiteral) {
     super.visitDictionaryLiteral(dictionaryLiteral);
-    ModuleType builtins = this.projectLevelTypeTable.getModule();
+    ModuleType builtins = this.projectLevelTypeTable.getBuiltinsModule();
     PythonType dictType = builtins.resolveMember("dict").orElse(PythonType.UNKNOWN);
     ((DictionaryLiteralImpl) dictionaryLiteral).typeV2(new ObjectType(dictType, new ArrayList<>(), new ArrayList<>()));
   }
 
   @Override
   public void visitSetLiteral(SetLiteral setLiteral) {
     super.visitSetLiteral(setLiteral);
-    ModuleType builtins = this.projectLevelTypeTable.getModule();
+    ModuleType builtins = this.projectLevelTypeTable.getBuiltinsModule();
     PythonType setType = builtins.resolveMember("set").orElse(PythonType.UNKNOWN);
     ((SetLiteralImpl) setLiteral).typeV2(new ObjectType(setType, new ArrayList<>(), new ArrayList<>()));
   }
 
   @Override
   public void visitNumericLiteral(NumericLiteral numericLiteral) {
-    ModuleType builtins = this.projectLevelTypeTable.getModule();
+    ModuleType builtins = this.projectLevelTypeTable.getBuiltinsModule();
     InferredType type = numericLiteral.type();
     String memberName = ((RuntimeType) type).getTypeClass().fullyQualifiedName();
     if (memberName != null) {
@@ -155,15 +155,15 @@ public void visitNumericLiteral(NumericLiteral numericLiteral) {
 
   @Override
   public void visitNone(NoneExpression noneExpression) {
-    ModuleType builtins = this.projectLevelTypeTable.getModule();
+    ModuleType builtins = this.projectLevelTypeTable.getBuiltinsModule();
     // TODO: SONARPY-1867 multiple object types to represent str instance?
     PythonType noneType = builtins.resolveMember("NoneType").orElse(PythonType.UNKNOWN);
     ((NoneExpressionImpl) noneExpression).typeV2(new ObjectType(noneType, new ArrayList<>(), new ArrayList<>()));
   }
 
   @Override
   public void visitListLiteral(ListLiteral listLiteral) {
-    var builtins = this.projectLevelTypeTable.getModule();
+    var builtins = this.projectLevelTypeTable.getBuiltinsModule();
     scan(listLiteral.elements());
 
     var candidateTypes = listLiteral.elements()
@@ -184,7 +184,7 @@ public void visitListLiteral(ListLiteral listLiteral) {
   @Override
   public void visitPyListOrSetCompExpression(ComprehensionExpression comprehensionExpression) {
     super.visitPyListOrSetCompExpression(comprehensionExpression);
-    var builtins = this.projectLevelTypeTable.getModule();
+    var builtins = this.projectLevelTypeTable.getBuiltinsModule();
     var pythonType = switch (comprehensionExpression.getKind()) {
       case LIST_COMPREHENSION -> builtins.resolveMember("list").orElse(PythonType.UNKNOWN);
       case SET_COMPREHENSION -> builtins.resolveMember("set").orElse(PythonType.UNKNOWN);
@@ -196,7 +196,7 @@ public void visitPyListOrSetCompExpression(ComprehensionExpression comprehension
   @Override
   public void visitDictCompExpression(DictCompExpression dictCompExpression) {
     super.visitDictCompExpression(dictCompExpression);
-    var builtins = this.projectLevelTypeTable.getModule();
+    var builtins = this.projectLevelTypeTable.getBuiltinsModule();
     var dictType = builtins.resolveMember("dict").orElse(PythonType.UNKNOWN);
     ((DictCompExpressionImpl) dictCompExpression).typeV2(new ObjectType(dictType, new ArrayList<>(), new ArrayList<>()));
   }
@@ -298,8 +298,11 @@ public void visitImportName(ImportName importName) {
         var fqn = names
           .stream().map(Name::name)
           .toList();
-        var module = projectLevelTypeTable.getModule(fqn);
+        var resolvedType = projectLevelTypeTable.getType(fqn);
 
+        if (!(resolvedType instanceof ModuleType module)) {
+          return;
+        }
         if (aliasedName.alias() != null) {
           setTypeToName(aliasedName.alias(), module);
         } else {
@@ -323,7 +326,7 @@ public void visitImportFrom(ImportFrom importFrom) {
           .stream().map(Name::name)
           .toList();
 
-        var module = projectLevelTypeTable.getModule(fqn);
+        var module = projectLevelTypeTable.getType(fqn);
         importFrom.importedNames().forEach(aliasedName -> aliasedName
           .dottedName()
           .names()
@@ -412,7 +415,7 @@ public void visitName(Name name) {
     SymbolV2 symbolV2 = name.symbolV2();
     if (symbolV2 == null) {
 //    This part could be affected by SONARPY-1802
-      projectLevelTypeTable.getModule().resolveMember(name.name())
+      projectLevelTypeTable.getBuiltinsModule().resolveMember(name.name())
         .ifPresent(type -> setTypeToName(name, type));
       return;
     }

python-frontend/src/main/java/org/sonar/python/tree/CallExpressionImpl.java

@@ -191,7 +191,11 @@ public PythonType typeV2() {
       return new ObjectType(classType);
     }
     if (callee().typeV2() instanceof FunctionType functionType) {
-      return functionType.returnType();
+      PythonType returnType = functionType.returnType();
+      if (returnType.equals(PythonType.UNKNOWN)) {
+        return PythonType.UNKNOWN;
+      }
+      return new ObjectType(returnType);
     }
     if (callee().typeV2() instanceof UnionType unionType) {
       PythonType result = null;