SONARPY-2161 Replace v2 TypeShed with TypeShedDescriptorsProvider

Author: ghislainpiot

its/ruling/src/test/resources/expected/python-S3699.json

@@ -15,9 +15,5 @@
 ],
 "project:tensorflow/tools/pip_package/setup.py": [
 141
-],
-"project:twisted-12.1.0/twisted/trial/test/test_script.py": [
-407,
-417
 ]
 }

its/ruling/src/test/resources/expected/python-S5756.json

@@ -0,0 +1,5 @@
+{
+"project:mypy-0.782/mypy/suggestions.py": [
+65
+]
+}

its/ruling/src/test/resources/expected_extended/salt/python-S3699.json

@@ -2,9 +2,6 @@
 "salt:salt/cloud/clouds/linode.py": [
 1814
 ],
-"salt:salt/ext/tornado/test/util.py": [
-113
-],
 "salt:salt/utils/http.py": [
 453
 ]

python-checks/src/test/java/org/sonar/python/checks/quickfix/PythonQuickFixVerifier.java

@@ -39,7 +39,6 @@
 import org.sonar.python.caching.CacheContextImpl;
 import org.sonar.python.parser.PythonParser;
 import org.sonar.python.semantic.ProjectLevelSymbolTable;
-import org.sonar.python.semantic.v2.TypeShed;
 import org.sonar.python.tree.IPythonTreeMaker;
 import org.sonar.python.tree.PythonTreeMaker;
 
@@ -161,7 +160,7 @@ private static PythonVisitorContext createVisitorContext(PythonParser parser, Py
 
     return new PythonVisitorContext(fileInput,
       pythonFile, null, "",
-      ProjectLevelSymbolTable.empty(), new TypeShed(ProjectLevelSymbolTable.empty()), CacheContextImpl.dummyCache(), SonarProduct.SONARLINT);
+      ProjectLevelSymbolTable.empty(), CacheContextImpl.dummyCache(), SonarProduct.SONARLINT);
   }
 
   private static String applyQuickFix(String codeWithIssue, PythonQuickFix quickFix) {

python-checks/src/test/resources/checks/nonCallableCalled.py

@@ -381,3 +381,10 @@ def no_fp_on_decorated_classes():
     unknown_decorated_class()  # OK
     employee = Employee("John Doe", 30, "Software Engineer")
     employee() # FN
+
+
+def fp_typed_dict():
+    from typing_extensions import TypedDict
+    # TypedDict is defined as "TypedDict: object" in typing_extensions.pyi
+    # Despite actually being a function
+    x = TypedDict('x', {'a': int, 'b': str}) # Noncompliant

python-frontend/src/main/java/org/sonar/plugins/python/api/PythonVisitorContext.java

@@ -35,7 +35,6 @@
 import org.sonar.python.semantic.v2.ProjectLevelTypeTable;
 import org.sonar.python.semantic.v2.SymbolTableBuilderV2;
 import org.sonar.python.semantic.v2.TypeInferenceV2;
-import org.sonar.python.semantic.v2.TypeShed;
 
 public class PythonVisitorContext extends PythonInputFileContext {
 
@@ -51,7 +50,7 @@ public PythonVisitorContext(FileInput rootTree, PythonFile pythonFile, @Nullable
     SymbolTableBuilder symbolTableBuilder = packageName != null ? new SymbolTableBuilder(packageName, pythonFile) : new SymbolTableBuilder(pythonFile);
     symbolTableBuilder.visitFileInput(rootTree);
     var symbolTable = new SymbolTableBuilderV2(rootTree).build();
-    var projectLevelTypeTable = new ProjectLevelTypeTable(ProjectLevelSymbolTable.empty(), new TypeShed(ProjectLevelSymbolTable.empty()));
+    var projectLevelTypeTable = new ProjectLevelTypeTable(ProjectLevelSymbolTable.empty());
     new TypeInferenceV2(projectLevelTypeTable, pythonFile, symbolTable).inferTypes(rootTree);
     this.typeChecker = new TypeChecker(projectLevelTypeTable);
   }
@@ -65,20 +64,20 @@ public PythonVisitorContext(FileInput rootTree, PythonFile pythonFile, @Nullable
 
     var symbolTable = new SymbolTableBuilderV2(rootTree)
       .build();
-    var projectLevelTypeTable = new ProjectLevelTypeTable(projectLevelSymbolTable, new TypeShed(projectLevelSymbolTable));
+    var projectLevelTypeTable = new ProjectLevelTypeTable(projectLevelSymbolTable);
     new TypeInferenceV2(projectLevelTypeTable, pythonFile, symbolTable).inferTypes(rootTree);
     this.typeChecker = new TypeChecker(projectLevelTypeTable);
   }
 
   public PythonVisitorContext(FileInput rootTree, PythonFile pythonFile, @Nullable File workingDirectory, String packageName,
-    ProjectLevelSymbolTable projectLevelSymbolTable, TypeShed typeShed, CacheContext cacheContext, SonarProduct sonarProduct) {
+    ProjectLevelSymbolTable projectLevelSymbolTable, CacheContext cacheContext, SonarProduct sonarProduct) {
     super(pythonFile, workingDirectory, cacheContext, sonarProduct);
     this.rootTree = rootTree;
     this.parsingException = null;
     new SymbolTableBuilder(packageName, pythonFile, projectLevelSymbolTable).visitFileInput(rootTree);
     var symbolTable = new SymbolTableBuilderV2(rootTree)
       .build();
-    var projectLevelTypeTable = new ProjectLevelTypeTable(projectLevelSymbolTable, typeShed);
+    var projectLevelTypeTable = new ProjectLevelTypeTable(projectLevelSymbolTable);
     new TypeInferenceV2(projectLevelTypeTable, pythonFile, symbolTable).inferTypes(rootTree);
     this.typeChecker = new TypeChecker(projectLevelTypeTable);
   }

python-frontend/src/main/java/org/sonar/python/semantic/v2/ProjectLevelTypeTable.java

@@ -43,14 +43,9 @@ public class ProjectLevelTypeTable {
     ))
   );
 
-
   public ProjectLevelTypeTable(ProjectLevelSymbolTable projectLevelSymbolTable) {
-    this(projectLevelSymbolTable, new TypeShed(projectLevelSymbolTable));
-  }
-
-  public ProjectLevelTypeTable(ProjectLevelSymbolTable projectLevelSymbolTable, TypeShed typeShed) {
     this.lazyTypesContext = new LazyTypesContext(this);
-    this.symbolsModuleTypeProvider = new SymbolsModuleTypeProvider(projectLevelSymbolTable, typeShed, lazyTypesContext);
+    this.symbolsModuleTypeProvider = new SymbolsModuleTypeProvider(projectLevelSymbolTable, lazyTypesContext);
     this.rootModule = this.symbolsModuleTypeProvider.createBuiltinModule();
   }
 

python-frontend/src/main/java/org/sonar/python/semantic/v2/SymbolsModuleTypeProvider.java

@@ -28,7 +28,6 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
-import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import javax.annotation.Nullable;
 import org.sonar.plugins.python.api.symbols.AmbiguousSymbol;
@@ -38,6 +37,8 @@
 import org.sonar.python.semantic.ClassSymbolImpl;
 import org.sonar.python.semantic.FunctionSymbolImpl;
 import org.sonar.python.semantic.ProjectLevelSymbolTable;
+import org.sonar.python.semantic.v2.converter.AnyDescriptorToPythonTypeConverter;
+import org.sonar.python.semantic.v2.typeshed.TypeShedDescriptorsProvider;
 import org.sonar.python.types.InferredTypes;
 import org.sonar.python.types.protobuf.SymbolsProtos;
 import org.sonar.python.types.v2.ClassType;
@@ -56,15 +57,22 @@
 public class SymbolsModuleTypeProvider {
   public static final String OBJECT_TYPE_FQN = "object";
   private final ProjectLevelSymbolTable projectLevelSymbolTable;
-  private final TypeShed typeShed;
-  private ModuleType rootModule;
-  private LazyTypesContext lazyTypesContext;
+  private final ModuleType rootModule;
+  private final LazyTypesContext lazyTypesContext;
+  private final TypeShedDescriptorsProvider typeShedDescriptorsProvider;
+  private final AnyDescriptorToPythonTypeConverter anyDescriptorToPythonTypeConverter;
 
-  public SymbolsModuleTypeProvider(ProjectLevelSymbolTable projectLevelSymbolTable, TypeShed typeShed, LazyTypesContext lazyTypeContext) {
+  public SymbolsModuleTypeProvider(ProjectLevelSymbolTable projectLevelSymbolTable, LazyTypesContext lazyTypeContext) {
     this.projectLevelSymbolTable = projectLevelSymbolTable;
-    this.typeShed = typeShed;
     this.lazyTypesContext = lazyTypeContext;
-    this.rootModule = createModuleFromSymbols(null, null, typeShed.builtinSymbols().values());
+    this.typeShedDescriptorsProvider = new TypeShedDescriptorsProvider(projectLevelSymbolTable.projectBasePackages());
+    this.anyDescriptorToPythonTypeConverter = new AnyDescriptorToPythonTypeConverter(lazyTypesContext);
+
+    var rootModuleMembers = typeShedDescriptorsProvider.builtinDescriptors()
+      .entrySet()
+      .stream()
+      .collect(Collectors.toMap(Map.Entry::getKey, e -> anyDescriptorToPythonTypeConverter.convert(e.getValue())));
+    this.rootModule = new ModuleType(null, null, rootModuleMembers);
   }
 
   public ModuleType createBuiltinModule() {
@@ -92,9 +100,11 @@ private Optional<ModuleType> createModuleTypeFromProjectLevelSymbolTable(String
   }
 
   private Optional<ModuleType> createModuleTypeFromTypeShed(String moduleName, String moduleFqn, ModuleType parent) {
-    return Optional.ofNullable(typeShed.symbolsForModule(moduleFqn))
-      .filter(Predicate.not(Map::isEmpty))
-      .map(typeShedModuleSymbols -> createModuleFromSymbols(moduleName, parent, typeShedModuleSymbols.values()));
+    var moduleMembers = typeShedDescriptorsProvider.descriptorsForModule(moduleFqn)
+      .entrySet().stream()
+      .collect(Collectors.toMap(Map.Entry::getKey, e -> anyDescriptorToPythonTypeConverter.convert(e.getValue())));
+    return Optional.of(moduleMembers).filter(m -> !m.isEmpty())
+      .map(m -> new ModuleType(moduleName, parent, m));
   }
 
   private ModuleType createModuleFromSymbols(@Nullable String name, @Nullable ModuleType parent, Collection<Symbol> symbols) {

python-frontend/src/test/java/org/sonar/plugins/python/api/PythonVisitorContextTest.java

@@ -102,10 +102,10 @@ void sonar_product() {
     PythonFile pythonFile = pythonFile("my_module.py");
     FileInput fileInput = mock(FileInputImpl.class);
 
-    PythonVisitorContext pythonVisitorContext = new PythonVisitorContext(fileInput, pythonFile, workingDirectory, myPackage, projectLevelSymbolTable, typeShed, cacheContext, SonarProduct.SONARLINT);
+    PythonVisitorContext pythonVisitorContext = new PythonVisitorContext(fileInput, pythonFile, workingDirectory, myPackage, projectLevelSymbolTable, cacheContext, SonarProduct.SONARLINT);
     assertThat(pythonVisitorContext.sonarProduct()).isEqualTo(SonarProduct.SONARLINT);
 
-    pythonVisitorContext = new PythonVisitorContext(fileInput, pythonFile, workingDirectory, myPackage, projectLevelSymbolTable, typeShed, cacheContext, SonarProduct.SONARQUBE);
+    pythonVisitorContext = new PythonVisitorContext(fileInput, pythonFile, workingDirectory, myPackage, projectLevelSymbolTable, cacheContext, SonarProduct.SONARQUBE);
     assertThat(pythonVisitorContext.sonarProduct()).isEqualTo(SonarProduct.SONARQUBE);
 
     pythonVisitorContext = new PythonVisitorContext(fileInput, pythonFile, workingDirectory, myPackage, projectLevelSymbolTable, cacheContext);

python-frontend/src/test/java/org/sonar/python/semantic/v2/ProjectLevelTypeTableTest.java

@@ -32,7 +32,7 @@ class ProjectLevelTypeTableTest {
   @Test
   void getBuiltinTypeTest() {
     var symbolTable = ProjectLevelSymbolTable.empty();
-    var table = new ProjectLevelTypeTable(symbolTable, new TypeShed(symbolTable));
+    var table = new ProjectLevelTypeTable(symbolTable);
 
     var listClassType = table.getType("list");
     Assertions.assertThat(listClassType).isNotNull().isInstanceOf(ClassType.class);
@@ -47,7 +47,7 @@ void getBuiltinTypeTest() {
   @Test
   void getTypeshedTypeTest() {
     var symbolTable = ProjectLevelSymbolTable.empty();
-    var table = new ProjectLevelTypeTable(symbolTable, new TypeShed(symbolTable));
+    var table = new ProjectLevelTypeTable(symbolTable);
 
     var generatorClassType = table.getType("typing.Generator");
     Assertions.assertThat(generatorClassType).isNotNull().isInstanceOf(ClassType.class);
@@ -60,7 +60,7 @@ void getTypeshedTypeTest() {
   @Test
   void updateTypeTableDuringGetTypeTest() {
     var symbolTable = ProjectLevelSymbolTable.empty();
-    var table = new ProjectLevelTypeTable(symbolTable, new TypeShed(symbolTable));
+    var table = new ProjectLevelTypeTable(symbolTable);
 
     var rootModuleType = table.getType();
     Assertions.assertThat(rootModuleType.hasMember("typing")).isNotEqualTo(TriBool.TRUE);