SONARPY-972 Remove Typeshed parsing logic

Author: guillaume-dequenne

python-frontend/src/main/java/org/sonar/python/semantic/ClassSymbolImpl.java

@@ -73,11 +73,8 @@ public class ClassSymbolImpl extends SymbolImpl implements ClassSymbol {
   public ClassSymbolImpl(ClassDef classDef, @Nullable String fullyQualifiedName, PythonFile pythonFile) {
     super(classDef.name().name(), fullyQualifiedName);
     this.setKind(Kind.CLASS);
-    String fileId = null;
-    if (!SymbolUtils.isTypeShedFile(pythonFile)) {
-      Path path = pathOf(pythonFile);
-      fileId = path != null ? path.toString() : pythonFile.toString();
-    }
+    Path path = pathOf(pythonFile);
+    String fileId = path != null ? path.toString() : pythonFile.toString();
     hasDecorators = !classDef.decorators().isEmpty();
     classDefinitionLocation = locationInFile(classDef.name(), fileId);
   }

python-frontend/src/main/java/org/sonar/python/semantic/FunctionSymbolImpl.java

@@ -46,7 +46,6 @@
 import org.sonar.python.types.TypeShed;
 import org.sonar.python.types.protobuf.SymbolsProtos;
 
-import static org.sonar.python.semantic.SymbolUtils.isTypeShedFile;
 import static org.sonar.python.semantic.SymbolUtils.pathOf;
 import static org.sonar.python.tree.TreeUtils.locationInFile;
 import static org.sonar.python.types.InferredTypes.anyType;
@@ -78,12 +77,8 @@ public class FunctionSymbolImpl extends SymbolImpl implements FunctionSymbol {
     isAsynchronous = functionDef.asyncKeyword() != null;
     hasDecorators = !functionDef.decorators().isEmpty();
     decorators = decorators(functionDef);
-    String fileId = null;
-    isStub = isTypeShedFile(pythonFile);
-    if (!isStub) {
-      Path path = pathOf(pythonFile);
-      fileId = path != null ? path.toString() : pythonFile.toString();
-    }
+    isStub = false;
+    String fileId = Optional.ofNullable(pathOf(pythonFile)).map(Path::toString).orElse(pythonFile.toString());
     functionDefinitionLocation = locationInFile(functionDef.name(), fileId);
   }
 
@@ -338,10 +333,6 @@ public ParameterType(InferredType inferredType, boolean isKeywordVariadic, boole
     }
   }
 
-  public void setAnnotatedReturnTypeName(@Nullable TypeAnnotation returnTypeAnnotation) {
-    annotatedReturnTypeName = annotatedTypeName(returnTypeAnnotation);
-  }
-
   private String annotatedTypeName(@Nullable TypeAnnotation typeAnnotation) {
     return Optional.ofNullable(typeAnnotation)
       .map(TypeAnnotation::expression)

python-frontend/src/main/java/org/sonar/python/semantic/SymbolTableBuilder.java

@@ -87,7 +87,6 @@
 import org.sonar.python.types.TypeShed;
 
 import static org.sonar.python.semantic.SymbolUtils.boundNamesFromExpression;
-import static org.sonar.python.semantic.SymbolUtils.isTypeShedFile;
 import static org.sonar.python.semantic.SymbolUtils.resolveTypeHierarchy;
 
 // SymbolTable based on https://docs.python.org/3/reference/executionmodel.html#naming-and-binding
@@ -99,7 +98,6 @@ public class SymbolTableBuilder extends BaseTreeVisitor {
   private FileInput fileInput = null;
   private Set<Tree> assignmentLeftHandSides = new HashSet<>();
   private final PythonFile pythonFile;
-  private static final List<String> BASE_MODULES = Arrays.asList("", "typing", "typing_extensions");
 
   public SymbolTableBuilder(PythonFile pythonFile) {
     fullyQualifiedModuleName = null;
@@ -132,9 +130,7 @@ public void visitFileInput(FileInput fileInput) {
     createAmbiguousSymbols();
     addSymbolsToTree((FileInputImpl) fileInput);
     fileInput.accept(new ThirdPhaseVisitor());
-    if (!isTypeShedFile(pythonFile)) {
-      TypeInference.inferTypes(fileInput, pythonFile);
-    }
+    TypeInference.inferTypes(fileInput, pythonFile);
   }
 
   private static class SymbolToUpdate {
@@ -273,11 +269,9 @@ public void visitFileInput(FileInput tree) {
       createScope(tree, null);
       enterScope(tree);
       moduleScope = currentScope();
-      if (!SymbolUtils.isTypeShedFile(pythonFile) || !BASE_MODULES.contains(pythonFile.fileName())) {
-        Map<String, Symbol> typeShedSymbols = TypeShed.builtinSymbols();
-        for (String name : BuiltinSymbols.all()) {
-          currentScope().createBuiltinSymbol(name, typeShedSymbols);
-        }
+      Map<String, Symbol> typeShedSymbols = TypeShed.builtinSymbols();
+      for (String name : BuiltinSymbols.all()) {
+        currentScope().createBuiltinSymbol(name, typeShedSymbols);
       }
       super.visitFileInput(tree);
     }
@@ -684,7 +678,7 @@ public void visitFunctionDef(FunctionDef functionDef) {
           functionSymbolImpl.setParametersWithType(parameters);
         }
         TypeAnnotation typeAnnotation = functionDef.returnTypeAnnotation();
-        if (typeAnnotation != null && !isTypeShedFile(pythonFile)) {
+        if (typeAnnotation != null) {
           functionSymbolImpl.setDeclaredReturnType(InferredTypes.fromTypeAnnotation(typeAnnotation));
         }
       }

python-frontend/src/main/java/org/sonar/python/semantic/SymbolUtils.java

@@ -57,7 +57,6 @@
 import org.sonar.plugins.python.api.tree.UnpackingExpression;
 import org.sonar.python.tree.TreeUtils;
 import org.sonar.python.types.TypeShed;
-import org.sonar.python.types.TypeShedPythonFile;
 
 import static org.sonar.plugins.python.api.symbols.Symbol.Kind.CLASS;
 import static org.sonar.plugins.python.api.symbols.Symbol.Kind.FUNCTION;
@@ -90,11 +89,6 @@ static void resolveTypeHierarchy(ClassDef classDef, @Nullable Symbol symbol, Pyt
       return;
     }
     ClassSymbolImpl classSymbol = (ClassSymbolImpl) symbol;
-    if (isBuiltinTypeshedFile(pythonFile) && "str".equals(classSymbol.fullyQualifiedName())) {
-      classSymbol.addSuperClass(symbolsByName.get("object"));
-      classSymbol.addSuperClass(symbolsByName.get("Sequence"));
-      return;
-    }
     ArgList argList = classDef.args();
     if (argList == null) {
       return;
@@ -126,36 +120,10 @@ private static void addParentClass(PythonFile pythonFile, Map<String, Symbol> sy
       if ("typing.Generic".equals(argumentSymbol.fullyQualifiedName())) {
         classSymbol.setSupportsGenerics(true);
       }
-      Symbol normalizedArgumentSymbol = normalizeSymbol(argumentSymbol, pythonFile, symbolsByName);
-      if (normalizedArgumentSymbol != null) {
-        classSymbol.addSuperClass(normalizedArgumentSymbol);
-      }
+      classSymbol.addSuperClass(argumentSymbol);
     }
   }
 
-  /**
-   * Hardcoding some 'typing' module symbols to avoid incomplete type hierarchy for type 'str'
-   */
-  @CheckForNull
-  private static Symbol normalizeSymbol(Symbol symbol, PythonFile pythonFile, Map<String, Symbol> symbolsByName) {
-    if (isTypeShedFile(pythonFile) && (symbol.name().equals("Protocol") || symbol.name().equals("Generic"))) {
-      // ignore Protocol and Generic to avoid having incomplete type hierarchies
-      return null;
-    }
-    if (isTypingFile(pythonFile) && symbol.name().equals("_Collection")) {
-      return symbolsByName.get("Collection");
-    }
-    return symbol;
-  }
-
-  private static boolean isBuiltinTypeshedFile(PythonFile pythonFile) {
-    return isTypeShedFile(pythonFile) && pythonFile.fileName().isEmpty();
-  }
-
-  private static boolean isTypingFile(PythonFile pythonFile) {
-    return isTypeShedFile(pythonFile) && pythonFile.fileName().equals("typing");
-  }
-
   @CheckForNull
   private static Symbol getSymbolFromArgument(RegularArgument regularArgument) {
     Expression expression = regularArgument.expression();
@@ -222,10 +190,6 @@ public static Path pathOf(PythonFile pythonFile) {
     }
   }
 
-  public static boolean isTypeShedFile(PythonFile pythonFile) {
-    return pythonFile instanceof TypeShedPythonFile;
-  }
-
   /**
    * @return the offset between parameter position and argument position:
    *   0 if there is no implicit first parameter (self, cls, etc...)

python-frontend/src/main/java/org/sonar/python/types/TypeShed.java

@@ -66,10 +66,6 @@ public class TypeShed {
   private static final Map<String, Set<Symbol>> builtinGlobalSymbols = new HashMap<>();
   private static final Set<String> modulesInProgress = new HashSet<>();
 
-  private static final String THIRD_PARTY_2AND3 = "typeshed/third_party/2and3/";
-  private static final String THIRD_PARTY_2 = "typeshed/third_party/2/";
-  private static final String THIRD_PARTY_3 = "typeshed/third_party/3/";
-  private static final String CUSTOM_THIRD_PARTY = "custom/";
   private static final String PROTOBUF_CUSTOM_STUBS = "custom_protobuf/";
   private static final String PROTOBUF = "stdlib_protobuf/";
   private static final String PROTOBUF_THIRD_PARTY = "third_party_protobuf/";