Use protobuf stubs for third parties

Author: andreaguarino

python-frontend/src/main/java/org/sonar/python/types/TypeShed.java

@@ -58,7 +58,6 @@
 import static org.sonar.plugins.python.api.types.BuiltinTypes.NONE_TYPE;
 import static org.sonar.plugins.python.api.types.BuiltinTypes.STR;
 import static org.sonar.plugins.python.api.types.BuiltinTypes.TUPLE;
-import static org.sonar.python.types.TypeShedThirdParties.commonSymbols;
 import static org.sonar.python.types.TypeShedThirdParties.getModuleSymbols;
 
 public class TypeShed {
@@ -73,6 +72,7 @@ public class TypeShed {
   private static final String THIRD_PARTY_3 = "typeshed/third_party/3/";
   private static final String CUSTOM_THIRD_PARTY = "custom/";
   private static final String PROTOBUF = "protobuf/";
+  private static final String PROTOBUF_THIRD_PARTY = "protobuf/stubs/";
   private static final String BUILTINS_FQN = "builtins";
   private static final String BUILTINS_PREFIX = BUILTINS_FQN + ".";
   // Those fundamentals builtins symbols need not to be ambiguous for the frontend to work properly
@@ -96,7 +96,7 @@ private TypeShed() {
   public static Map<String, Symbol> builtinSymbols() {
     if ((TypeShed.builtins == null)) {
       supportedPythonVersions = ProjectPythonVersion.currentVersions().stream().map(PythonVersionUtils.Version::serializedValue).collect(Collectors.toSet());
-      Map<String, Symbol> builtins = getSymbolsFromProtobufModule(BUILTINS_FQN);
+      Map<String, Symbol> builtins = getSymbolsFromProtobufModule(BUILTINS_FQN, false);
       builtins.put(NONE_TYPE, new ClassSymbolImpl(NONE_TYPE, NONE_TYPE));
       TypeShed.builtins = Collections.unmodifiableMap(builtins);
       TypeShed.builtinGlobalSymbols.put("", new HashSet<>(builtins.values()));
@@ -271,16 +271,13 @@ private static Map<String, Symbol> searchTypeShedForModule(String moduleName) {
       modulesInProgress.remove(moduleName);
       return customSymbols;
     }
-    Map<String, Symbol> symbolsFromProtobuf = getSymbolsFromProtobufModule(moduleName);
+    Map<String, Symbol> symbolsFromProtobuf = getSymbolsFromProtobufModule(moduleName, false);
     if (!symbolsFromProtobuf.isEmpty()) {
       modulesInProgress.remove(moduleName);
       return symbolsFromProtobuf;
     }
-    Map<String, Symbol> thirdPartySymbols = getModuleSymbols(moduleName, THIRD_PARTY_2AND3, builtinGlobalSymbols);
-    if (thirdPartySymbols.isEmpty()) {
-      thirdPartySymbols = commonSymbols(getModuleSymbols(moduleName, THIRD_PARTY_2, builtinGlobalSymbols),
-        getModuleSymbols(moduleName, THIRD_PARTY_3, builtinGlobalSymbols), moduleName);
-    }
+
+    Map<String, Symbol> thirdPartySymbols = getSymbolsFromProtobufModule(moduleName, true);
     modulesInProgress.remove(moduleName);
     return thirdPartySymbols;
   }
@@ -311,8 +308,9 @@ private static boolean isAmbiguousSymbolOfClasses(Symbol symbol) {
     return false;
   }
 
-  private static Map<String, Symbol> getSymbolsFromProtobufModule(String moduleName) {
-    InputStream resource = TypeShed.class.getResourceAsStream(PROTOBUF + moduleName + ".protobuf");
+  private static Map<String, Symbol> getSymbolsFromProtobufModule(String moduleName, boolean isThirdParty) {
+    String protobufDir = isThirdParty ? PROTOBUF_THIRD_PARTY : PROTOBUF;
+    InputStream resource = TypeShed.class.getResourceAsStream(protobufDir + moduleName + ".protobuf");
     if (resource == null) {
       return Collections.emptyMap();
     }

python-frontend/src/test/java/org/sonar/python/semantic/FullyQualifiedNameTest.java

@@ -82,7 +82,7 @@ public void alias_import_preserves_fqn() {
       "from flask import redirect as flask_redirect",
       "flask_redirect()"
     );
-    assertNameAndQualifiedName(tree, "flask_redirect", "werkzeug.utils.redirect");
+    assertNameAndQualifiedName(tree, "flask_redirect", "flask.redirect");
   }
 
   @Test

python-frontend/src/test/java/org/sonar/python/types/TypeShedTest.java

@@ -287,7 +287,8 @@ public void deserialize_annoy_protobuf() {
     Map<String, Symbol> deserializedAnnoySymbols = symbolsForModule("annoy").values().stream()
       .collect(Collectors.toMap(Symbol::fullyQualifiedName, s -> s));
     assertThat(deserializedAnnoySymbols.values()).extracting(Symbol::kind, Symbol::fullyQualifiedName)
-      .containsExactlyInAnyOrder(tuple(Kind.CLASS, "annoy._Vector"), tuple(Kind.CLASS, "annoy.AnnoyIndex"));
+      .containsExactlyInAnyOrder(tuple(Kind.CLASS, "annoy._Vector"), tuple(Kind.CLASS, "annoy.AnnoyIndex"), tuple(Kind.OTHER, "annoy.Literal"),
+        tuple(Kind.CLASS, "annoy.Sized"), tuple(Kind.FUNCTION, "annoy.overload"), tuple(Kind.OTHER, "annoy.Protocol"));
 
     ClassSymbol vector = (ClassSymbol) deserializedAnnoySymbols.get("annoy._Vector");
     assertThat(vector.superClasses()).extracting(Symbol::kind, Symbol::fullyQualifiedName)
@@ -315,7 +316,8 @@ public void deserialize_annoy_protobuf() {
       tuple(Kind.FUNCTION, "get_n_items"),
       tuple(Kind.FUNCTION, "get_n_trees"),
       tuple(Kind.FUNCTION, "verbose"),
-      tuple(Kind.FUNCTION, "set_seed")
+      tuple(Kind.FUNCTION, "set_seed"),
+      tuple(Kind.OTHER, "f")
     );
     assertThat(annoyIndex.hasDecorators()).isFalse();
     assertThat(annoyIndex.definitionLocation()).isNull();

sonar-python-plugin/pom.xml

@@ -150,8 +150,8 @@
             <configuration>
               <rules>
                 <requireFilesSize>
-                  <maxsize>8300000</maxsize>
-                  <minsize>7300000</minsize>
+                  <maxsize>9300000</maxsize>
+                  <minsize>8300000</minsize>
                   <files>
                     <file>${project.build.directory}/${project.build.finalName}.jar</file>
                   </files>