SONARPY-869 Deserialize protobuf files containing symbol information … (#938)

Author: andrea-guarino-sonarsource

pom.xml

@@ -94,6 +94,7 @@
     <sonar-analyzer-commons.version>1.15.0.699</sonar-analyzer-commons.version>
     <sonarlint-core.version>6.0.0.32513</sonarlint-core.version>
     <sslr.version>1.23</sslr.version>
+    <protobuf.version>3.17.3</protobuf.version>
   </properties>
 
   <dependencyManagement>

python-frontend/pom.xml

@@ -37,6 +37,68 @@
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java</artifactId>
+      <version>${protobuf.version}</version>
+    </dependency>
   </dependencies>
-
+  <build>
+    <extensions>
+      <extension>
+        <groupId>kr.motd.maven</groupId>
+        <artifactId>os-maven-plugin</artifactId>
+        <version>1.6.2</version>
+      </extension>
+    </extensions>
+    <plugins>
+      <plugin>
+        <groupId>org.xolstice.maven.plugins</groupId>
+        <artifactId>protobuf-maven-plugin</artifactId>
+        <version>0.6.1</version>
+        <executions>
+          <execution>
+            <id>generate-protobuf-java-sources</id>
+            <goals>
+              <goal>compile</goal>
+            </goals>
+            <configuration>
+              <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
+              </protocArtifact>
+              <protoSourceRoot>${project.basedir}/src/main/protobuf</protoSourceRoot>
+              <outputDirectory>${project.build.directory}/generated-sources/protobuf</outputDirectory>
+            </configuration>
+          </execution>
+        </executions>
+        <dependencies>
+          <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protoc</artifactId>
+            <version>${protobuf.version}</version>
+            <type>exe</type>
+            <classifier>${os.detected.classifier}</classifier>
+          </dependency>
+        </dependencies>
+      </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>build-helper-maven-plugin</artifactId>
+        <version>3.2.0</version>
+        <executions>
+          <execution>
+            <id>add-protobuf-generated-sources</id>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>add-source</goal>
+            </goals>
+            <configuration>
+              <sources>
+                <source>${project.build.directory}/generated-sources/protobuf</source>
+              </sources>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
 </project>

python-frontend/src/main/java/org/sonar/python/semantic/ClassSymbolImpl.java

@@ -40,13 +40,16 @@
 import org.sonar.plugins.python.api.symbols.ClassSymbol;
 import org.sonar.plugins.python.api.symbols.Symbol;
 import org.sonar.plugins.python.api.tree.ClassDef;
+import org.sonar.python.types.TypeShed;
+import org.sonar.python.types.protobuf.SymbolsProtos;
 
 import static org.sonar.python.semantic.SymbolUtils.pathOf;
 import static org.sonar.python.tree.TreeUtils.locationInFile;
 
 public class ClassSymbolImpl extends SymbolImpl implements ClassSymbol {
 
   private final List<Symbol> superClasses = new ArrayList<>();
+  private List<String> superClassesFqns = new ArrayList<>();
   private Set<Symbol> allSuperClasses = null;
   private Set<Symbol> allSuperClassesIncludingAmbiguousSymbols = null;
   private boolean hasSuperClassWithoutSymbol = false;
@@ -88,6 +91,23 @@ public ClassSymbolImpl(String name, @Nullable String fullyQualifiedName, @Nullab
     setKind(Kind.CLASS);
   }
 
+  public ClassSymbolImpl(SymbolsProtos.ClassSymbol classSymbolProto) {
+    super(classSymbolProto.getName(), classSymbolProto.getFullyQualifiedName());
+    setKind(Kind.CLASS);
+    classDefinitionLocation = null;
+    hasDecorators = classSymbolProto.getHasDecorators();
+    hasMetaClass = classSymbolProto.getHasMetaclass();
+    metaclassFQN = classSymbolProto.getMetaclassName();
+    supportsGenerics = classSymbolProto.getIsGeneric();
+    List<SymbolsProtos.FunctionSymbol> methodsList = classSymbolProto.getMethodsList();
+    Set<Symbol> methods = methodsList.stream().map(m -> new FunctionSymbolImpl(m, true)).collect(Collectors.toSet());
+    for (SymbolsProtos.OverloadedFunctionSymbol overloadedMethod : classSymbolProto.getOverloadedMethodsList()) {
+      methods.add(AmbiguousSymbolImpl.create(overloadedMethod.getDefinitionsList().stream().map(m -> new FunctionSymbolImpl(m, true)).collect(Collectors.toSet())));
+    }
+    addMembers(methods);
+    superClassesFqns = classSymbolProto.getSuperClassesList();
+  }
+
   @Override
   ClassSymbolImpl copyWithoutUsages() {
     ClassSymbolImpl copiedClassSymbol = new ClassSymbolImpl(name(), fullyQualifiedName(), definitionLocation(), hasDecorators, hasMetaClass, metaclassFQN, supportsGenerics);
@@ -111,10 +131,21 @@ ClassSymbolImpl copyWithoutUsages() {
 
   @Override
   public List<Symbol> superClasses() {
+    // In case of symbols coming from TypeShed protobuf, we resolve superclasses lazily
+    if (!hasAlreadyReadSuperClasses && superClasses.isEmpty() && !superClassesFqns.isEmpty()) {
+      superClassesFqns.stream().map(ClassSymbolImpl::resolveSuperClass).forEach(this::addSuperClass);
+    }
     hasAlreadyReadSuperClasses = true;
     return Collections.unmodifiableList(superClasses);
   }
 
+  private static Symbol resolveSuperClass(String superClassFqn) {
+    String[] fqnSplitByDot = superClassFqn.split("\\.");
+    String localName = fqnSplitByDot[fqnSplitByDot.length - 1];
+    Symbol symbol = TypeShed.symbolWithFQN(superClassFqn);
+    return symbol == null ? new SymbolImpl(localName, superClassFqn) : symbol;
+  }
+
   public void addSuperClass(Symbol symbol) {
     if (hasAlreadyReadSuperClasses) {
       throw new IllegalStateException("Cannot call addSuperClass, super classes were already read");

python-frontend/src/main/java/org/sonar/python/semantic/FunctionSymbolImpl.java

@@ -41,6 +41,7 @@
 import org.sonar.plugins.python.api.types.InferredType;
 import org.sonar.python.tree.TreeUtils;
 import org.sonar.python.types.InferredTypes;
+import org.sonar.python.types.protobuf.SymbolsProtos;
 
 import static org.sonar.python.semantic.SymbolUtils.isTypeShedFile;
 import static org.sonar.python.semantic.SymbolUtils.pathOf;
@@ -80,6 +81,36 @@ public class FunctionSymbolImpl extends SymbolImpl implements FunctionSymbol {
     functionDefinitionLocation = locationInFile(functionDef.name(), fileId);
   }
 
+  public FunctionSymbolImpl(SymbolsProtos.FunctionSymbol functionSymbolProto) {
+    this(functionSymbolProto, false);
+  }
+
+  public FunctionSymbolImpl(SymbolsProtos.FunctionSymbol functionSymbolProto, boolean insideClass) {
+    super(functionSymbolProto.getName(), functionSymbolProto.getFullyQualifiedName());
+    setKind(Kind.FUNCTION);
+    isInstanceMethod = insideClass && !functionSymbolProto.getIsStatic() && !functionSymbolProto.getIsClassMethod();
+    isAsynchronous = functionSymbolProto.getIsCoroutine() || functionSymbolProto.getIsAsyncGenerator();
+    hasDecorators = functionSymbolProto.getHasDecorators();
+    decorators = functionSymbolProto.getResolvedDecoratorNamesList();
+    SymbolsProtos.Type returnAnnotation = functionSymbolProto.getReturnAnnotation();
+    String returnTypeName = returnAnnotation.getSimpleName();
+    annotatedReturnTypeName = returnTypeName.isEmpty() ? null : returnTypeName;
+    for (SymbolsProtos.ParameterSymbol parameterSymbol : functionSymbolProto.getParametersList()) {
+      ParameterState parameterState = new ParameterState();
+      parameterState.positionalOnly = parameterSymbol.getKind() == SymbolsProtos.ParameterKind.POSITIONAL_ONLY;
+      parameterState.keywordOnly = parameterSymbol.getKind() == SymbolsProtos.ParameterKind.KEYWORD_ONLY;
+      boolean isVariadic = (parameterSymbol.getKind() == SymbolsProtos.ParameterKind.VAR_KEYWORD) || parameterSymbol.getKind() == SymbolsProtos.ParameterKind.VAR_POSITIONAL;
+      hasVariadicParameter |= isVariadic;
+      ParameterImpl parameter = new ParameterImpl(
+        parameterSymbol.getName(), InferredTypes.fromTypeshedProtobuf(parameterSymbol.getTypeAnnotation()), parameterSymbol.getHasDefault(), isVariadic, parameterState, null);
+      parameters.add(parameter);
+    }
+    functionDefinitionLocation = null;
+    declaredReturnType = InferredTypes.fromTypeshedProtobuf(returnAnnotation);
+    isStub = true;
+    isDjangoView = false;
+  }
+
   public void setParametersWithType(ParameterList parametersList) {
     this.parameters.clear();
     createParameterNames(parametersList.all(), functionDefinitionLocation == null ? null : functionDefinitionLocation.fileId());

python-frontend/src/main/java/org/sonar/python/types/InferredTypes.java

@@ -45,6 +45,7 @@
 import org.sonar.plugins.python.api.types.BuiltinTypes;
 import org.sonar.plugins.python.api.types.InferredType;
 import org.sonar.python.tree.TreeUtils;
+import org.sonar.python.types.protobuf.SymbolsProtos;
 
 import static org.sonar.plugins.python.api.symbols.Symbol.Kind.CLASS;
 
@@ -154,6 +155,27 @@ public static InferredType fromTypeshedTypeAnnotation(TypeAnnotation typeAnnotat
     return runtimeTypefromTypeAnnotation(typeAnnotation.expression(), builtins);
   }
 
+  public static InferredType fromTypeshedProtobuf(SymbolsProtos.Type type) {
+    switch (type.getKind()) {
+      case INSTANCE:
+        String typeName = type.getSimpleName();
+        return typeName.isEmpty() ? anyType() : runtimeType(TypeShed.symbolWithFQN(typeName));
+      case TYPE_ALIAS:
+      case CALLABLE:
+        return fromTypeshedProtobuf(type.getArgs(0));
+      case UNION:
+        return union(type.getArgsList().stream().map(InferredTypes::fromTypeshedProtobuf));
+      case TUPLE:
+        return TUPLE;
+      case NONE:
+        return NONE;
+      case TYPED_DICT:
+        return DICT;
+      default:
+        return anyType();
+    }
+  }
+
   @CheckForNull
   private static DeclaredType declaredTypeFromTypeAnnotation(Expression expression, Map<String, Symbol> builtinSymbols) {
     if (expression.is(Kind.NAME) && !((Name) expression).name().equals("Any")) {

python-frontend/src/main/java/org/sonar/python/types/TypeShed.java

@@ -20,6 +20,7 @@
 package org.sonar.python.types;
 
 import com.sonar.sslr.api.AstNode;
+import java.io.IOException;
 import java.io.InputStream;
 import java.util.Arrays;
 import java.util.Collection;
@@ -33,7 +34,10 @@
 import java.util.stream.Collectors;
 import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
 import org.sonar.plugins.python.api.PythonFile;
+import org.sonar.plugins.python.api.symbols.AmbiguousSymbol;
 import org.sonar.plugins.python.api.symbols.ClassSymbol;
 import org.sonar.plugins.python.api.symbols.FunctionSymbol;
 import org.sonar.plugins.python.api.symbols.Symbol;
@@ -54,6 +58,8 @@
 import org.sonar.python.semantic.SymbolTableBuilder;
 import org.sonar.python.tree.FunctionDefImpl;
 import org.sonar.python.tree.PythonTreeMaker;
+import org.sonar.python.types.protobuf.SymbolsProtos.ModuleSymbol;
+import org.sonar.python.types.protobuf.SymbolsProtos.OverloadedFunctionSymbol;
 
 import static org.sonar.plugins.python.api.types.BuiltinTypes.NONE_TYPE;
 
@@ -62,7 +68,7 @@ public class TypeShed {
   private static final String TYPING = "typing";
   private static final String TYPING_EXTENSIONS = "typing_extensions";
   private static Map<String, Symbol> builtins;
-  private static final Map<String, Set<Symbol>> typeShedSymbols = new HashMap<>();
+  private static final Map<String, Map<String, Symbol>> typeShedSymbols = new HashMap<>();
   private static final Map<String, Set<Symbol>> builtinGlobalSymbols = new HashMap<>();
   private static final Set<String> modulesInProgress = new HashSet<>();
 
@@ -73,6 +79,9 @@ public class TypeShed {
   private static final String THIRD_PARTY_2 = "typeshed/third_party/2/";
   private static final String THIRD_PARTY_3 = "typeshed/third_party/3/";
   private static final String CUSTOM_THIRD_PARTY = "custom/";
+  private static final String PROTOBUF = "protobuf/";
+
+  private static final Logger LOG = Loggers.get(TypeShed.class);
 
   private TypeShed() {
   }
@@ -161,10 +170,11 @@ static Set<Symbol> typingExtensionsSymbols(Map<String, Set<Symbol>> typingSymbol
   public static Set<Symbol> symbolsForModule(String moduleName) {
     if (!TypeShed.typeShedSymbols.containsKey(moduleName)) {
       Set<Symbol> symbols = searchTypeShedForModule(moduleName);
-      typeShedSymbols.put(moduleName, symbols);
+      Map<String, Symbol> symbolsByFqn = symbols.stream().collect(Collectors.toMap(Symbol::fullyQualifiedName, s -> s));
+      typeShedSymbols.put(moduleName, symbolsByFqn);
       return symbols;
     }
-    return TypeShed.typeShedSymbols.get(moduleName);
+    return new HashSet<>(TypeShed.typeShedSymbols.get(moduleName).values());
   }
 
   @CheckForNull
@@ -195,6 +205,11 @@ private static Set<Symbol> searchTypeShedForModule(String moduleName) {
       return new HashSet<>();
     }
     modulesInProgress.add(moduleName);
+    Collection<Symbol> symbolsFromProtobuf = getSymbolsFromProtobufModule(moduleName);
+    if (!symbolsFromProtobuf.isEmpty()) {
+      modulesInProgress.remove(moduleName);
+      return new HashSet<>(symbolsFromProtobuf);
+    }
     Set<Symbol> customSymbols = new HashSet<>(getModuleSymbols(moduleName, CUSTOM_THIRD_PARTY, builtinGlobalSymbols).values());
     if (!customSymbols.isEmpty()) {
       modulesInProgress.remove(moduleName);
@@ -268,7 +283,7 @@ public static ClassSymbol typeShedClass(String fullyQualifiedName) {
 
   public static Collection<Symbol> stubFilesSymbols() {
     Set<Symbol> symbols = new HashSet<>(TypeShed.builtinSymbols().values());
-    typeShedSymbols.values().forEach(symbols::addAll);
+    typeShedSymbols.values().forEach(symbolsByFqn -> symbols.addAll(symbolsByFqn.values()));
     return symbols;
   }
 
@@ -340,4 +355,57 @@ private static class ModuleDescription {
     }
   }
 
+
+  private static Collection<Symbol> getSymbolsFromProtobufModule(String moduleName) {
+    InputStream resource = TypeShed.class.getResourceAsStream(PROTOBUF + moduleName + ".protobuf");
+    if (resource == null) {
+      return Collections.emptySet();
+    }
+    return getSymbolsFromProtobufModule(deserializedModule(moduleName, resource)).values();
+  }
+
+  @CheckForNull
+  static ModuleSymbol deserializedModule(String moduleName, InputStream resource) {
+    try {
+      return ModuleSymbol.parseFrom(resource);
+    } catch (IOException e) {
+      LOG.debug("Error while deserializing protobuf for module " + moduleName, e);
+      return null;
+    }
+  }
+
+  static Map<String, Symbol> getSymbolsFromProtobufModule(@Nullable ModuleSymbol moduleSymbol) {
+    if (moduleSymbol == null) {
+      return Collections.emptyMap();
+    }
+    Map<String, Symbol> deserializedSymbols = new HashMap<>();
+    moduleSymbol.getClassesList().forEach(proto -> deserializedSymbols.put(proto.getFullyQualifiedName(), new ClassSymbolImpl(proto)));
+    moduleSymbol.getFunctionsList().forEach(proto -> deserializedSymbols.put(proto.getFullyQualifiedName(), new FunctionSymbolImpl(proto)));
+    moduleSymbol.getOverloadedFunctionsList().forEach(proto -> deserializedSymbols.put(proto.getFullname(), fromOverloadedFunction(proto)));
+    return deserializedSymbols;
+  }
+
+  private static AmbiguousSymbol fromOverloadedFunction(OverloadedFunctionSymbol overloadedFunctionSymbol) {
+    Set<Symbol> overloadedSymbols = overloadedFunctionSymbol.getDefinitionsList().stream()
+      .map(FunctionSymbolImpl::new)
+      .collect(Collectors.toSet());
+    return AmbiguousSymbolImpl.create(overloadedSymbols);
+  }
+
+  @CheckForNull
+  public static Symbol symbolWithFQN(String fullyQualifiedName) {
+    String[] fqnSplittedByDot = fullyQualifiedName.split("\\.");
+    String localName = fqnSplittedByDot[fqnSplittedByDot.length - 1];
+    if (fqnSplittedByDot.length == 2 && fqnSplittedByDot[0].equals("builtins") && builtins.containsKey(localName)) {
+      return builtins.get(localName);
+    }
+    for (Map<String, Symbol> symbolsByFqn : typeShedSymbols.values()) {
+      Symbol symbol = symbolsByFqn.get(fullyQualifiedName);
+      if (symbol != null) {
+        return symbol;
+      }
+    }
+    return null;
+  }
+
 }