SONARPY-868 Serialize "annoy" library using mypy and protobuf (#939)

Author: guillaume-dequenne

.gitignore

@@ -31,3 +31,7 @@ Desktop.ini
 # ---- Sonar
 .sonar
 .scannerwork
+
+# ---- Python
+venv
+__pycache__

.gitmodules

@@ -4,3 +4,7 @@
 [submodule "python-frontend/src/main/resources/org/sonar/python/types/typeshed"]
 	path = python-frontend/src/main/resources/org/sonar/python/types/typeshed
 	url = https://github.com/python/typeshed.git
+# TODO: Remove duplicated Typeshed once we now longer rely on parsing it ourselves
+[submodule "tools/typeshed_serializer/resources/typeshed"]
+	path = python-frontend/typeshed_serializer/resources/typeshed
+	url = https://github.com/python/typeshed.git

python-frontend/pom.xml

@@ -69,6 +69,18 @@
               <outputDirectory>${project.build.directory}/generated-sources/protobuf</outputDirectory>
             </configuration>
           </execution>
+          <execution>
+            <id>generate-protobuf-python-sources</id>
+            <goals>
+              <goal>compile-python</goal>
+            </goals>
+            <configuration>
+              <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
+              </protocArtifact>
+              <protoSourceRoot>${project.basedir}/src/main/protobuf</protoSourceRoot>
+              <outputDirectory>${project.basedir}/typeshed_serializer/serializer/proto_out</outputDirectory>
+            </configuration>
+          </execution>
         </executions>
         <dependencies>
           <dependency>
@@ -101,4 +113,52 @@
       </plugin>
     </plugins>
   </build>
+
+  <profiles>
+    <profile>
+      <id>
+        typeshed_serializer
+      </id>
+      <activation>
+        <property>
+          <name>
+            generateTypeshed
+          </name>
+        </property>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>exec-maven-plugin</artifactId>
+            <version>1.6.0</version>
+            <configuration>
+              <workingDirectory>typeshed_serializer</workingDirectory>
+            </configuration>
+            <executions>
+              <execution>
+                <id>tox</id>
+                <phase>generate-resources</phase>
+                <configuration>
+                  <executable>tox</executable>
+                </configuration>
+                <goals>
+                  <goal>exec</goal>
+                </goals>
+              </execution>
+              <execution>
+                <id>tox -e serialize</id>
+                <phase>generate-resources</phase>
+                <goals><goal>exec</goal></goals>
+                <configuration>
+                  <executable>tox</executable>
+                  <arguments>-e serialize</arguments>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
 </project>

python-frontend/src/main/java/org/sonar/python/semantic/FunctionSymbolImpl.java

@@ -89,11 +89,11 @@ public FunctionSymbolImpl(SymbolsProtos.FunctionSymbol functionSymbolProto, bool
     super(functionSymbolProto.getName(), functionSymbolProto.getFullyQualifiedName());
     setKind(Kind.FUNCTION);
     isInstanceMethod = insideClass && !functionSymbolProto.getIsStatic() && !functionSymbolProto.getIsClassMethod();
-    isAsynchronous = functionSymbolProto.getIsCoroutine() || functionSymbolProto.getIsAsyncGenerator();
+    isAsynchronous = functionSymbolProto.getIsAsynchronous();
     hasDecorators = functionSymbolProto.getHasDecorators();
     decorators = functionSymbolProto.getResolvedDecoratorNamesList();
     SymbolsProtos.Type returnAnnotation = functionSymbolProto.getReturnAnnotation();
-    String returnTypeName = returnAnnotation.getSimpleName();
+    String returnTypeName = returnAnnotation.getFullyQualifiedName();
     annotatedReturnTypeName = returnTypeName.isEmpty() ? null : returnTypeName;
     for (SymbolsProtos.ParameterSymbol parameterSymbol : functionSymbolProto.getParametersList()) {
       ParameterState parameterState = new ParameterState();

python-frontend/src/main/java/org/sonar/python/types/InferredTypes.java

@@ -158,7 +158,7 @@ public static InferredType fromTypeshedTypeAnnotation(TypeAnnotation typeAnnotat
   public static InferredType fromTypeshedProtobuf(SymbolsProtos.Type type) {
     switch (type.getKind()) {
       case INSTANCE:
-        String typeName = type.getSimpleName();
+        String typeName = type.getFullyQualifiedName();
         return typeName.isEmpty() ? anyType() : runtimeType(TypeShed.symbolWithFQN(typeName));
       case TYPE_ALIAS:
       case CALLABLE:

python-frontend/src/main/protobuf/symbols.proto

@@ -34,7 +34,7 @@ message Type {
   string pretty_printed_name = 1;
   TypeKind kind = 2;
   repeated Type args = 3;
-  optional string simple_name = 4;
+  optional string fully_qualified_name = 4;
 }
 
 message ParameterSymbol {
@@ -52,23 +52,18 @@ message FunctionSymbol {
   bool has_decorators = 6;
   repeated string resolved_decorator_names = 7;
   bool is_abstract = 8;
-  bool is_async_generator = 9;
-  bool is_awaitable_coroutine = 10;
-  bool is_coroutine = 11;
-  bool is_final = 12;
-  bool is_overload = 13;
-  bool is_property = 14;
-  bool is_static = 15;
-  bool is_class_method = 16;
+  bool is_asynchronous = 9;
+  bool is_final = 10;
+  bool is_overload = 11;
+  bool is_property = 12;
+  bool is_static = 13;
+  bool is_class_method = 14;
 }
 
 message OverloadedFunctionSymbol {
   string name = 1;
   string fullname = 2;
-  bool is_final = 3;
-  bool is_property = 4;
-  bool is_static = 5;
-  repeated FunctionSymbol definitions = 6;
+  repeated FunctionSymbol definitions = 3;
 }
 
 message ClassSymbol {
@@ -82,10 +77,8 @@ message ClassSymbol {
   bool has_metaclass = 9;
   bool is_enum = 10;
   bool is_generic = 11;
-  bool is_named_tuple = 12;
-  bool is_protocol = 13;
-  optional string metaclass_name = 14;
-  optional Type metaclass_type = 15;
+  bool is_protocol = 12;
+  optional string metaclass_name = 13;
 }
 
 message ModuleSymbol {

python-frontend/src/main/resources/org/sonar/python/types/protobuf/annoy.protobuf

@@ -6,7 +6,7 @@
 self
 annoy._Vector"annoy._Vector*!
 
i
-builtins.int"builtins.inth
�/
+builtins.int"builtins.int`
�/
 
 AnnoyIndexannoy.AnnoyIndex"builtins.object2�
 __init__annoy.AnnoyIndex.__init__"
@@ -114,8 +114,8 @@ tUnion[Literal[builtins.str],Literal[builtins.str],Literal[builtins.str],Literal
 self
$
 annoy.AnnoyIndex"annoy.AnnoyIndex*#
 __s
-builtins.int"builtins.int:�
-get_nns_by_item annoy.AnnoyIndex.get_nns_by_item2�
+builtins.int"builtins.int:�
+get_nns_by_item annoy.AnnoyIndex.get_nns_by_item�
 get_nns_by_item annoy.AnnoyIndex.get_nns_by_item"J
 builtins.list[builtins.int]
 builtins.int"builtins.int"builtins.list*.
@@ -129,7 +129,7 @@ tUnion[Literal[builtins.str],Literal[builtins.str],Literal[builtins.str],Literal
 builtins.int"builtins.int 
*S
 include_distances
:
 Literal[builtins.bool]	
-builtins.bool"builtins.bool 
0
:typing.overloadh
2�
+builtins.bool"builtins.bool 
0
:overloadX
�
 get_nns_by_item annoy.AnnoyIndex.get_nns_by_item"�

 @Tuple[builtins.list[builtins.int],builtins.list[builtins.float]]J
 builtins.list[builtins.int]
@@ -146,7 +146,7 @@ tUnion[Literal[builtins.str],Literal[builtins.str],Literal[builtins.str],Literal
 builtins.int"builtins.int*Q
 include_distances
:
 Literal[builtins.bool]	
-builtins.bool"builtins.bool0
:typing.overloadh
2�
+builtins.bool"builtins.bool0
:overloadX
�
 get_nns_by_item annoy.AnnoyIndex.get_nns_by_item"�

 @Tuple[builtins.list[builtins.int],builtins.list[builtins.float]]J
 builtins.list[builtins.int]
@@ -163,8 +163,8 @@ tUnion[Literal[builtins.str],Literal[builtins.str],Literal[builtins.str],Literal
 builtins.int"builtins.int 
*Q
 include_distances:
 Literal[builtins.bool]	
-builtins.bool"builtins.bool0
:typing.overloadh
:�
-get_nns_by_vector"annoy.AnnoyIndex.get_nns_by_vector2�
+builtins.bool"builtins.bool0
:overloadX
:�
+get_nns_by_vector"annoy.AnnoyIndex.get_nns_by_vector�
 get_nns_by_vector"annoy.AnnoyIndex.get_nns_by_vector"J
 builtins.list[builtins.int]
 builtins.int"builtins.int"builtins.list*.
@@ -178,7 +178,7 @@ tUnion[Literal[builtins.str],Literal[builtins.str],Literal[builtins.str],Literal
 builtins.int"builtins.int 
*S
 include_distances
:
 Literal[builtins.bool]	
-builtins.bool"builtins.bool 
0
:typing.overloadh
2�
+builtins.bool"builtins.bool 
0
:overloadX
�
 get_nns_by_vector"annoy.AnnoyIndex.get_nns_by_vector"�

 @Tuple[builtins.list[builtins.int],builtins.list[builtins.float]]J
 builtins.list[builtins.int]
@@ -195,7 +195,7 @@ tUnion[Literal[builtins.str],Literal[builtins.str],Literal[builtins.str],Literal
 builtins.int"builtins.int*Q
 include_distances
:
 Literal[builtins.bool]	
-builtins.bool"builtins.bool0
:typing.overloadh
2�
+builtins.bool"builtins.bool0
:overloadX
�
 get_nns_by_vector"annoy.AnnoyIndex.get_nns_by_vector"�

 @Tuple[builtins.list[builtins.int],builtins.list[builtins.float]]J
 builtins.list[builtins.int]
@@ -212,4 +212,4 @@ tUnion[Literal[builtins.str],Literal[builtins.str],Literal[builtins.str],Literal
 builtins.int"builtins.int 
*Q
 include_distances:
 Literal[builtins.bool]	
-builtins.bool"builtins.bool0
:typing.overloadh

+builtins.bool"builtins.bool0
:overloadX

python-frontend/src/test/java/org/sonar/python/semantic/ClassSymbolImplTest.java

@@ -237,11 +237,7 @@ public void from_protobuf() throws TextFormat.ParseException {
       "super_classes: \"builtins.object\"\n" +
       "has_decorators: true\n" +
       "has_metaclass: true\n" +
-      "metaclass_name: \"abc.ABCMeta\"\n" +
-      "metaclass_type {\n" +
-      "  pretty_printed_name: \"abc.ABCMeta\"\n" +
-      "  simple_name: \"abc.ABCMeta\"\n" +
-      "}";
+      "metaclass_name: \"abc.ABCMeta\"";
     ClassSymbolImpl classSymbol = new ClassSymbolImpl(classSymbol(protobuf));
     assertThat(classSymbol.name()).isEqualTo("A");
     assertThat(classSymbol.fullyQualifiedName()).isEqualTo("mod.A");

python-frontend/src/test/java/org/sonar/python/semantic/FunctionSymbolTest.java

@@ -269,7 +269,7 @@ public void from_protobuf() throws TextFormat.ParseException {
       "  kind: POSITIONAL_OR_KEYWORD\n" +
       "  type_annotation {\n" +
       "    pretty_printed_name: \"builtins.str\"\n" +
-      "    simple_name: \"builtins.str\"\n" +
+      "    fully_qualified_name: \"builtins.str\"\n" +
       "  }\n" +
       "}\n" +
       "parameters {\n" +

python-frontend/src/test/java/org/sonar/python/types/InferredTypesTest.java

@@ -344,19 +344,19 @@ public void test_type_from_protobuf() throws TextFormat.ParseException {
     assertThat(protobufType("kind: TUPLE")).isEqualTo(TUPLE);
     assertThat(protobufType(
       "pretty_printed_name: \"builtins.str\"\n" +
-      "simple_name: \"builtins.str\"\n")).isEqualTo(STR);
+      "fully_qualified_name: \"builtins.str\"\n")).isEqualTo(STR);
     assertThat(protobufType(
       "kind: TYPE_ALIAS\n" +
       "args {\n" +
       "  kind: UNION\n" +
       "  args {\n" +
-      "    simple_name: \"builtins.str\"\n" +
+      "    fully_qualified_name: \"builtins.str\"\n" +
       "  }\n" +
       "  args {\n" +
-      "    simple_name: \"builtins.int\"\n" +
+      "    fully_qualified_name: \"builtins.int\"\n" +
       "  }\n" +
       "}\n" +
-      "simple_name: \"mod.t\""
+      "fully_qualified_name: \"mod.t\""
       ))
       .isEqualTo(InferredTypes.or(STR, INT));
   }