SONARPY-957 Deserialize typeshed variable symbols from protobuf. Fix FN on S4423

Author: andreaguarino

its/ruling/src/test/resources/expected/python-S4423.json

@@ -7,6 +7,15 @@
 174,
 213,
 ],
+'project:tornado-2.3/tornado/simple_httpclient.py':[
+210,
+],
+'project:tornado-2.3/tornado/test/httpserver_test.py':[
+110,
+115,
+120,
+125,
+],
 'project:twisted-12.1.0/doc/core/examples/echoserv_ssl.py':[
 15,
 ],

python-checks/src/test/java/org/sonar/python/checks/WeakSSLProtocolCheckTest.java

@@ -29,4 +29,8 @@ public void test() {
     PythonCheckVerifier.verify("src/test/resources/checks/weakSSLProtocol.py", new WeakSSLProtocolCheck());
   }
 
+  @Test
+  public void test_fallback_import() {
+    PythonCheckVerifier.verify("src/test/resources/checks/weakSSLProtocol_fallback_import.py", new WeakSSLProtocolCheck());
+  }
 }

python-checks/src/test/resources/checks/weakSSLProtocol_fallback_import.py

@@ -0,0 +1,7 @@
+try:
+    import ssl
+except ImportError:
+    ssl = None
+
+if ssl is not None:
+    ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv2) # Noncompliant

python-frontend/src/main/java/org/sonar/python/semantic/SymbolImpl.java

@@ -22,6 +22,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -39,6 +40,8 @@
 import org.sonar.plugins.python.api.types.InferredType;
 import org.sonar.python.tree.NameImpl;
 import org.sonar.python.types.InferredTypes;
+import org.sonar.python.types.TypeShed;
+import org.sonar.python.types.protobuf.SymbolsProtos;
 
 public class SymbolImpl implements Symbol {
 
@@ -65,6 +68,17 @@ public SymbolImpl(String name, @Nullable String fullyQualifiedName, @Nullable St
     this.kind = Kind.OTHER;
   }
 
+  public SymbolImpl(SymbolsProtos.VarSymbol varSymbol) {
+    this.name = varSymbol.getName();
+    this.fullyQualifiedName = TypeShed.normalizedFqn(varSymbol.getFullyQualifiedName());
+    String fqn = varSymbol.getTypeAnnotation().getFullyQualifiedName();
+    if (!fqn.isEmpty()) {
+      this.annotatedTypeName = TypeShed.normalizedFqn(fqn);
+    }
+    this.validForPythonVersions = new HashSet<>(varSymbol.getValidForList());
+    this.kind = Kind.OTHER;
+  }
+
   @Override
   public String name() {
     return name;

python-frontend/src/main/java/org/sonar/python/types/TypeShed.java

@@ -403,13 +403,17 @@ static Map<String, Symbol> getSymbolsFromProtobufModule(@Nullable ModuleSymbol m
     moduleSymbol.getOverloadedFunctionsList().stream()
       .filter(d -> isValidForProjectPythonVersion(d.getValidForList()))
       .forEach(proto -> descriptorsByName.computeIfAbsent(proto.getName(), d -> new HashSet<>()).add(proto));
+    moduleSymbol.getVarsList().stream()
+      .filter(d -> isValidForProjectPythonVersion(d.getValidForList()))
+      .forEach(proto -> descriptorsByName.computeIfAbsent(proto.getName(), d -> new HashSet<>()).add(proto));
 
     Map<String, Symbol> deserializedSymbols = new HashMap<>();
 
     for (Map.Entry<String, Set<Object>> entry : descriptorsByName.entrySet()) {
       String name = entry.getKey();
       Set<Symbol> symbols = symbolsFromDescriptor(entry.getValue(), false);
-      deserializedSymbols.put(name, disambiguateSymbolsWithSameName(name, symbols, moduleSymbol.getFullyQualifiedName()));
+      Symbol disambiguatedSymbol = disambiguateSymbolsWithSameName(name, symbols, moduleSymbol.getFullyQualifiedName());
+      deserializedSymbols.put(name, disambiguatedSymbol);
     }
     return deserializedSymbols;
   }
@@ -465,6 +469,9 @@ public static Set<Symbol> symbolsFromDescriptor(Set<Object> descriptors, boolean
         }
         symbols.add(fromOverloadedFunction(((OverloadedFunctionSymbol) descriptor), isInsideClass));
       }
+      if (descriptor instanceof SymbolsProtos.VarSymbol) {
+        symbols.add(new SymbolImpl((SymbolsProtos.VarSymbol) descriptor));
+      }
     }
     return symbols;
   }

python-frontend/src/test/java/org/sonar/python/types/TypeShedTest.java

@@ -466,4 +466,25 @@ private static Map<String, Symbol> symbolsForModule(String moduleName) {
     assertThat(symbols.stream().map(Symbol::name)).doesNotHaveDuplicates();
     return symbols.stream().collect(Collectors.toMap(Symbol::name, Function.identity()));
   }
+
+  @Test
+  public void variables_from_protobuf() throws TextFormat.ParseException {
+    SymbolsProtos.ModuleSymbol moduleSymbol = moduleSymbol(
+        "fully_qualified_name: \"mod\"\n" +
+        "vars {\n" +
+        "  name: \"foo\"\n" +
+        "  fully_qualified_name: \"mod.foo\"\n" +
+        "  type_annotation {\n" +
+        "    pretty_printed_name: \"builtins.str\"\n" +
+        "    fully_qualified_name: \"builtins.str\"\n" +
+        "  }\n" +
+        "}\n" +
+        "vars {\n" +
+        "  name: \"bar\"\n" +
+        "  fully_qualified_name: \"mod.bar\"\n" +
+        "}\n");
+    Map<String, Symbol> symbols = TypeShed.getSymbolsFromProtobufModule(moduleSymbol);
+    assertThat(symbols.values()).extracting(Symbol::kind, Symbol::fullyQualifiedName, Symbol::annotatedTypeName)
+      .containsExactlyInAnyOrder(tuple(Kind.OTHER, "mod.foo", "str"), tuple(Kind.OTHER, "mod.bar", null));
+  }
 }