SONARPY-1232 Load all Typeshed symbols used by the project in PR analysis context (#1323)

Author: guillaume-dequenne

python-frontend/src/main/java/org/sonar/plugins/python/api/PythonInputFileContext.java

@@ -20,10 +20,13 @@
 package org.sonar.plugins.python.api;
 
 import java.io.File;
+import java.util.Collection;
 import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
 import org.sonar.api.Beta;
 import org.sonar.plugins.python.api.caching.CacheContext;
+import org.sonar.plugins.python.api.symbols.Symbol;
+import org.sonar.python.types.TypeShed;
 
 public class PythonInputFileContext {
 
@@ -46,6 +49,11 @@ public CacheContext cacheContext() {
     return cacheContext;
   }
 
+  @Beta
+  public Collection<Symbol> stubFilesSymbols() {
+    return TypeShed.stubFilesSymbols();
+  }
+
   @CheckForNull
   public File workingDirectory() {
     return workingDirectory;

python-frontend/src/main/java/org/sonar/python/types/TypeShed.java

@@ -182,6 +182,16 @@ public static Collection<Symbol> stubFilesSymbols() {
     return symbols;
   }
 
+  public static Set<String> stubModules() {
+    Set<String> modules = new HashSet<>();
+    for (Map.Entry<String, Map<String, Symbol>> entry : typeShedSymbols.entrySet()) {
+      if (!entry.getValue().isEmpty()) {
+        modules.add(entry.getKey());
+      }
+    }
+    return modules;
+  }
+
   public static String normalizedFqn(String fqn) {
     if (fqn.startsWith(BUILTINS_PREFIX)) {
       return fqn.substring(BUILTINS_PREFIX.length());
@@ -259,7 +269,7 @@ public static SymbolsProtos.ClassSymbol classDescriptorWithFQN(String fullyQuali
   //================================================================================
 
   // used by tests whenever 'sonar.python.version' changes
-  static void resetBuiltinSymbols() {
+  public static void resetBuiltinSymbols() {
     builtins = null;
     typeShedSymbols.clear();
     builtinSymbols();

python-frontend/src/test/java/org/sonar/python/PythonVisitorCheckTest.java

@@ -31,6 +31,8 @@
 import org.sonar.plugins.python.api.IssueLocation;
 import org.sonar.plugins.python.api.PythonCheck;
 import org.sonar.plugins.python.api.PythonCheck.PreciseIssue;
+import org.sonar.plugins.python.api.PythonFile;
+import org.sonar.plugins.python.api.PythonInputFileContext;
 import org.sonar.plugins.python.api.PythonSubscriptionCheck;
 import org.sonar.plugins.python.api.PythonVisitorCheck;
 import org.sonar.plugins.python.api.PythonVisitorContext;
@@ -40,10 +42,12 @@
 import org.sonar.plugins.python.api.tree.FunctionDef;
 import org.sonar.plugins.python.api.tree.Name;
 import org.sonar.plugins.python.api.tree.Tree;
+import org.sonar.python.caching.CacheContextImpl;
 import org.sonar.python.semantic.ProjectLevelSymbolTable;
 import org.sonar.python.types.TypeShed;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
 
 public class PythonVisitorCheckTest {
 
@@ -143,7 +147,7 @@ public void initialize(SubscriptionCheck.Context context) {
     };
     File tmpFile = Files.createTempFile("foo", "py").toFile();
 
-    var cache = Mockito.mock(CacheContext.class);
+    var cache = mock(CacheContext.class);
 
     PythonVisitorContext context = TestPythonVisitorRunner.createContext(tmpFile, null, "", ProjectLevelSymbolTable.empty(), cache);
     assertThat(context.workingDirectory()).isNull();
@@ -164,6 +168,9 @@ public void stubFilesSymbols() {
     SubscriptionVisitor.analyze(Collections.singletonList(check), context);
 
     assertThat(check.symbols).isEqualTo(TypeShed.stubFilesSymbols());
+
+    PythonInputFileContext inputFileContext = new PythonInputFileContext(mock(PythonFile.class), null, CacheContextImpl.dummyCache());
+    assertThat(inputFileContext.stubFilesSymbols()).isEqualTo(TypeShed.stubFilesSymbols());
   }
 
   private static class TestPythonCheck extends PythonVisitorCheck {

python-frontend/src/test/java/org/sonar/python/types/TypeShedTest.java

@@ -474,6 +474,13 @@ public void symbolWithFQN_should_be_consistent() {
     assertThat(sequence).isSameAs(typing.get("Sequence"));
   }
 
+  @Test
+  public void stubModules() {
+    TypeShed.symbolsForModule("doesnotexist");
+    TypeShed.symbolsForModule("math");
+    assertThat(TypeShed.stubModules()).containsExactly("math");
+  }
+
   private static SymbolsProtos.ModuleSymbol moduleSymbol(String protobuf) throws TextFormat.ParseException {
     SymbolsProtos.ModuleSymbol.Builder builder = SymbolsProtos.ModuleSymbol.newBuilder();
     TextFormat.merge(protobuf, builder);

sonar-python-plugin/src/main/java/org/sonar/plugins/python/caching/Caching.java

@@ -43,6 +43,7 @@ public class Caching {
   public static final String IMPORTS_MAP_CACHE_KEY_PREFIX = "python:imports:";
   public static final String PROJECT_SYMBOL_TABLE_CACHE_KEY_PREFIX = "python:descriptors:";
   public static final String PROJECT_FILES_KEY = "python:files";
+  public static final String TYPESHED_MODULES_KEY = "python:typeshed_modules";
   public static final String CACHE_VERSION_KEY = "python:cache_version";
 
   private static final Logger LOG = Loggers.get(Caching.class);
@@ -65,6 +66,11 @@ public void writeFilesList(List<String> mainFiles) {
     cacheContext.getWriteCache().write(PROJECT_FILES_KEY, projectFiles);
   }
 
+  public void writeTypeshedModules(Set<String> stubModules) {
+    byte[] stubModulesBytes = String.join(";", stubModules).getBytes(StandardCharsets.UTF_8);
+    cacheContext.getWriteCache().write(TYPESHED_MODULES_KEY, stubModulesBytes);
+  }
+
   public void writeCacheVersion() {
     cacheContext.getWriteCache().write(CACHE_VERSION_KEY, cacheVersion.getBytes(StandardCharsets.UTF_8));
   }
@@ -106,7 +112,15 @@ public Set<String> readImportMapEntry(String moduleFqn) {
   }
 
   public Set<String> readFilesList() {
-    byte[] bytes = cacheContext.getReadCache().readBytes(PROJECT_FILES_KEY);
+    return readSet(PROJECT_FILES_KEY);
+  }
+
+  public Set<String> readTypeshedModules() {
+    return readSet(TYPESHED_MODULES_KEY);
+  }
+
+  private Set<String> readSet(String cacheKey) {
+    byte[] bytes = cacheContext.getReadCache().readBytes(cacheKey);
     if (bytes != null) {
       return new HashSet<>(Arrays.asList(new String(bytes, StandardCharsets.UTF_8).split(";")));
     }

sonar-python-plugin/src/main/java/org/sonar/plugins/python/indexer/SonarQubePythonIndexer.java

@@ -36,6 +36,7 @@
 import org.sonar.python.index.Descriptor;
 import org.sonar.python.semantic.DependencyGraph;
 import org.sonar.python.semantic.SymbolUtils;
+import org.sonar.python.types.TypeShed;
 import org.sonarsource.performance.measure.PerformanceMeasure;
 
 import static org.sonar.plugins.python.api.PythonVersionUtils.PYTHON_VERSION_KEY;
@@ -88,6 +89,7 @@ private boolean shouldOptimizeAnalysis(SensorContext context) {
   }
 
   private void computeGlobalSymbolsUsingCache(SensorContext context) {
+    loadTypeshedSymbols();
     LOG.info("Using cached data to retrieve global symbols.");
     Set<String> currentProjectModulesFQNs = new HashSet<>(inputFileToFQN.values());
     Set<String> deletedModulesFQNs = deletedModulesFQNs(currentProjectModulesFQNs);
@@ -121,6 +123,17 @@ private void computeGlobalSymbolsUsingCache(SensorContext context) {
     computeGlobalSymbols(impactfulFiles, context);
   }
 
+  /*
+    In a full analysis, Typeshed symbols are loaded lazily depending on which module is encountered during parsing.
+    SonarSecurity needs all Typeshed symbols used in the project to be properly loaded.
+    For that reason, we load all symbols that were used in the previous analysis upfront, even if the file using them will not be parsed.
+   */
+  private void loadTypeshedSymbols() {
+    TypeShed.builtinSymbols();
+    Set<String> typeShedModules = caching.readTypeshedModules();
+    typeShedModules.forEach(TypeShed::symbolsForModule);
+  }
+
   private boolean tryToUseCache(Map<String, Set<String>> importsByModule, InputFile inputFile, String currFQN) {
     if (!inputFile.status().equals(InputFile.Status.SAME)) {
       return false;
@@ -150,6 +163,10 @@ public void computeGlobalSymbols(List<InputFile> files, SensorContext context) {
     if (caching.isCacheEnabled()) {
       saveGlobalSymbolsInCache(files);
       saveMainFilesListInCache(new HashSet<>(inputFileToFQN.values()));
+      Set<String> stubModules = TypeShed.stubModules();
+      if (!stubModules.isEmpty()) {
+        caching.writeTypeshedModules(stubModules);
+      }
       caching.writeCacheVersion();
     }
   }

sonar-python-plugin/src/test/java/org/sonar/plugins/python/indexer/SonarQubePythonIndexerTest.java

@@ -45,13 +45,16 @@
 import org.sonar.python.caching.PythonReadCacheImpl;
 import org.sonar.python.caching.PythonWriteCacheImpl;
 import org.sonar.python.index.VariableDescriptor;
+import org.sonar.python.types.TypeShed;
 
+import static org.assertj.core.api.Assertions.as;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
 import static org.sonar.plugins.python.caching.Caching.CACHE_VERSION_KEY;
 import static org.sonar.plugins.python.caching.Caching.PROJECT_FILES_KEY;
 import static org.sonar.plugins.python.caching.Caching.PROJECT_SYMBOL_TABLE_CACHE_KEY_PREFIX;
+import static org.sonar.plugins.python.caching.Caching.TYPESHED_MODULES_KEY;
 import static org.sonar.python.index.DescriptorsToProtobuf.toProtobufModuleDescriptor;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.sonar.plugins.python.TestUtils.createInputFile;
@@ -75,6 +78,7 @@ public class SonarQubePythonIndexerTest {
 
   @Before
   public void init() throws IOException {
+    TypeShed.resetBuiltinSymbols();
     context = SensorContextTester.create(baseDir);
     Path workDir = Files.createTempDirectory("workDir");
     context.fileSystem().setWorkDir(workDir);
@@ -375,6 +379,35 @@ public void test_disabled_cache() {
     assertThat(logTester.logs(LoggerLevel.INFO)).doesNotContain("Using cached data to retrieve global symbols.");
   }
 
+  @Test
+  public void test_typeshed_modules_cached() {
+    file1 = createInputFile(baseDir, "uses_typeshed.py", InputFile.Status.CHANGED, InputFile.Type.MAIN);
+
+    List<InputFile> inputFiles = new ArrayList<>(List.of(file1));
+
+    pythonIndexer = new SonarQubePythonIndexer(inputFiles, cacheContext, context);
+    pythonIndexer.buildOnce(context);
+
+    assertThat(pythonIndexer.canBeScannedWithoutParsing(file1)).isFalse();
+
+    byte[] bytes = writeCache.getData().get(TYPESHED_MODULES_KEY);
+    Set<String> resolvedTypeshedModules = new HashSet<>(Arrays.asList(new String(bytes, StandardCharsets.UTF_8).split(";")));
+    assertThat(resolvedTypeshedModules).containsExactlyInAnyOrder("math");
+  }
+
+  @Test
+  public void test_typeshed_modules_not_cached_if_empty() {
+    file1 = createInputFile(baseDir, "main.py", InputFile.Status.CHANGED, InputFile.Type.MAIN);
+
+    List<InputFile> inputFiles = new ArrayList<>(List.of(file1));
+
+    pythonIndexer = new SonarQubePythonIndexer(inputFiles, cacheContext, context);
+    pythonIndexer.buildOnce(context);
+
+    assertThat(pythonIndexer.canBeScannedWithoutParsing(file1)).isFalse();
+    assertThat(writeCache.getData()).doesNotContainKey(TYPESHED_MODULES_KEY);
+  }
+
   @Test
   public void test_regular_scan_when_scan_without_parsing_fails() {
     List<InputFile> files = List.of(createInputFile(baseDir, "main.py", InputFile.Status.SAME, InputFile.Type.MAIN));

sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/uses_typeshed.py

@@ -0,0 +1,4 @@
+import math
+
+def func(a):
+    ...