Add performance measurement for project level symbol table (#925)

Author: guillaume-dequenne

docs/properties.md

@@ -0,0 +1,9 @@
+# Internal analysis properties of the Python analyzer
+
+``sonar.python.performance.measure``: Boolean; if set to true, will enable performance monitoring of the analyzer (default: `false`).
+
+``sonar.python.performance.measure.path``: Path where the performance monitoring report will be saved, relative to the work dir (default: `sonar-python-performance-measure.json`).
+
+``sonar.internal.analysis.failFast``: Boolean; if set to true, exceptions will fail the analysis (default: `false`).
+
+``sonar.python.sonarlint.maxlines``: Maximum number of lines in a project above which the project symbol table won't be computed in SonarLint context (default: `150000`).

its/plugin/it-python-plugin-test/src/test/java/com/sonar/python/it/plugin/SonarLintTest.java

@@ -104,7 +104,7 @@ public void should_raise_issues() throws IOException {
     sonarlintEngine.declareModule(new ModuleInfo("myModule", (a, b) -> Stream.of(inputFile)));
     sonarlintEngine.analyze(configuration, issues::add, logOutput, null);
 
-    assertThat(logsByLevel.get(LogOutput.Level.WARN)).isNull();
+    assertThat(logsByLevel.get(LogOutput.Level.WARN)).containsExactly("No workDir in SonarLint");
     assertThat(issues).extracting("ruleKey", "startLine", "inputFile.path", "severity").containsOnly(
       tuple("python:BackticksUsage", 2, inputFile.getPath(), "BLOCKER"),
       tuple("python:S1542", 1, inputFile.getPath(), "MAJOR"));

pom.xml

@@ -91,7 +91,7 @@
     <mockito.version>3.9.0</mockito.version>
     <sonar.version>8.9.0.43852</sonar.version>
     <sonar.orchestrator.version>3.35.1.2719</sonar.orchestrator.version>
-    <sonar-analyzer-commons.version>1.14.1.690</sonar-analyzer-commons.version>
+    <sonar-analyzer-commons.version>1.15.0.696</sonar-analyzer-commons.version>
     <sonarlint-core.version>5.4.0.31923</sonarlint-core.version>
     <sslr.version>1.23</sslr.version>
   </properties>
@@ -120,6 +120,11 @@
         <artifactId>sonar-xml-parsing</artifactId>
         <version>${sonar-analyzer-commons.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.sonarsource.analyzer-commons</groupId>
+        <artifactId>sonar-performance-measure</artifactId>
+        <version>${sonar-analyzer-commons.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.sonarsource.analyzer-commons</groupId>
         <artifactId>sonar-analyzer-test-commons</artifactId>

sonar-python-plugin/pom.xml

@@ -56,6 +56,10 @@
       <groupId>org.sonarsource.analyzer-commons</groupId>
       <artifactId>sonar-xml-parsing</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.sonarsource.analyzer-commons</groupId>
+      <artifactId>sonar-performance-measure</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.sonarsource.sslr</groupId>
       <artifactId>sslr-testing-harness</artifactId>
@@ -146,8 +150,8 @@
             <configuration>
               <rules>
                 <requireFilesSize>
-                  <maxsize>4810000</maxsize>
-                  <minsize>4310000</minsize>
+                  <maxsize>5000000</maxsize>
+                  <minsize>4500000</minsize>
                   <files>
                     <file>${project.build.directory}/${project.build.finalName}.jar</file>
                   </files>

sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonSensor.java

@@ -21,10 +21,12 @@
 
 import com.sonar.sslr.api.AstNode;
 import com.sonar.sslr.api.RecognitionException;
+import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Optional;
 import javax.annotation.Nullable;
 import org.sonar.api.batch.fs.FilePredicates;
 import org.sonar.api.batch.fs.InputFile;
@@ -47,9 +49,14 @@
 import org.sonar.python.parser.PythonParser;
 import org.sonar.python.semantic.ProjectLevelSymbolTable;
 import org.sonar.python.tree.PythonTreeMaker;
+import org.sonarsource.performance.measure.PerformanceMeasure;
 
 public final class PythonSensor implements Sensor {
 
+  private static final String PERFORMANCE_MEASURE_PROPERTY = "sonar.python.performance.measure";
+  private static final String PERFORMANCE_MEASURE_FILE_PATH_PROPERTY = "sonar.python.performance.measure.path";
+  private static final String PERFORMANCE_MEASURE_DESTINATION_FILE = "sonar-python-performance-measure.json";
+
   private final PythonChecks checks;
   private final FileLinesContextFactory fileLinesContextFactory;
   private final NoSonarFilter noSonarFilter;
@@ -92,11 +99,13 @@ public void describe(SensorDescriptor descriptor) {
 
   @Override
   public void execute(SensorContext context) {
+    PerformanceMeasure.Duration durationReport = createPerformanceMeasureReport(context);
     List<InputFile> mainFiles = getInputFiles(Type.MAIN, context);
     List<InputFile> testFiles = getInputFiles(Type.TEST, context);
     PythonIndexer pythonIndexer = this.indexer != null ? this.indexer : new SonarQubePythonIndexer(mainFiles);
     PythonScanner scanner = new PythonScanner(context, checks, fileLinesContextFactory, noSonarFilter, pythonIndexer);
     scanner.execute(mainFiles, context);
+    durationReport.stop();
     if (!testFiles.isEmpty()) {
       new TestHighlightingScanner(context).execute(testFiles, context);
     }
@@ -110,6 +119,19 @@ private static List<InputFile> getInputFiles(InputFile.Type type, SensorContext
     return Collections.unmodifiableList(list);
   }
 
+  private static PerformanceMeasure.Duration createPerformanceMeasureReport(SensorContext context) {
+    return PerformanceMeasure.reportBuilder()
+      .activate(context.config().getBoolean(PERFORMANCE_MEASURE_PROPERTY).orElse(Boolean.FALSE))
+      .toFile(context.config().get(PERFORMANCE_MEASURE_FILE_PATH_PROPERTY)
+        .filter(path -> !path.isEmpty())
+        .orElseGet(() -> Optional.ofNullable(context.fileSystem().workDir())
+          .filter(File::exists)
+          .map(file -> file.toPath().resolve(PERFORMANCE_MEASURE_DESTINATION_FILE).toString())
+          .orElse(null)))
+      .appendMeasurementCost()
+      .start("PythonSensor");
+  }
+
   private static class TestHighlightingScanner extends Scanner {
 
     private static final Logger LOG = Loggers.get(TestHighlightingScanner.class);

sonar-python-plugin/src/main/java/org/sonar/plugins/python/indexer/SonarLintPythonIndexer.java

@@ -53,11 +53,8 @@ public void buildOnce(SensorContext context) {
     List<InputFile> files = getInputFiles(moduleFileSystem);
     LOG.debug("Input files for indexing: " + files);
     // computes "globalSymbolsByModuleName"
-    long startTime = System.currentTimeMillis();
     GlobalSymbolsScanner globalSymbolsStep = new GlobalSymbolsScanner(context);
     globalSymbolsStep.execute(files, context);
-    long stopTime = System.currentTimeMillis() - startTime;
-    LOG.debug("Time to build the project level symbol table: " + stopTime + "ms");
     shouldBuildProjectSymbolTable = false;
   }
 

sonar-python-plugin/src/main/java/org/sonar/plugins/python/indexer/SonarQubePythonIndexer.java

@@ -24,6 +24,7 @@
 import org.sonar.api.batch.sensor.SensorContext;
 import org.sonar.api.utils.log.Logger;
 import org.sonar.api.utils.log.Loggers;
+import org.sonarsource.performance.measure.PerformanceMeasure;
 
 public class SonarQubePythonIndexer extends PythonIndexer {
 
@@ -37,12 +38,11 @@ public SonarQubePythonIndexer(List<InputFile> files) {
   @Override
   public void buildOnce(SensorContext context) {
     this.projectBaseDirAbsolutePath = context.fileSystem().baseDir().getAbsolutePath();
+    PerformanceMeasure.Duration duration = PerformanceMeasure.start("ProjectLevelSymbolTable");
     LOG.debug("Input files for indexing: " + files);
     // computes "globalSymbolsByModuleName"
-    long startTime = System.currentTimeMillis();
     GlobalSymbolsScanner globalSymbolsStep = new GlobalSymbolsScanner(context);
     globalSymbolsStep.execute(files, context);
-    long stopTime = System.currentTimeMillis() - startTime;
-    LOG.debug("Time to build the project level symbol table: " + stopTime + "ms");
+    duration.stop();
   }
 }

sonar-python-plugin/src/test/java/org/sonar/plugins/python/PythonSensorTest.java

@@ -447,6 +447,65 @@ public void cancelled_analysis() {
     assertThat(context.allAnalysisErrors()).isEmpty();
   }
 
+  @Test
+  public void saving_performance_measure_not_activated_by_default() throws IOException {
+    activeRules = (new ActiveRulesBuilder()).build();
+
+    inputFile("main.py");
+    sensor().execute(context);
+    assertThat(context.allIssues()).isEmpty();
+    assertThat(logTester.logs(LoggerLevel.INFO)).noneMatch(s -> s.matches(".*performance measures.*"));
+    Path defaultPerformanceFile = workDir.resolve("sonar-python-performance-measure.json");
+    assertThat(defaultPerformanceFile).doesNotExist();
+  }
+
+  @Test
+  public void saving_performance_measure() throws IOException {
+    context.setSettings(new MapSettings().setProperty("sonar.python.performance.measure", "true"));
+    activeRules = (new ActiveRulesBuilder()).build();
+
+    inputFile("main.py");
+    sensor().execute(context);
+    Path defaultPerformanceFile = workDir.resolve("sonar-python-performance-measure.json");
+    assertThat(logTester.logs(LoggerLevel.INFO)).anyMatch(s -> s.matches(".*performance measures.*"));
+    assertThat(defaultPerformanceFile).exists();
+    assertThat(new String(Files.readAllBytes(defaultPerformanceFile), StandardCharsets.UTF_8)).contains("\"PythonSensor\"");
+  }
+
+  @Test
+  public void saving_performance_measure_custom_path() throws IOException {
+    Path customPerformanceFile = workDir.resolve("custom.performance.measure.json");
+    MapSettings mapSettings = new MapSettings();
+    mapSettings.setProperty("sonar.python.performance.measure", "true");
+    mapSettings.setProperty("sonar.python.performance.measure.path", customPerformanceFile.toString());
+    context.setSettings(mapSettings);
+    activeRules = (new ActiveRulesBuilder()).build();
+
+    inputFile("main.py");
+    sensor().execute(context);
+    assertThat(logTester.logs(LoggerLevel.INFO)).anyMatch(s -> s.matches(".*performance measures.*"));
+    Path defaultPerformanceFile = workDir.resolve("sonar-python-performance-measure.json");
+    assertThat(defaultPerformanceFile).doesNotExist();
+    assertThat(customPerformanceFile).exists();
+    assertThat(new String(Files.readAllBytes(customPerformanceFile), StandardCharsets.UTF_8)).contains("\"PythonSensor\"");
+  }
+
+  @Test
+  public void saving_performance_measure_empty_path() throws IOException {
+    MapSettings mapSettings = new MapSettings();
+    mapSettings.setProperty("sonar.python.performance.measure", "true");
+    mapSettings.setProperty("sonar.python.performance.measure.path", "");
+    context.setSettings(mapSettings);
+    activeRules = (new ActiveRulesBuilder()).build();
+
+    inputFile("main.py");
+    sensor().execute(context);
+    assertThat(logTester.logs(LoggerLevel.INFO)).anyMatch(s -> s.matches(".*performance measures.*"));
+    Path defaultPerformanceFile = workDir.resolve("sonar-python-performance-measure.json");
+    assertThat(defaultPerformanceFile).exists();
+    assertThat(new String(Files.readAllBytes(defaultPerformanceFile), StandardCharsets.UTF_8)).contains("\"PythonSensor\"");
+  }
+
   private PythonSensor sensor() {
     return sensor(CUSTOM_RULES, null);
   }
@@ -480,12 +539,12 @@ private InputFile inputFile(String name) {
 
   private DefaultInputFile createInputFile(String name) {
     return TestInputFileBuilder.create("moduleKey", name)
-        .setModuleBaseDir(baseDir.toPath())
-        .setCharset(StandardCharsets.UTF_8)
-        .setType(Type.MAIN)
-        .setLanguage(Python.KEY)
-        .initMetadata(TestUtils.fileContent(new File(baseDir, name), StandardCharsets.UTF_8))
-        .build();
+      .setModuleBaseDir(baseDir.toPath())
+      .setCharset(StandardCharsets.UTF_8)
+      .setType(Type.MAIN)
+      .setLanguage(Python.KEY)
+      .initMetadata(TestUtils.fileContent(new File(baseDir, name), StandardCharsets.UTF_8))
+      .build();
   }
 
   private void verifyUsages(String componentKey, int line, int offset, TextRange... trs) {