SONARPY-845 Introduce PythonIndexer wrapper for ProjectLevelSymbolTable (#916)

Author: andrea-guarino-sonarsource

sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonIndexer.java

@@ -0,0 +1,94 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2011-2021 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.plugins.python;
+
+import com.sonar.sslr.api.AstNode;
+import java.io.IOException;
+import java.net.URI;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.sensor.SensorContext;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
+import org.sonar.plugins.python.api.PythonFile;
+import org.sonar.plugins.python.api.tree.FileInput;
+import org.sonar.python.parser.PythonParser;
+import org.sonar.python.semantic.ProjectLevelSymbolTable;
+import org.sonar.python.tree.PythonTreeMaker;
+
+import static org.sonar.python.semantic.SymbolUtils.pythonPackageName;
+
+public class PythonIndexer {
+
+  private static final Logger LOG = Loggers.get(PythonIndexer.class);
+
+  private final Map<URI, String> packageNames = new HashMap<>();
+  private final PythonParser parser = PythonParser.create();
+  private final ProjectLevelSymbolTable projectLevelSymbolTable = new ProjectLevelSymbolTable();
+
+
+  void buildOnce(SensorContext context, List<InputFile> files) {
+    LOG.debug("Input files for indexing: " + files);
+    // computes "globalSymbolsByModuleName"
+    long startTime = System.currentTimeMillis();
+    GlobalSymbolsScanner globalSymbolsStep = new GlobalSymbolsScanner(context);
+    globalSymbolsStep.execute(files, context);
+    long stopTime = System.currentTimeMillis() - startTime;
+    LOG.debug("Time to build the project level symbol table: " + stopTime + "ms");
+  }
+
+  String packageName(URI uri) {
+    return packageNames.get(uri);
+  }
+
+  ProjectLevelSymbolTable projectLevelSymbolTable() {
+    return projectLevelSymbolTable;
+  }
+
+  private class GlobalSymbolsScanner extends Scanner {
+
+    private GlobalSymbolsScanner(SensorContext context) {
+      super(context);
+    }
+
+    @Override
+    protected String name() {
+      return "global symbols computation";
+    }
+
+    @Override
+    protected void scanFile(InputFile inputFile) throws IOException {
+      AstNode astNode = parser.parse(inputFile.contents());
+      FileInput astRoot = new PythonTreeMaker().fileInput(astNode);
+      String packageName = pythonPackageName(inputFile.file(), context.fileSystem().baseDir());
+      packageNames.put(inputFile.uri(), packageName);
+      PythonFile pythonFile = SonarQubePythonFile.create(inputFile);
+      projectLevelSymbolTable.addModule(astRoot, packageName, pythonFile);
+    }
+
+    @Override
+    protected void processException(Exception e, InputFile file) {
+      LOG.debug("Unable to construct project-level symbol table for file: " + file.toString());
+      LOG.debug(e.getMessage());
+    }
+  }
+}

sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonScanner.java

@@ -22,13 +22,10 @@
 import com.sonar.sslr.api.AstNode;
 import com.sonar.sslr.api.RecognitionException;
 import java.io.File;
-import java.io.IOException;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Deque;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 import javax.annotation.CheckForNull;
 import org.sonar.api.SonarProduct;
@@ -57,22 +54,18 @@
 import org.sonar.python.metrics.FileLinesVisitor;
 import org.sonar.python.metrics.FileMetrics;
 import org.sonar.python.parser.PythonParser;
-import org.sonar.python.semantic.ProjectLevelSymbolTable;
 import org.sonar.python.tree.PythonTreeMaker;
 
-import static org.sonar.python.semantic.SymbolUtils.pythonPackageName;
-
 public class PythonScanner extends Scanner {
 
   private static final Logger LOG = Loggers.get(PythonScanner.class);
 
   private final PythonParser parser;
-  private final Map<InputFile, String> packageNames = new HashMap<>();
   private final PythonChecks checks;
   private final FileLinesContextFactory fileLinesContextFactory;
   private final NoSonarFilter noSonarFilter;
   private final PythonCpdAnalyzer cpdAnalyzer;
-  private final ProjectLevelSymbolTable projectLevelSymbolTable = new ProjectLevelSymbolTable();
+  private final PythonIndexer indexer;
 
 
   public PythonScanner(
@@ -85,13 +78,8 @@ public PythonScanner(
     this.noSonarFilter = noSonarFilter;
     this.cpdAnalyzer = new PythonCpdAnalyzer(context);
     this.parser = PythonParser.create();
-
-    // computes "globalSymbolsByModuleName"
-    long startTime = System.currentTimeMillis();
-    GlobalSymbolsScanner globalSymbolsStep = new GlobalSymbolsScanner(context);
-    globalSymbolsStep.execute(files, context);
-    long stopTime = System.currentTimeMillis() - startTime;
-    LOG.debug("Time to build the project level symbol table: " + stopTime + "ms");
+    this.indexer = new PythonIndexer();
+    this.indexer.buildOnce(context, files);
   }
 
   @Override
@@ -106,7 +94,7 @@ protected void scanFile(InputFile inputFile) {
     try {
       AstNode astNode = parser.parse(pythonFile.content());
       FileInput parse = new PythonTreeMaker().fileInput(astNode);
-      visitorContext = new PythonVisitorContext(parse, pythonFile, getWorkingDirectory(context), packageNames.get(inputFile), projectLevelSymbolTable);
+      visitorContext = new PythonVisitorContext(parse, pythonFile, getWorkingDirectory(context), indexer.packageName(inputFile.uri()), indexer.projectLevelSymbolTable());
       saveMeasures(inputFile, visitorContext);
     } catch (RecognitionException e) {
       visitorContext = new PythonVisitorContext(pythonFile, e);
@@ -245,32 +233,4 @@ private void saveMetricOnFile(InputFile inputFile, Metric<Integer> metric, Integ
       .on(inputFile)
       .save();
   }
-
-  private class GlobalSymbolsScanner extends Scanner {
-
-    private GlobalSymbolsScanner(SensorContext context) {
-      super(context);
-    }
-
-    @Override
-    protected String name() {
-      return "global symbols computation";
-    }
-
-    @Override
-    protected void scanFile(InputFile inputFile) throws IOException {
-      AstNode astNode = parser.parse(inputFile.contents());
-      FileInput astRoot = new PythonTreeMaker().fileInput(astNode);
-      String packageName = pythonPackageName(inputFile.file(), context.fileSystem().baseDir());
-      packageNames.put(inputFile, packageName);
-      PythonFile pythonFile = SonarQubePythonFile.create(inputFile);
-      projectLevelSymbolTable.addModule(astRoot, packageName, pythonFile);
-    }
-
-    @Override
-    protected void processException(Exception e, InputFile file) {
-      LOG.debug("Unable to construct project-level symbol table for file: " + file.toString());
-      LOG.debug(e.getMessage());
-    }
-  }
 }