Skip to content

Commit b823950

Browse files
guillaume-dequenneandrea-guarino-sonarsource
guillaume-dequenne
authored and
andrea-guarino-sonarsource
committed
SONARPY-852 Declare PythonIndexer as SonarLint component with module scope (#919)
1 parent 13097b3 commit b823950

File tree

18 files changed

+516
-169
lines changed

18 files changed

+516
-169
lines changed

‎its/plugin/it-python-plugin-test/src/test/java/com/sonar/python/it/plugin/SonarLintTest.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,17 +28,21 @@
2828
import java.nio.file.Files;
2929
import java.nio.file.Path;
3030
import java.util.ArrayList;
31+
import java.util.Collections;
3132
import java.util.HashMap;
3233
import java.util.List;
3334
import java.util.Map;
35+
import java.util.stream.Stream;
3436
import org.apache.commons.io.FileUtils;
3537
import org.junit.AfterClass;
3638
import org.junit.BeforeClass;
3739
import org.junit.ClassRule;
3840
import org.junit.Test;
3941
import org.junit.rules.TemporaryFolder;
4042
import org.sonarsource.sonarlint.core.StandaloneSonarLintEngineImpl;
43+
import org.sonarsource.sonarlint.core.client.api.common.Language;
4144
import org.sonarsource.sonarlint.core.client.api.common.LogOutput;
45+
import org.sonarsource.sonarlint.core.client.api.common.ModuleInfo;
4246
import org.sonarsource.sonarlint.core.client.api.common.analysis.ClientInputFile;
4347
import org.sonarsource.sonarlint.core.client.api.common.analysis.Issue;
4448
import org.sonarsource.sonarlint.core.client.api.standalone.StandaloneAnalysisConfiguration;
@@ -62,8 +66,10 @@ public static void prepare() throws Exception {
6266
StandaloneGlobalConfiguration sonarLintConfig = StandaloneGlobalConfiguration.builder()
6367
.addPlugin(Tests.PLUGIN_LOCATION.getFile().toURI().toURL())
6468
.setSonarLintUserHome(TEMP.newFolder().toPath())
69+
.addEnabledLanguage(Language.PYTHON)
6570
.setLogOutput((formattedMessage, level) -> {
6671
/* Don't pollute logs */ })
72+
.setModulesProvider(Collections::emptyList)
6773
.build();
6874
sonarlintEngine = new StandaloneSonarLintEngineImpl(sonarLintConfig);
6975
baseDir = TEMP.newFolder();
@@ -86,6 +92,7 @@ public void should_raise_issues() throws IOException {
8692
StandaloneAnalysisConfiguration.builder()
8793
.setBaseDir(baseDir.toPath())
8894
.addInputFile(inputFile)
95+
.setModuleKey("myModule")
8996
.build();
9097

9198
Map<LogOutput.Level, List<String>> logsByLevel = new HashMap<>();
@@ -94,7 +101,7 @@ public void should_raise_issues() throws IOException {
94101
logs.add(s);
95102
logsByLevel.putIfAbsent(level, logs);
96103
};
97-
104+
sonarlintEngine.declareModule(new ModuleInfo("myModule", (a, b) -> Stream.of(inputFile)));
98105
sonarlintEngine.analyze(configuration, issues::add, logOutput, null);
99106

100107
assertThat(logsByLevel.get(LogOutput.Level.WARN)).isNull();

‎pom.xml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@
9292
<sonar.version>8.9.0.43852</sonar.version>
9393
<sonar.orchestrator.version>3.35.1.2719</sonar.orchestrator.version>
9494
<sonar-analyzer-commons.version>1.14.1.690</sonar-analyzer-commons.version>
95-
<sonarlint-core.version>4.4.0.2561</sonarlint-core.version>
95+
<sonarlint-core.version>5.4.0.31832</sonarlint-core.version>
9696
<sslr.version>1.23</sslr.version>
9797
</properties>
9898

@@ -193,6 +193,12 @@
193193
<version>${sonarlint-core.version}</version>
194194
<scope>test</scope>
195195
</dependency>
196+
<dependency>
197+
<groupId>org.sonarsource.sonarlint.core</groupId>
198+
<artifactId>sonarlint-plugin-api</artifactId>
199+
<version>${sonarlint-core.version}</version>
200+
<scope>provided</scope>
201+
</dependency>
196202
<dependency>
197203
<groupId>com.google.guava</groupId>
198204
<artifactId>guava</artifactId>

‎python-frontend/src/main/java/org/sonar/python/semantic/ProjectLevelSymbolTable.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,8 @@ private ProjectLevelSymbolTable(Map<String, Set<Symbol>> globalSymbolsByModuleNa
6363
this.globalSymbolsByModuleName = new HashMap<>(globalSymbolsByModuleName);
6464
}
6565

66-
public void removeModule(String packageName, PythonFile pythonFile) {
67-
String fullyQualifiedModuleName = SymbolUtils.fullyQualifiedModuleName(packageName, pythonFile.fileName());
66+
public void removeModule(String packageName, String fileName) {
67+
String fullyQualifiedModuleName = SymbolUtils.fullyQualifiedModuleName(packageName, fileName);
6868
globalSymbolsByModuleName.remove(fullyQualifiedModuleName);
6969
// ensure globalSymbolsByFQN is re-computed
7070
this.globalSymbolsByFQN = null;

‎python-frontend/src/test/java/org/sonar/python/semantic/ProjectLevelSymbolTableTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -406,7 +406,7 @@ public void test_remove_module() {
406406
ProjectLevelSymbolTable projectLevelSymbolTable = new ProjectLevelSymbolTable();
407407
projectLevelSymbolTable.addModule(tree, "", pythonFile("mod.py"));
408408
assertThat(projectLevelSymbolTable.getSymbolsFromModule("mod")).extracting(Symbol::name).containsExactlyInAnyOrder("A");
409-
projectLevelSymbolTable.removeModule("", pythonFile("mod.py"));
409+
projectLevelSymbolTable.removeModule("", "mod.py");
410410
assertThat(projectLevelSymbolTable.getSymbolsFromModule("mod")).isNull();
411411
}
412412

‎sonar-python-plugin/pom.xml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,10 @@
2828
<groupId>org.sonarsource.sonarqube</groupId>
2929
<artifactId>sonar-plugin-api</artifactId>
3030
</dependency>
31+
<dependency>
32+
<groupId>org.sonarsource.sonarlint.core</groupId>
33+
<artifactId>sonarlint-plugin-api</artifactId>
34+
</dependency>
3135
<dependency>
3236
<groupId>org.sonarsource.sonarqube</groupId>
3337
<artifactId>sonar-plugin-api-impl</artifactId>

‎sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonPlugin.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.sonar.plugins.python.coverage.PythonCoverageSensor;
3131
import org.sonar.plugins.python.flake8.Flake8RulesDefinition;
3232
import org.sonar.plugins.python.flake8.Flake8Sensor;
33+
import org.sonar.plugins.python.indexer.SonarLintPythonIndexer;
3334
import org.sonar.plugins.python.pylint.PylintRulesDefinition;
3435
import org.sonar.plugins.python.pylint.PylintSensor;
3536
import org.sonar.plugins.python.warnings.DefaultAnalysisWarningsWrapper;
@@ -79,6 +80,9 @@ public void define(Context context) {
7980
addBanditExtensions(context);
8081
addFlake8Extensions(context);
8182
}
83+
if (sonarRuntime.getProduct() == SonarProduct.SONARLINT) {
84+
context.addExtension(SonarLintPythonIndexer.class);
85+
}
8286
}
8387

8488
private static void addCoberturaExtensions(Context context) {

‎sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonScanner.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
import org.sonar.plugins.python.api.PythonVisitorContext;
5151
import org.sonar.plugins.python.api.tree.FileInput;
5252
import org.sonar.plugins.python.cpd.PythonCpdAnalyzer;
53+
import org.sonar.plugins.python.indexer.PythonIndexer;
5354
import org.sonar.python.SubscriptionVisitor;
5455
import org.sonar.python.metrics.FileLinesVisitor;
5556
import org.sonar.python.metrics.FileMetrics;
@@ -70,16 +71,15 @@ public class PythonScanner extends Scanner {
7071

7172
public PythonScanner(
7273
SensorContext context, PythonChecks checks,
73-
FileLinesContextFactory fileLinesContextFactory, NoSonarFilter noSonarFilter, List<InputFile> files
74-
) {
74+
FileLinesContextFactory fileLinesContextFactory, NoSonarFilter noSonarFilter, PythonIndexer indexer) {
7575
super(context);
7676
this.checks = checks;
7777
this.fileLinesContextFactory = fileLinesContextFactory;
7878
this.noSonarFilter = noSonarFilter;
7979
this.cpdAnalyzer = new PythonCpdAnalyzer(context);
8080
this.parser = PythonParser.create();
81-
this.indexer = new PythonIndexer();
82-
this.indexer.buildOnce(context, files);
81+
this.indexer = indexer;
82+
this.indexer.buildOnce(context);
8383
}
8484

8585
@Override
@@ -94,7 +94,7 @@ protected void scanFile(InputFile inputFile) {
9494
try {
9595
AstNode astNode = parser.parse(pythonFile.content());
9696
FileInput parse = new PythonTreeMaker().fileInput(astNode);
97-
visitorContext = new PythonVisitorContext(parse, pythonFile, getWorkingDirectory(context), indexer.packageName(inputFile.uri()), indexer.projectLevelSymbolTable());
97+
visitorContext = new PythonVisitorContext(parse, pythonFile, getWorkingDirectory(context), indexer.packageName(inputFile), indexer.projectLevelSymbolTable());
9898
saveMeasures(inputFile, visitorContext);
9999
} catch (RecognitionException e) {
100100
visitorContext = new PythonVisitorContext(pythonFile, e);

‎sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonSensor.java

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@
4141
import org.sonar.plugins.python.api.PythonFile;
4242
import org.sonar.plugins.python.api.PythonVisitorContext;
4343
import org.sonar.plugins.python.api.tree.FileInput;
44+
import org.sonar.plugins.python.indexer.PythonIndexer;
45+
import org.sonar.plugins.python.indexer.SonarQubePythonIndexer;
4446
import org.sonar.python.checks.CheckList;
4547
import org.sonar.python.parser.PythonParser;
4648
import org.sonar.python.semantic.ProjectLevelSymbolTable;
@@ -51,21 +53,33 @@ public final class PythonSensor implements Sensor {
5153
private final PythonChecks checks;
5254
private final FileLinesContextFactory fileLinesContextFactory;
5355
private final NoSonarFilter noSonarFilter;
56+
private final PythonIndexer indexer;
5457

5558
/**
56-
* Constructor to be used by pico if no PythonCustomRuleRepository are to be found and injected.
59+
* Constructor to be used by pico if neither PythonCustomRuleRepository nor PythonIndexer are to be found and injected.
5760
*/
5861
public PythonSensor(FileLinesContextFactory fileLinesContextFactory, CheckFactory checkFactory, NoSonarFilter noSonarFilter) {
59-
this(fileLinesContextFactory, checkFactory, noSonarFilter, null);
62+
this(fileLinesContextFactory, checkFactory, noSonarFilter, null, null);
6063
}
6164

6265
public PythonSensor(FileLinesContextFactory fileLinesContextFactory, CheckFactory checkFactory, NoSonarFilter noSonarFilter,
63-
@Nullable PythonCustomRuleRepository[] customRuleRepositories) {
66+
PythonIndexer indexer) {
67+
this(fileLinesContextFactory, checkFactory, noSonarFilter, null, indexer);
68+
}
69+
70+
public PythonSensor(FileLinesContextFactory fileLinesContextFactory, CheckFactory checkFactory, NoSonarFilter noSonarFilter,
71+
PythonCustomRuleRepository[] customRuleRepositories) {
72+
this(fileLinesContextFactory, checkFactory, noSonarFilter, customRuleRepositories, null);
73+
}
74+
75+
public PythonSensor(FileLinesContextFactory fileLinesContextFactory, CheckFactory checkFactory, NoSonarFilter noSonarFilter,
76+
@Nullable PythonCustomRuleRepository[] customRuleRepositories, @Nullable PythonIndexer indexer) {
6477
this.checks = new PythonChecks(checkFactory)
6578
.addChecks(CheckList.REPOSITORY_KEY, CheckList.getChecks())
6679
.addCustomChecks(customRuleRepositories);
6780
this.fileLinesContextFactory = fileLinesContextFactory;
6881
this.noSonarFilter = noSonarFilter;
82+
this.indexer = indexer;
6983
}
7084

7185
@Override
@@ -80,7 +94,8 @@ public void describe(SensorDescriptor descriptor) {
8094
public void execute(SensorContext context) {
8195
List<InputFile> mainFiles = getInputFiles(Type.MAIN, context);
8296
List<InputFile> testFiles = getInputFiles(Type.TEST, context);
83-
PythonScanner scanner = new PythonScanner(context, checks, fileLinesContextFactory, noSonarFilter, mainFiles);
97+
PythonIndexer pythonIndexer = this.indexer != null ? this.indexer : new SonarQubePythonIndexer(mainFiles);
98+
PythonScanner scanner = new PythonScanner(context, checks, fileLinesContextFactory, noSonarFilter, pythonIndexer);
8499
scanner.execute(mainFiles, context);
85100
if (!testFiles.isEmpty()) {
86101
new TestHighlightingScanner(context).execute(testFiles, context);

‎sonar-python-plugin/src/main/java/org/sonar/plugins/python/Scanner.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,16 +29,16 @@
2929
import org.sonar.api.utils.log.Loggers;
3030
import org.sonarsource.analyzer.commons.ProgressReport;
3131

32-
abstract class Scanner {
32+
public abstract class Scanner {
3333
private static final Logger LOG = Loggers.get(Scanner.class);
3434
private static final String FAIL_FAST_PROPERTY_NAME = "sonar.internal.analysis.failFast";
3535
protected final SensorContext context;
3636

37-
Scanner(SensorContext context) {
37+
protected Scanner(SensorContext context) {
3838
this.context = context;
3939
}
4040

41-
void execute(List<InputFile> files, SensorContext context) {
41+
public void execute(List<InputFile> files, SensorContext context) {
4242
ProgressReport progressReport = new ProgressReport(this.name() + " progress", TimeUnit.SECONDS.toMillis(10));
4343
LOG.info("Starting " + this.name());
4444
List<String> filenames = files.stream().map(InputFile::toString).collect(Collectors.toList());
@@ -63,9 +63,9 @@ void execute(List<InputFile> files, SensorContext context) {
6363
progressReport.stop();
6464
}
6565

66-
abstract String name();
66+
protected abstract String name();
6767

68-
abstract void scanFile(InputFile file) throws IOException;
68+
protected abstract void scanFile(InputFile file) throws IOException;
6969

70-
abstract void processException(Exception e, InputFile file);
70+
protected abstract void processException(Exception e, InputFile file);
7171
}

‎sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonIndexer.java renamed to ‎sonar-python-plugin/src/main/java/org/sonar/plugins/python/indexer/PythonIndexer.java

Lines changed: 20 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,20 @@
1717
* along with this program; if not, write to the Free Software Foundation,
1818
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
1919
*/
20-
package org.sonar.plugins.python;
20+
package org.sonar.plugins.python.indexer;
2121

2222
import com.sonar.sslr.api.AstNode;
2323
import java.io.File;
2424
import java.io.IOException;
2525
import java.net.URI;
2626
import java.util.HashMap;
27-
import java.util.List;
2827
import java.util.Map;
2928
import org.sonar.api.batch.fs.InputFile;
3029
import org.sonar.api.batch.sensor.SensorContext;
3130
import org.sonar.api.utils.log.Logger;
3231
import org.sonar.api.utils.log.Loggers;
32+
import org.sonar.plugins.python.Scanner;
33+
import org.sonar.plugins.python.SonarQubePythonFile;
3334
import org.sonar.plugins.python.api.PythonFile;
3435
import org.sonar.plugins.python.api.tree.FileInput;
3536
import org.sonar.python.parser.PythonParser;
@@ -38,40 +39,32 @@
3839

3940
import static org.sonar.python.semantic.SymbolUtils.pythonPackageName;
4041

41-
public class PythonIndexer {
42+
public abstract class PythonIndexer {
4243

4344
private static final Logger LOG = Loggers.get(PythonIndexer.class);
4445

46+
protected File projectBaseDir;
47+
4548
private final Map<URI, String> packageNames = new HashMap<>();
4649
private final PythonParser parser = PythonParser.create();
4750
private final ProjectLevelSymbolTable projectLevelSymbolTable = new ProjectLevelSymbolTable();
48-
private File projectBaseDir;
49-
50-
51-
void buildOnce(SensorContext context, List<InputFile> files) {
52-
this.projectBaseDir = context.fileSystem().baseDir();
53-
LOG.debug("Input files for indexing: " + files);
54-
// computes "globalSymbolsByModuleName"
55-
long startTime = System.currentTimeMillis();
56-
GlobalSymbolsScanner globalSymbolsStep = new GlobalSymbolsScanner(context);
57-
globalSymbolsStep.execute(files, context);
58-
long stopTime = System.currentTimeMillis() - startTime;
59-
LOG.debug("Time to build the project level symbol table: " + stopTime + "ms");
60-
}
6151

62-
String packageName(URI uri) {
63-
return packageNames.get(uri);
52+
public ProjectLevelSymbolTable projectLevelSymbolTable() {
53+
return projectLevelSymbolTable;
6454
}
6555

66-
ProjectLevelSymbolTable projectLevelSymbolTable() {
67-
return projectLevelSymbolTable;
56+
public String packageName(InputFile inputFile) {
57+
return packageNames.computeIfAbsent(inputFile.uri(), k -> pythonPackageName(inputFile.file(), projectBaseDir));
6858
}
6959

7060
void removeFile(InputFile inputFile) {
7161
String packageName = packageNames.get(inputFile.uri());
62+
if (packageName == null) {
63+
LOG.debug("Failed to remove file \"{}\" from project-level symbol table (file not indexed)", inputFile.filename());
64+
return;
65+
}
7266
packageNames.remove(inputFile.uri());
73-
PythonFile pythonFile = SonarQubePythonFile.create(inputFile);
74-
projectLevelSymbolTable.removeModule(packageName, pythonFile);
67+
projectLevelSymbolTable.removeModule(packageName, inputFile.filename());
7568
}
7669

7770
void addFile(InputFile inputFile) throws IOException {
@@ -83,9 +76,11 @@ void addFile(InputFile inputFile) throws IOException {
8376
projectLevelSymbolTable.addModule(astRoot, packageName, pythonFile);
8477
}
8578

86-
private class GlobalSymbolsScanner extends Scanner {
79+
public abstract void buildOnce(SensorContext context);
80+
81+
class GlobalSymbolsScanner extends Scanner {
8782

88-
private GlobalSymbolsScanner(SensorContext context) {
83+
protected GlobalSymbolsScanner(SensorContext context) {
8984
super(context);
9085
}
9186

@@ -101,7 +96,7 @@ protected void scanFile(InputFile inputFile) throws IOException {
10196

10297
@Override
10398
protected void processException(Exception e, InputFile file) {
104-
LOG.debug("Unable to construct project-level symbol table for file: " + file.toString());
99+
LOG.debug("Unable to construct project-level symbol table for file: " + file);
105100
LOG.debug(e.getMessage());
106101
}
107102
}

0 commit comments

Comments
 (0)