Skip to content

Commit 13097b3

Browse files
guillaume-dequenneandrea-guarino-sonarsource
guillaume-dequenne
authored and
andrea-guarino-sonarsource
committed
SONARPY-848 Add method to remove modules from project symbol table (#917)
1 parent ae9fcb8 commit 13097b3

File tree

7 files changed

+192
-6
lines changed

7 files changed

+192
-6
lines changed

‎python-frontend/src/main/java/org/sonar/python/semantic/ProjectLevelSymbolTable.java

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,13 @@ private ProjectLevelSymbolTable(Map<String, Set<Symbol>> globalSymbolsByModuleNa
6363
this.globalSymbolsByModuleName = new HashMap<>(globalSymbolsByModuleName);
6464
}
6565

66+
public void removeModule(String packageName, PythonFile pythonFile) {
67+
String fullyQualifiedModuleName = SymbolUtils.fullyQualifiedModuleName(packageName, pythonFile.fileName());
68+
globalSymbolsByModuleName.remove(fullyQualifiedModuleName);
69+
// ensure globalSymbolsByFQN is re-computed
70+
this.globalSymbolsByFQN = null;
71+
}
72+
6673
public void addModule(FileInput fileInput, String packageName, PythonFile pythonFile) {
6774
SymbolTableBuilder symbolTableBuilder = new SymbolTableBuilder(packageName, pythonFile);
6875
String fullyQualifiedModuleName = SymbolUtils.fullyQualifiedModuleName(packageName, pythonFile.fileName());
@@ -84,10 +91,21 @@ public void addModule(FileInput fileInput, String packageName, PythonFile python
8491
}
8592
}
8693
globalSymbolsByModuleName.put(fullyQualifiedModuleName, globalSymbols);
94+
if (globalSymbolsByFQN != null) {
95+
// TODO: build globalSymbolsByFQN incrementally
96+
addModuleToGlobalSymbolsByFQN(globalSymbols);
97+
}
8798
DjangoViewsVisitor djangoViewsVisitor = new DjangoViewsVisitor();
8899
fileInput.accept(djangoViewsVisitor);
89100
}
90101

102+
private void addModuleToGlobalSymbolsByFQN(Set<Symbol> symbols) {
103+
Map<String, Symbol> moduleSymbolsByFQN = symbols.stream()
104+
.filter(symbol -> symbol.fullyQualifiedName() != null)
105+
.collect(Collectors.toMap(Symbol::fullyQualifiedName, Function.identity(), AmbiguousSymbolImpl::create));
106+
globalSymbolsByFQN.putAll(moduleSymbolsByFQN);
107+
}
108+
91109
private Map<String, Symbol> globalSymbolsByFQN() {
92110
if (globalSymbolsByFQN == null) {
93111
globalSymbolsByFQN = globalSymbolsByModuleName.values()

‎python-frontend/src/test/java/org/sonar/python/semantic/ProjectLevelSymbolTableTest.java

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -398,6 +398,39 @@ private static Set<Symbol> globalSymbols(FileInput fileInput, String packageName
398398
return projectLevelSymbolTable.getSymbolsFromModule(packageName.isEmpty() ? "mod" : packageName + ".mod");
399399
}
400400

401+
@Test
402+
public void test_remove_module() {
403+
FileInput tree = parseWithoutSymbols(
404+
"class A: pass"
405+
);
406+
ProjectLevelSymbolTable projectLevelSymbolTable = new ProjectLevelSymbolTable();
407+
projectLevelSymbolTable.addModule(tree, "", pythonFile("mod.py"));
408+
assertThat(projectLevelSymbolTable.getSymbolsFromModule("mod")).extracting(Symbol::name).containsExactlyInAnyOrder("A");
409+
projectLevelSymbolTable.removeModule("", pythonFile("mod.py"));
410+
assertThat(projectLevelSymbolTable.getSymbolsFromModule("mod")).isNull();
411+
}
412+
413+
@Test
414+
public void test_add_module_after_creation() {
415+
FileInput tree = parseWithoutSymbols(
416+
"class A: pass"
417+
);
418+
ProjectLevelSymbolTable projectLevelSymbolTable = new ProjectLevelSymbolTable();
419+
projectLevelSymbolTable.addModule(tree, "", pythonFile("mod.py"));
420+
assertThat(projectLevelSymbolTable.getSymbolsFromModule("mod")).extracting(Symbol::name).containsExactlyInAnyOrder("A");
421+
assertThat(projectLevelSymbolTable.getSymbolsFromModule("mod2")).isNull();
422+
assertThat(projectLevelSymbolTable.getSymbol("mod.A")).isNotNull();
423+
assertThat(projectLevelSymbolTable.getSymbol("mod2.B")).isNull();
424+
425+
tree = parseWithoutSymbols(
426+
"class B: pass"
427+
);
428+
projectLevelSymbolTable.addModule(tree, "", pythonFile("mod2.py"));
429+
assertThat(projectLevelSymbolTable.getSymbolsFromModule("mod")).extracting(Symbol::name).containsExactlyInAnyOrder("A");
430+
assertThat(projectLevelSymbolTable.getSymbolsFromModule("mod2")).extracting(Symbol::name).containsExactlyInAnyOrder("B");
431+
assertThat(projectLevelSymbolTable.getSymbol("mod2.B")).isNotNull();
432+
}
433+
401434
@Test
402435
public void global_symbols() {
403436
FileInput tree = parseWithoutSymbols(

‎sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonIndexer.java

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.sonar.plugins.python;
2121

2222
import com.sonar.sslr.api.AstNode;
23+
import java.io.File;
2324
import java.io.IOException;
2425
import java.net.URI;
2526
import java.util.HashMap;
@@ -44,9 +45,11 @@ public class PythonIndexer {
4445
private final Map<URI, String> packageNames = new HashMap<>();
4546
private final PythonParser parser = PythonParser.create();
4647
private final ProjectLevelSymbolTable projectLevelSymbolTable = new ProjectLevelSymbolTable();
48+
private File projectBaseDir;
4749

4850

4951
void buildOnce(SensorContext context, List<InputFile> files) {
52+
this.projectBaseDir = context.fileSystem().baseDir();
5053
LOG.debug("Input files for indexing: " + files);
5154
// computes "globalSymbolsByModuleName"
5255
long startTime = System.currentTimeMillis();
@@ -64,6 +67,22 @@ ProjectLevelSymbolTable projectLevelSymbolTable() {
6467
return projectLevelSymbolTable;
6568
}
6669

70+
void removeFile(InputFile inputFile) {
71+
String packageName = packageNames.get(inputFile.uri());
72+
packageNames.remove(inputFile.uri());
73+
PythonFile pythonFile = SonarQubePythonFile.create(inputFile);
74+
projectLevelSymbolTable.removeModule(packageName, pythonFile);
75+
}
76+
77+
void addFile(InputFile inputFile) throws IOException {
78+
AstNode astNode = parser.parse(inputFile.contents());
79+
FileInput astRoot = new PythonTreeMaker().fileInput(astNode);
80+
String packageName = pythonPackageName(inputFile.file(), projectBaseDir);
81+
packageNames.put(inputFile.uri(), packageName);
82+
PythonFile pythonFile = SonarQubePythonFile.create(inputFile);
83+
projectLevelSymbolTable.addModule(astRoot, packageName, pythonFile);
84+
}
85+
6786
private class GlobalSymbolsScanner extends Scanner {
6887

6988
private GlobalSymbolsScanner(SensorContext context) {
@@ -77,12 +96,7 @@ protected String name() {
7796

7897
@Override
7998
protected void scanFile(InputFile inputFile) throws IOException {
80-
AstNode astNode = parser.parse(inputFile.contents());
81-
FileInput astRoot = new PythonTreeMaker().fileInput(astNode);
82-
String packageName = pythonPackageName(inputFile.file(), context.fileSystem().baseDir());
83-
packageNames.put(inputFile.uri(), packageName);
84-
PythonFile pythonFile = SonarQubePythonFile.create(inputFile);
85-
projectLevelSymbolTable.addModule(astRoot, packageName, pythonFile);
99+
addFile(inputFile);
86100
}
87101

88102
@Override

‎sonar-python-plugin/src/test/java/org/sonar/plugins/python/PythonIndexerTest.java

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
/*
2+
* SonarQube Python Plugin
3+
* Copyright (C) 2011-2021 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the GNU Lesser General Public
8+
* License as published by the Free Software Foundation; either
9+
* version 3 of the License, or (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
* Lesser General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Lesser General Public License
17+
* along with this program; if not, write to the Free Software Foundation,
18+
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19+
*/
20+
package org.sonar.plugins.python;
21+
22+
import java.io.File;
23+
import java.io.IOException;
24+
import java.nio.charset.StandardCharsets;
25+
import java.nio.file.Files;
26+
import java.nio.file.Path;
27+
import java.util.Arrays;
28+
import java.util.List;
29+
import java.util.stream.Collectors;
30+
import org.junit.Before;
31+
import org.junit.Test;
32+
import org.sonar.api.batch.fs.InputFile;
33+
import org.sonar.api.batch.fs.internal.DefaultInputFile;
34+
import org.sonar.api.batch.fs.internal.TestInputFileBuilder;
35+
import org.sonar.api.batch.sensor.internal.SensorContextTester;
36+
import org.sonar.plugins.python.api.symbols.Symbol;
37+
import org.sonar.python.semantic.ProjectLevelSymbolTable;
38+
39+
import static org.assertj.core.api.Assertions.assertThat;
40+
41+
public class PythonIndexerTest {
42+
43+
private final File baseDir = new File("src/test/resources/org/sonar/plugins/python/indexer").getAbsoluteFile();
44+
private SensorContextTester context;
45+
private static Path workDir;
46+
47+
@Before
48+
public void init() throws IOException {
49+
context = SensorContextTester.create(baseDir);
50+
workDir = Files.createTempDirectory("workDir");
51+
context.fileSystem().setWorkDir(workDir);
52+
}
53+
54+
@Test
55+
public void test_indexer() {
56+
PythonIndexer pythonIndexer = new PythonIndexer();
57+
InputFile file1 = inputFile("main.py");
58+
InputFile file2 = inputFile("mod.py");
59+
pythonIndexer.buildOnce(context, Arrays.asList(file1, file2));
60+
ProjectLevelSymbolTable projectLevelSymbolTable = pythonIndexer.projectLevelSymbolTable();
61+
assertThat(projectLevelSymbolTable.getSymbolsFromModule("main")).hasSize(1);
62+
assertThat(projectLevelSymbolTable.getSymbolsFromModule("mod")).hasSize(1);
63+
Symbol modAddSymbol = projectLevelSymbolTable.getSymbol("mod.add");
64+
assertThat(modAddSymbol).isNotNull();
65+
assertThat(modAddSymbol.is(Symbol.Kind.FUNCTION)).isTrue();
66+
}
67+
68+
@Test
69+
public void test_indexer_removed_file() {
70+
PythonIndexer pythonIndexer = new PythonIndexer();
71+
InputFile file1 = inputFile("main.py");
72+
InputFile file2 = inputFile("mod.py");
73+
pythonIndexer.buildOnce(context, Arrays.asList(file1, file2));
74+
ProjectLevelSymbolTable projectLevelSymbolTable = pythonIndexer.projectLevelSymbolTable();
75+
76+
pythonIndexer.removeFile(file2);
77+
assertThat(projectLevelSymbolTable.getSymbolsFromModule("main")).hasSize(1);
78+
assertThat(projectLevelSymbolTable.getSymbolsFromModule("mod")).isNull();
79+
Symbol modAddSymbol = projectLevelSymbolTable.getSymbol("mod.add");
80+
assertThat(modAddSymbol).isNull();
81+
}
82+
83+
@Test
84+
public void test_indexer_added_file() throws IOException {
85+
PythonIndexer pythonIndexer = new PythonIndexer();
86+
InputFile file1 = inputFile("main.py");
87+
InputFile file2 = inputFile("mod.py");
88+
pythonIndexer.buildOnce(context, Arrays.asList(file1, file2));
89+
ProjectLevelSymbolTable projectLevelSymbolTable = pythonIndexer.projectLevelSymbolTable();
90+
91+
InputFile file3 = createInputFile("added.py");
92+
pythonIndexer.addFile(file3);
93+
assertThat(projectLevelSymbolTable.getSymbolsFromModule("main")).hasSize(1);
94+
assertThat(projectLevelSymbolTable.getSymbolsFromModule("added")).hasSize(1);
95+
Symbol newFuncSymbol = projectLevelSymbolTable.getSymbol("added.new_func");
96+
assertThat(newFuncSymbol).isNotNull();
97+
assertThat(newFuncSymbol.is(Symbol.Kind.FUNCTION)).isTrue();
98+
}
99+
100+
private InputFile inputFile(String name) {
101+
DefaultInputFile inputFile = createInputFile(name);
102+
context.fileSystem().add(inputFile);
103+
return inputFile;
104+
}
105+
106+
private DefaultInputFile createInputFile(String name) {
107+
return TestInputFileBuilder.create("moduleKey", name)
108+
.setModuleBaseDir(baseDir.toPath())
109+
.setCharset(StandardCharsets.UTF_8)
110+
.setType(InputFile.Type.MAIN)
111+
.setLanguage(Python.KEY)
112+
.initMetadata(TestUtils.fileContent(new File(baseDir, name), StandardCharsets.UTF_8))
113+
.build();
114+
}
115+
}

‎sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/added.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
def new_func(): ...

‎sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/main.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
from mod import add
2+
3+
x = add(1, 2, 3) # Noncompliant S930

‎sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/mod.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
def add(p1, p2):
2+
return p1 + p2

0 commit comments

Comments
 (0)