Skip to content

Commit c23e301

Browse files
guillaume-dequenneguillaume-dequenne
committed
SONARPY-2303 Stop running V1 type inference during the indexing phase
1 parent a8b7fc1 commit c23e301

File tree

21 files changed

+270
-63
lines changed

21 files changed

+270
-63
lines changed

‎python-frontend/src/main/java/org/sonar/plugins/python/api/PythonInputFileContext.java

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.sonar.api.SonarProduct;
2525
import org.sonar.plugins.python.api.caching.CacheContext;
2626
import org.sonar.plugins.python.api.symbols.Symbol;
27+
import org.sonar.python.semantic.ProjectLevelSymbolTable;
2728
import org.sonar.python.types.TypeShed;
2829

2930
public class PythonInputFileContext {
@@ -33,19 +34,23 @@ public class PythonInputFileContext {
3334
private final CacheContext cacheContext;
3435

3536
private final SonarProduct sonarProduct;
37+
private final ProjectLevelSymbolTable projectLevelSymbolTable;
3638

37-
public PythonInputFileContext(PythonFile pythonFile, @Nullable File workingDirectory, CacheContext cacheContext, SonarProduct sonarProduct) {
39+
public PythonInputFileContext(PythonFile pythonFile, @Nullable File workingDirectory, CacheContext cacheContext,
40+
SonarProduct sonarProduct, ProjectLevelSymbolTable projectLevelSymbolTable) {
3841
this.pythonFile = pythonFile;
3942
this.workingDirectory = workingDirectory;
4043
this.cacheContext = cacheContext;
4144
this.sonarProduct = sonarProduct;
45+
this.projectLevelSymbolTable = projectLevelSymbolTable;
4246
}
4347

44-
public PythonInputFileContext(PythonFile pythonFile, @Nullable File workingDirectory, CacheContext cacheContext) {
48+
public PythonInputFileContext(PythonFile pythonFile, @Nullable File workingDirectory, CacheContext cacheContext, ProjectLevelSymbolTable projectLevelSymbolTable) {
4549
this.pythonFile = pythonFile;
4650
this.workingDirectory = workingDirectory;
4751
this.cacheContext = cacheContext;
4852
this.sonarProduct = SonarProduct.SONARQUBE;
53+
this.projectLevelSymbolTable = projectLevelSymbolTable;
4954
}
5055

5156
public PythonFile pythonFile() {
@@ -59,7 +64,7 @@ public CacheContext cacheContext() {
5964

6065
@Beta
6166
public Collection<Symbol> stubFilesSymbols() {
62-
return TypeShed.stubFilesSymbols();
67+
return projectLevelSymbolTable.stubFilesSymbols();
6368
}
6469

6570
@CheckForNull

‎python-frontend/src/main/java/org/sonar/plugins/python/api/PythonVisitorContext.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public class PythonVisitorContext extends PythonInputFileContext {
4141
private final TypeChecker typeChecker;
4242

4343
public PythonVisitorContext(FileInput rootTree, PythonFile pythonFile, @Nullable File workingDirectory, String packageName) {
44-
super(pythonFile, workingDirectory, CacheContextImpl.dummyCache());
44+
super(pythonFile, workingDirectory, CacheContextImpl.dummyCache(), ProjectLevelSymbolTable.empty());
4545
this.rootTree = rootTree;
4646
this.parsingException = null;
4747
SymbolTableBuilder symbolTableBuilder = new SymbolTableBuilder(packageName, pythonFile);
@@ -54,7 +54,7 @@ public PythonVisitorContext(FileInput rootTree, PythonFile pythonFile, @Nullable
5454

5555
public PythonVisitorContext(FileInput rootTree, PythonFile pythonFile, @Nullable File workingDirectory, String packageName,
5656
ProjectLevelSymbolTable projectLevelSymbolTable, CacheContext cacheContext) {
57-
super(pythonFile, workingDirectory, cacheContext);
57+
super(pythonFile, workingDirectory, cacheContext, projectLevelSymbolTable);
5858
this.rootTree = rootTree;
5959
this.parsingException = null;
6060
new SymbolTableBuilder(packageName, pythonFile, projectLevelSymbolTable).visitFileInput(rootTree);
@@ -68,7 +68,7 @@ public PythonVisitorContext(FileInput rootTree, PythonFile pythonFile, @Nullable
6868

6969
public PythonVisitorContext(FileInput rootTree, PythonFile pythonFile, @Nullable File workingDirectory, String packageName,
7070
ProjectLevelSymbolTable projectLevelSymbolTable, CacheContext cacheContext, SonarProduct sonarProduct) {
71-
super(pythonFile, workingDirectory, cacheContext, sonarProduct);
71+
super(pythonFile, workingDirectory, cacheContext, sonarProduct, projectLevelSymbolTable);
7272
this.rootTree = rootTree;
7373
this.parsingException = null;
7474
new SymbolTableBuilder(packageName, pythonFile, projectLevelSymbolTable).visitFileInput(rootTree);
@@ -80,14 +80,14 @@ public PythonVisitorContext(FileInput rootTree, PythonFile pythonFile, @Nullable
8080
}
8181

8282
public PythonVisitorContext(PythonFile pythonFile, RecognitionException parsingException) {
83-
super(pythonFile, null, CacheContextImpl.dummyCache());
83+
super(pythonFile, null, CacheContextImpl.dummyCache(), ProjectLevelSymbolTable.empty());
8484
this.rootTree = null;
8585
this.parsingException = parsingException;
8686
this.typeChecker = new TypeChecker(new ProjectLevelTypeTable(ProjectLevelSymbolTable.empty()));
8787
}
8888

8989
public PythonVisitorContext(PythonFile pythonFile, RecognitionException parsingException, SonarProduct sonarProduct) {
90-
super(pythonFile, null, CacheContextImpl.dummyCache(), sonarProduct);
90+
super(pythonFile, null, CacheContextImpl.dummyCache(), sonarProduct, ProjectLevelSymbolTable.empty());
9191
this.rootTree = null;
9292
this.parsingException = parsingException;
9393
this.typeChecker = new TypeChecker(new ProjectLevelTypeTable(ProjectLevelSymbolTable.empty()));

‎python-frontend/src/main/java/org/sonar/python/SubscriptionVisitor.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ public PythonFile pythonFile() {
166166

167167
@Override
168168
public Collection<Symbol> stubFilesSymbols() {
169-
return TypeShed.stubFilesSymbols();
169+
return pythonVisitorContext.stubFilesSymbols();
170170
}
171171

172172
@Override

‎python-frontend/src/main/java/org/sonar/python/semantic/ProjectLevelSymbolTable.java

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.sonar.python.index.Descriptor;
3838
import org.sonar.python.index.DescriptorUtils;
3939
import org.sonar.python.semantic.v2.BasicTypeTable;
40+
import org.sonar.python.semantic.v2.ProjectLevelTypeTable;
4041
import org.sonar.python.semantic.v2.SymbolTableBuilderV2;
4142
import org.sonar.python.semantic.v2.TypeInferenceV2;
4243
import org.sonar.python.semantic.v2.UsageV2;
@@ -59,6 +60,7 @@ public class ProjectLevelSymbolTable {
5960
private final Map<String, Set<String>> importsByModule = new HashMap<>();
6061
private final Set<String> projectBasePackages = new HashSet<>();
6162
private TypeShedDescriptorsProvider typeShedDescriptorsProvider = null;
63+
private Set<Symbol> cachedSymbols = null;
6264

6365
public static ProjectLevelSymbolTable empty() {
6466
return new ProjectLevelSymbolTable();
@@ -87,12 +89,9 @@ public void removeModule(String packageName, String fileName) {
8789
}
8890

8991
public void addModule(FileInput fileInput, String packageName, PythonFile pythonFile) {
90-
SymbolTableBuilder symbolTableBuilder = new SymbolTableBuilder(packageName, pythonFile);
91-
fileInput.accept(symbolTableBuilder);
92-
9392
String fullyQualifiedModuleName = SymbolUtils.fullyQualifiedModuleName(packageName, pythonFile.fileName());
9493
var symbolTable = new SymbolTableBuilderV2(fileInput).build();
95-
var typeInferenceV2 = new TypeInferenceV2(new BasicTypeTable(), pythonFile, symbolTable, packageName);
94+
var typeInferenceV2 = new TypeInferenceV2(new BasicTypeTable(new ProjectLevelTypeTable(this)), pythonFile, symbolTable, packageName);
9695
var typesBySymbol = typeInferenceV2.inferTypes(fileInput);
9796
importsByModule.put(fullyQualifiedModuleName, typeInferenceV2.importedModulesFQN());
9897
var moduleDescriptors = typesBySymbol.entrySet()
@@ -201,6 +200,26 @@ public TypeShedDescriptorsProvider typeShedDescriptorsProvider() {
201200
return typeShedDescriptorsProvider;
202201
}
203202

203+
/**
204+
* Returns stub symbols to be used by SonarSecurity.
205+
* Ambiguous symbols that only contain class symbols are disambiguated with latest Python version.
206+
*/
207+
public Collection<Symbol> stubFilesSymbols() {
208+
if (cachedSymbols != null) {
209+
return cachedSymbols;
210+
}
211+
Map<String, Symbol> symbolsByFqn = new HashMap<>();
212+
cachedSymbols = new HashSet<>();
213+
for (Descriptor descriptor : typeShedDescriptorsProvider.stubFilesDescriptors()) {
214+
if (descriptor.fullyQualifiedName() != null) {
215+
Symbol symbol = symbolsByFqn.computeIfAbsent(descriptor.fullyQualifiedName(), k ->
216+
DescriptorUtils.symbolFromDescriptor(descriptor, this, null, new HashMap<>(), new HashMap<>()));
217+
cachedSymbols.add(symbol);
218+
}
219+
}
220+
return cachedSymbols;
221+
}
222+
204223
private class DjangoViewsVisitor extends BaseTreeVisitor {
205224

206225
String fullyQualifiedModuleName;
@@ -226,8 +245,8 @@ public void visitCallExpression(CallExpression callExpression) {
226245
}
227246
}
228247

229-
private static boolean isCallRegisteringDjangoView(CallExpression callExpression) {
230-
TypeChecker typeChecker = new TypeChecker(new BasicTypeTable());
248+
private boolean isCallRegisteringDjangoView(CallExpression callExpression) {
249+
TypeChecker typeChecker = new TypeChecker(new BasicTypeTable(new ProjectLevelTypeTable(ProjectLevelSymbolTable.this)));
231250
TriBool isConfPathCall = typeChecker.typeCheckBuilder().isTypeWithName("django.urls.conf.path").check(callExpression.callee().typeV2());
232251
TriBool isPathCall = typeChecker.typeCheckBuilder().isTypeWithName("django.urls.path").check(callExpression.callee().typeV2());
233252
return isConfPathCall.equals(TriBool.TRUE) || isPathCall.equals(TriBool.TRUE);

‎python-frontend/src/main/java/org/sonar/python/semantic/v2/BasicTypeTable.java

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,28 +21,41 @@
2121
import org.sonar.python.types.v2.UnknownType;
2222

2323
public class BasicTypeTable implements TypeTable {
24+
25+
private final ProjectLevelTypeTable projectLevelTypeTable;
26+
27+
public BasicTypeTable(ProjectLevelTypeTable fakeTypeTable) {
28+
// used to trigger typeshed resolutions
29+
this.projectLevelTypeTable = fakeTypeTable;
30+
}
31+
2432
@Override
2533
public PythonType getBuiltinsModule() {
34+
this.projectLevelTypeTable.getBuiltinsModule();
2635
return new UnknownType.UnresolvedImportType("");
2736
}
2837

2938
@Override
3039
public PythonType getType(String typeFqn) {
40+
this.projectLevelTypeTable.getType(typeFqn);
3141
return new UnknownType.UnresolvedImportType(typeFqn);
3242
}
3343

3444
@Override
3545
public PythonType getType(String... typeFqnParts) {
46+
this.projectLevelTypeTable.getType(typeFqnParts);
3647
return new UnknownType.UnresolvedImportType(String.join(".", typeFqnParts));
3748
}
3849

3950
@Override
4051
public PythonType getType(List<String> typeFqnParts) {
52+
this.projectLevelTypeTable.getType(typeFqnParts);
4153
return new UnknownType.UnresolvedImportType(String.join(".", typeFqnParts));
4254
}
4355

4456
@Override
4557
public PythonType getModuleType(List<String> typeFqnParts) {
58+
this.projectLevelTypeTable.getModuleType(typeFqnParts);
4659
return new UnknownType.UnresolvedImportType(String.join(".", typeFqnParts));
4760
}
4861
}

‎python-frontend/src/main/java/org/sonar/python/semantic/v2/typeshed/TypeShedDescriptorsProvider.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,14 @@
1818

1919
import java.io.IOException;
2020
import java.io.InputStream;
21+
import java.util.ArrayList;
2122
import java.util.Collections;
2223
import java.util.HashMap;
24+
import java.util.List;
2325
import java.util.Map;
2426
import java.util.Optional;
2527
import java.util.Set;
28+
import java.util.TreeMap;
2629
import java.util.function.Predicate;
2730
import java.util.stream.Stream;
2831
import javax.annotation.CheckForNull;
@@ -91,6 +94,10 @@ public Map<String, Descriptor> descriptorsForModule(String moduleName) {
9194
return cachedDescriptors.computeIfAbsent(moduleName, this::searchTypeShedForModule);
9295
}
9396

97+
public Set<String> stubModules() {
98+
return cachedDescriptors.keySet();
99+
}
100+
94101
//================================================================================
95102
// Private methods
96103
//================================================================================
@@ -128,4 +135,9 @@ static ModuleSymbol deserializedModule(String moduleName, InputStream resource)
128135
}
129136
}
130137

138+
public List<Descriptor> stubFilesDescriptors() {
139+
List<Descriptor> descriptors = new ArrayList<>(new TreeMap<>(builtinDescriptors()).values());
140+
new TreeMap<>(cachedDescriptors).values().forEach(entry -> descriptors.addAll(new TreeMap<>(entry).values()));
141+
return descriptors;
142+
}
131143
}

‎python-frontend/src/test/java/org/sonar/python/PythonVisitorCheckTest.java

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
import java.util.Collections;
2525
import java.util.List;
2626
import org.junit.jupiter.api.Test;
27-
import org.mockito.Mockito;
2827
import org.sonar.plugins.python.api.IssueLocation;
2928
import org.sonar.plugins.python.api.PythonCheck;
3029
import org.sonar.plugins.python.api.PythonCheck.PreciseIssue;
@@ -41,7 +40,6 @@
4140
import org.sonar.plugins.python.api.tree.Tree;
4241
import org.sonar.python.caching.CacheContextImpl;
4342
import org.sonar.python.semantic.ProjectLevelSymbolTable;
44-
import org.sonar.python.types.TypeShed;
4543

4644
import static org.assertj.core.api.Assertions.assertThat;
4745
import static org.mockito.Mockito.mock;
@@ -158,16 +156,16 @@ public void initialize(SubscriptionCheck.Context context) {
158156

159157
@Test
160158
void stubFilesSymbols() {
161-
PythonVisitorContext context = TestPythonVisitorRunner.createContext(FILE);
162-
159+
ProjectLevelSymbolTable projectLevelSymbolTable = ProjectLevelSymbolTable.empty();
160+
PythonVisitorContext context = TestPythonVisitorRunner.createContext(FILE, null, "my_package", projectLevelSymbolTable, CacheContextImpl.dummyCache());
163161
SymbolsRecordingCheck check = new SymbolsRecordingCheck();
162+
164163
check.scanFile(context);
165164
SubscriptionVisitor.analyze(Collections.singletonList(check), context);
166165

167-
assertThat(check.symbols).isEqualTo(TypeShed.stubFilesSymbols());
168-
169-
PythonInputFileContext inputFileContext = new PythonInputFileContext(mock(PythonFile.class), null, CacheContextImpl.dummyCache());
170-
assertThat(inputFileContext.stubFilesSymbols()).isEqualTo(TypeShed.stubFilesSymbols());
166+
PythonInputFileContext inputFileContext = new PythonInputFileContext(mock(PythonFile.class), null, CacheContextImpl.dummyCache(), projectLevelSymbolTable);
167+
assertThat(check.symbols).isEqualTo(inputFileContext.stubFilesSymbols());
168+
assertThat(inputFileContext.stubFilesSymbols()).isEqualTo(projectLevelSymbolTable.stubFilesSymbols());
171169
}
172170

173171
private static class TestPythonCheck extends PythonVisitorCheck {

‎python-frontend/src/test/java/org/sonar/python/semantic/ProjectLevelSymbolTableTest.java

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@
3434
import org.sonar.plugins.python.api.symbols.Symbol;
3535
import org.sonar.plugins.python.api.symbols.Usage;
3636
import org.sonar.plugins.python.api.tree.CallExpression;
37-
import org.sonar.plugins.python.api.tree.ClassDef;
3837
import org.sonar.plugins.python.api.tree.FileInput;
3938
import org.sonar.plugins.python.api.tree.FunctionDef;
4039
import org.sonar.plugins.python.api.tree.ImportFrom;
@@ -46,7 +45,6 @@
4645
import org.sonar.python.index.Descriptor;
4746
import org.sonar.python.index.FunctionDescriptor;
4847
import org.sonar.python.index.VariableDescriptor;
49-
import org.sonar.python.tree.TreeUtils;
5048
import org.sonar.python.types.DeclaredType;
5149
import org.sonar.python.types.InferredTypes;
5250
import org.sonar.python.types.TypeShed;
@@ -515,6 +513,17 @@ void test_imported_modules() {
515513
);
516514
}
517515

516+
@Test
517+
void importedStubModules() {
518+
FileInput tree = parseWithoutSymbols("""
519+
import math
520+
import os
521+
""");
522+
ProjectLevelSymbolTable projectLevelSymbolTable = ProjectLevelSymbolTable.empty();
523+
projectLevelSymbolTable.addModule(tree, "my_package", pythonFile("mod.py"));
524+
assertThat(projectLevelSymbolTable.typeShedDescriptorsProvider().stubModules()).containsExactlyInAnyOrder("math", "os");
525+
}
526+
518527
@Test
519528
void global_symbols() {
520529
FileInput tree = parseWithoutSymbols(
@@ -732,11 +741,12 @@ void class_having_another_class_with_same_name_should_not_trigger_error() {
732741
"class A:",
733742
" class B(B): pass"
734743
);
735-
globalSymbols(fileInput, "mod");
736-
ClassDef outerClassDef = (ClassDef) fileInput.statements().statements().get(1);
737-
ClassDef innerClassDef = (ClassDef) outerClassDef.body().statements().get(0);
738-
// SONARPY-1350: Parent should be external.B
739-
assertThat(TreeUtils.getParentClassesFQN(innerClassDef)).containsExactly("mod.mod.A.B");
744+
Set<Symbol> symbols = globalSymbols(fileInput, "mod");
745+
// SONARPY-1829: Parent should be external.B
746+
ClassSymbol outerClassSymbol = ((ClassSymbol) symbols.stream().findFirst().get());
747+
ClassSymbol innerClassSymbol = (ClassSymbol) outerClassSymbol.resolveMember("B").get();
748+
assertThat(innerClassSymbol.superClasses()).isEmpty();
749+
assertThat(innerClassSymbol.hasUnresolvedTypeHierarchy()).isTrue();
740750
}
741751

742752
@Test

‎python-frontend/src/test/java/org/sonar/python/semantic/v2/BasicTypeTableTest.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,15 @@
1919
import java.util.List;
2020
import java.util.Optional;
2121
import org.junit.jupiter.api.Test;
22+
import org.sonar.python.semantic.ProjectLevelSymbolTable;
2223
import org.sonar.python.types.v2.PythonType;
2324
import org.sonar.python.types.v2.UnknownType;
2425

2526
import static org.assertj.core.api.Assertions.assertThat;
2627

2728
class BasicTypeTableTest {
2829

29-
private final BasicTypeTable basicTypeTable = new BasicTypeTable();
30+
private final BasicTypeTable basicTypeTable = new BasicTypeTable(new ProjectLevelTypeTable(ProjectLevelSymbolTable.empty()));
3031

3132
@Test
3233
void testGetBuiltinsModule() {

‎python-frontend/src/test/java/org/sonar/python/semantic/v2/TypeInferenceV2Test.java

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2972,12 +2972,10 @@ void convertTypeshedModuleWithAliases() {
29722972
PythonType concurrency = symbolsModuleTypeProvider.convertModuleType(List.of("fastapi", "concurrency"), builtinModule);
29732973
assertThat(concurrency.resolveMember("iterate_in_threadpool")).containsInstanceOf(FunctionType.class);
29742974

2975-
List<Symbol> fileResponseSymbols = empty.typeShedDescriptorsProvider()
2976-
.stubFilesSymbols(empty).stream().filter(s -> "fastapi.responses.FileResponse".equals(s.fullyQualifiedName())).toList();
2975+
List<Symbol> fileResponseSymbols = empty.stubFilesSymbols().stream().filter(s -> "fastapi.responses.FileResponse".equals(s.fullyQualifiedName())).toList();
29772976
assertThat(fileResponseSymbols).hasSize(1);
29782977
assertThat(fileResponseSymbols.get(0).kind()).isEqualTo(Symbol.Kind.CLASS);
2979-
List<Symbol> runInThreadPoolSymbols = empty.typeShedDescriptorsProvider()
2980-
.stubFilesSymbols(empty).stream().filter(s -> "fastapi.concurrency.run_in_threadpool".equals(s.fullyQualifiedName())).toList();
2978+
List<Symbol> runInThreadPoolSymbols = empty.stubFilesSymbols().stream().filter(s -> "fastapi.concurrency.run_in_threadpool".equals(s.fullyQualifiedName())).toList();
29812979
assertThat(runInThreadPoolSymbols).hasSize(1);
29822980
assertThat(runInThreadPoolSymbols.get(0).kind()).isEqualTo(Symbol.Kind.FUNCTION);
29832981
}

0 commit comments

Comments
 (0)