Skip to content

Commit 3326647

Browse files
maksim-grebeniuk-sonarsourceguillaume-dequenne
maksim-grebeniuk-sonarsource
authored and
guillaume-dequenne
committed
SONARPY-1798 Try to resolve built-in types for names which have no symbol (#1774)
1 parent 9776160 commit 3326647

File tree

9 files changed

+115
-46
lines changed

9 files changed

+115
-46
lines changed

‎python-checks/src/test/resources/checks/nonCallableCalled.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,10 @@ def call_noncallable(p):
3131
set_literal() # Noncompliant
3232

3333
set_var = set()
34-
set_var() # FN
34+
set_var() # Noncompliant
3535

3636
frozenset_var = frozenset()
37-
frozenset_var() # FN
37+
frozenset_var() # Noncompliant
3838

3939
if p:
4040
x = 42
@@ -206,7 +206,7 @@ def assigning_global(my_func):
206206
some_global_func = my_func
207207

208208
def calling_global_func():
209-
some_global_func() # OK
209+
some_global_func() # OK
210210

211211

212212
some_nonlocal_var = 42

‎python-frontend/src/main/java/org/sonar/plugins/python/api/PythonVisitorContext.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ public PythonVisitorContext(FileInput rootTree, PythonFile pythonFile, @Nullable
6060
new SymbolTableBuilder(packageName, pythonFile, projectLevelSymbolTable).visitFileInput(rootTree);
6161
SymbolTableBuilderV2 symbolTableBuilderV2 = new SymbolTableBuilderV2();
6262
symbolTableBuilderV2.visitFileInput(rootTree);
63-
rootTree.accept(new TypeInferenceV2(new ProjectLevelTypeTable(ProjectLevelSymbolTable.empty())));
63+
rootTree.accept(new TypeInferenceV2(new ProjectLevelTypeTable(projectLevelSymbolTable)));
6464
}
6565

6666
public PythonVisitorContext(FileInput rootTree, PythonFile pythonFile, @Nullable File workingDirectory, String packageName,

‎python-frontend/src/main/java/org/sonar/python/semantic/v2/ProjectLevelTypeTable.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,8 @@ public class ProjectLevelTypeTable {
3131
private final ModuleType rootModule;
3232

3333
public ProjectLevelTypeTable(ProjectLevelSymbolTable projectLevelSymbolTable) {
34-
this.rootModule = new ModuleType(null);
3534
this.symbolsModuleTypeProvider = new SymbolsModuleTypeProvider(projectLevelSymbolTable);
36-
this.symbolsModuleTypeProvider.createBuiltinModule(rootModule);
35+
this.rootModule = this.symbolsModuleTypeProvider.createBuiltinModule();
3736
}
3837

3938
public ModuleType getModule(String... moduleName) {

‎python-frontend/src/main/java/org/sonar/python/semantic/v2/SymbolV2.java

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,6 @@ void addUsage(Name name, UsageV2.Kind kind) {
3737
if (name instanceof NameImpl ni) {
3838
ni.symbolV2(this);
3939
}
40-
/* if (tree.is(Tree.Kind.NAME)) {
41-
((NameImpl) tree).setSymbol(this);
42-
((NameImpl) tree).setUsage(usage);
43-
}*/
4440
}
4541

4642
boolean hasSingleBindingUsage() {

‎python-frontend/src/main/java/org/sonar/python/semantic/v2/SymbolsModuleTypeProvider.java

Lines changed: 34 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import java.util.Set;
2828
import java.util.function.Predicate;
2929
import java.util.stream.Collectors;
30+
import javax.annotation.Nullable;
3031
import org.sonar.plugins.python.api.symbols.AmbiguousSymbol;
3132
import org.sonar.plugins.python.api.symbols.ClassSymbol;
3233
import org.sonar.plugins.python.api.symbols.FunctionSymbol;
@@ -37,6 +38,7 @@
3738
import org.sonar.python.types.v2.FunctionType;
3839
import org.sonar.python.types.v2.Member;
3940
import org.sonar.python.types.v2.ModuleType;
41+
import org.sonar.python.types.v2.ParameterV2;
4042
import org.sonar.python.types.v2.PythonType;
4143
import org.sonar.python.types.v2.UnionType;
4244

@@ -47,9 +49,8 @@ public SymbolsModuleTypeProvider(ProjectLevelSymbolTable projectLevelSymbolTable
4749
this.projectLevelSymbolTable = projectLevelSymbolTable;
4850
}
4951

50-
public void createBuiltinModule(ModuleType parent) {
51-
var name = "builtins";
52-
createModuleFromSymbols(name, parent, TypeShed.builtinSymbols().values());
52+
public ModuleType createBuiltinModule() {
53+
return createModuleFromSymbols(null, null, TypeShed.builtinSymbols().values());
5354
}
5455

5556
public ModuleType createModuleType(List<String> moduleFqn, ModuleType parent) {
@@ -81,7 +82,7 @@ private static ModuleType createEmptyModule(String moduleName, ModuleType parent
8182
return emptyModule;
8283
}
8384

84-
private ModuleType createModuleFromSymbols(String name, ModuleType parent, Collection<Symbol> symbols) {
85+
private ModuleType createModuleFromSymbols(@Nullable String name, @Nullable ModuleType parent, Collection<Symbol> symbols) {
8586
var members = new HashMap<String, PythonType>();
8687
symbols.forEach(symbol -> {
8788
var type = convertToType(symbol, new HashMap<>());
@@ -90,7 +91,10 @@ private ModuleType createModuleFromSymbols(String name, ModuleType parent, Colle
9091
var module = new ModuleType(name, parent);
9192
module.members().putAll(members);
9293

93-
parent.members().put(name, module);
94+
Optional.ofNullable(parent)
95+
.map(ModuleType::members)
96+
.ifPresent(m -> m.put(name, module));
97+
9498
return module;
9599
}
96100

@@ -103,27 +107,45 @@ private static PythonType convertToFunctionType(FunctionSymbol symbol, Map<Symbo
103107
if (createdTypesBySymbol.containsKey(symbol)) {
104108
return createdTypesBySymbol.get(symbol);
105109
}
110+
111+
var parameters = symbol.parameters()
112+
.stream()
113+
.map(SymbolsModuleTypeProvider::convertParameter)
114+
.toList();
115+
106116
FunctionTypeBuilder functionTypeBuilder =
107117
new FunctionTypeBuilder(symbol.name())
108-
.withAttributes(List.of())
109-
.withParameters(List.of())
118+
.withAttributes(List.of())
119+
.withParameters(parameters)
110120
.withReturnType(PythonType.UNKNOWN)
111-
.withAsynchronous(false)
112-
.withHasDecorators(false)
113-
.withInstanceMethod(false)
114-
.withHasVariadicParameter(false);
121+
.withAsynchronous(symbol.isAsynchronous())
122+
.withHasDecorators(symbol.hasDecorators())
123+
.withInstanceMethod(symbol.isInstanceMethod())
124+
.withHasVariadicParameter(symbol.hasVariadicParameter());
115125
FunctionType functionType = functionTypeBuilder.build();
116126
createdTypesBySymbol.put(symbol, functionType);
117127
return functionType;
118128
}
119129

130+
private static ParameterV2 convertParameter(FunctionSymbol.Parameter parameter) {
131+
return new ParameterV2(parameter.name(),
132+
PythonType.UNKNOWN,
133+
parameter.hasDefaultValue(),
134+
parameter.isKeywordOnly(),
135+
parameter.isPositionalOnly(),
136+
parameter.isKeywordVariadic(),
137+
parameter.isPositionalVariadic(),
138+
null);
139+
}
140+
120141
private PythonType convertToClassType(ClassSymbol symbol, Map<Symbol, PythonType> createdTypesBySymbol) {
121142
if (createdTypesBySymbol.containsKey(symbol)) {
122143
return createdTypesBySymbol.get(symbol);
123144
}
124145
ClassType classType = new ClassType(symbol.name());
125146
createdTypesBySymbol.put(symbol, classType);
126-
Set<Member> members = symbol.declaredMembers().stream().map(m -> new Member(m.name(), convertToType(m, createdTypesBySymbol))).collect(Collectors.toSet());
147+
Set<Member> members =
148+
symbol.declaredMembers().stream().map(m -> new Member(m.name(), convertToType(m, createdTypesBySymbol))).collect(Collectors.toSet());
127149
classType.members().addAll(members);
128150
List<PythonType> superClasses = symbol.superClasses().stream().map(s -> convertToType(s, createdTypesBySymbol)).toList();
129151
classType.superClasses().addAll(superClasses);

‎python-frontend/src/main/java/org/sonar/python/semantic/v2/TypeInferenceV2.java

Lines changed: 30 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,6 @@ public TypeInferenceV2(ProjectLevelTypeTable projectLevelTypeTable) {
7575
this.projectLevelTypeTable = projectLevelTypeTable;
7676
}
7777

78-
private static final String BUILTINS = "builtins";
79-
8078
@Override
8179
public void visitFileInput(FileInput fileInput) {
8280
var type = new ModuleType("somehow get its name");
@@ -85,7 +83,7 @@ public void visitFileInput(FileInput fileInput) {
8583

8684
@Override
8785
public void visitStringLiteral(StringLiteral stringLiteral) {
88-
ModuleType builtins = this.projectLevelTypeTable.getModule(BUILTINS);
86+
ModuleType builtins = this.projectLevelTypeTable.getModule();
8987
// TODO: multiple object types to represent str instance?
9088
PythonType strType = builtins.resolveMember("str").orElse(PythonType.UNKNOWN);
9189
((StringLiteralImpl) stringLiteral).typeV2(new ObjectType(strType, new ArrayList<>(), new ArrayList<>()));
@@ -99,30 +97,30 @@ public void visitTuple(Tuple tuple) {
9997
if (contentTypes.size() == 1 && !contentTypes.get(0).equals(PythonType.UNKNOWN)) {
10098
attributes = contentTypes;
10199
}
102-
ModuleType builtins = this.projectLevelTypeTable.getModule(BUILTINS);
100+
ModuleType builtins = this.projectLevelTypeTable.getModule();
103101
PythonType tupleType = builtins.resolveMember("tuple").orElse(PythonType.UNKNOWN);
104102
((TupleImpl) tuple).typeV2(new ObjectType(tupleType, attributes, new ArrayList<>()));
105103
}
106104

107105
@Override
108106
public void visitDictionaryLiteral(DictionaryLiteral dictionaryLiteral) {
109107
super.visitDictionaryLiteral(dictionaryLiteral);
110-
ModuleType builtins = this.projectLevelTypeTable.getModule(BUILTINS);
108+
ModuleType builtins = this.projectLevelTypeTable.getModule();
111109
PythonType dictType = builtins.resolveMember("dict").orElse(PythonType.UNKNOWN);
112110
((DictionaryLiteralImpl) dictionaryLiteral).typeV2(new ObjectType(dictType, new ArrayList<>(), new ArrayList<>()));
113111
}
114112

115113
@Override
116114
public void visitSetLiteral(SetLiteral setLiteral) {
117115
super.visitSetLiteral(setLiteral);
118-
ModuleType builtins = this.projectLevelTypeTable.getModule(BUILTINS);
116+
ModuleType builtins = this.projectLevelTypeTable.getModule();
119117
PythonType setType = builtins.resolveMember("set").orElse(PythonType.UNKNOWN);
120118
((SetLiteralImpl) setLiteral).typeV2(new ObjectType(setType, new ArrayList<>(), new ArrayList<>()));
121119
}
122120

123121
@Override
124122
public void visitNumericLiteral(NumericLiteral numericLiteral) {
125-
ModuleType builtins = this.projectLevelTypeTable.getModule(BUILTINS);
123+
ModuleType builtins = this.projectLevelTypeTable.getModule();
126124
InferredType type = numericLiteral.type();
127125
String memberName = ((RuntimeType) type).getTypeClass().fullyQualifiedName();
128126
if (memberName != null) {
@@ -133,15 +131,15 @@ public void visitNumericLiteral(NumericLiteral numericLiteral) {
133131

134132
@Override
135133
public void visitNone(NoneExpression noneExpression) {
136-
ModuleType builtins = this.projectLevelTypeTable.getModule(BUILTINS);
134+
ModuleType builtins = this.projectLevelTypeTable.getModule();
137135
// TODO: multiple object types to represent str instance?
138136
PythonType noneType = builtins.resolveMember("NoneType").orElse(PythonType.UNKNOWN);
139137
((NoneExpressionImpl) noneExpression).typeV2(new ObjectType(noneType, new ArrayList<>(), new ArrayList<>()));
140138
}
141139

142140
@Override
143141
public void visitListLiteral(ListLiteral listLiteral) {
144-
ModuleType builtins = this.projectLevelTypeTable.getModule(BUILTINS);
142+
ModuleType builtins = this.projectLevelTypeTable.getModule();
145143
scan(listLiteral.elements());
146144
List<PythonType> pythonTypes = listLiteral.elements().expressions().stream().map(Expression::typeV2).distinct().toList();
147145
// TODO: cleanly reduce attributes
@@ -312,29 +310,40 @@ public void visitQualifiedExpression(QualifiedExpression qualifiedExpression) {
312310
}
313311
}
314312

313+
315314
@Override
316315
public void visitName(Name name) {
317316
SymbolV2 symbolV2 = name.symbolV2();
318317
if (symbolV2 == null) {
318+
// This part could be affected by SONARPY-1802
319+
projectLevelTypeTable.getModule().resolveMember(name.name())
320+
.ifPresent(type -> setTypeToName(name, type));
319321
return;
320322
}
321-
List<PythonType> types = new ArrayList<>();
322-
for (UsageV2 usage : symbolV2.usages()) {
323+
324+
var bindingUsages = new ArrayList<UsageV2>();
325+
for (var usage : symbolV2.usages()) {
323326
if (usage.kind().equals(UsageV2.Kind.GLOBAL_DECLARATION)) {
324327
// Don't infer type for global variables
325328
return;
326329
}
327-
Optional.of(usage)
328-
.filter(UsageV2::isBindingUsage)
329-
.map(UsageV2::tree)
330-
.filter(Expression.class::isInstance)
331-
.map(Expression.class::cast)
332-
.map(Expression::typeV2)
333-
.ifPresent(types::add);
334-
}
335-
if (types.size() == 1) {
336-
setTypeToName(name, types.get(0));
330+
if (usage.isBindingUsage()) {
331+
bindingUsages.add(usage);
332+
}
333+
if (bindingUsages.size() > 1) {
334+
// no need to iterate over usages if there is more than one binding usage
335+
return;
336+
}
337337
}
338+
339+
bindingUsages.stream()
340+
.findFirst()
341+
.filter(UsageV2::isBindingUsage)
342+
.map(UsageV2::tree)
343+
.filter(Expression.class::isInstance)
344+
.map(Expression.class::cast)
345+
.map(Expression::typeV2)
346+
.ifPresent(type -> setTypeToName(name, type));
338347
}
339348

340349
private PythonType currentType() {

‎python-frontend/src/main/java/org/sonar/python/semantic/v2/UsageV2.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ public boolean isBindingUsage() {
2828
return kind() != UsageV2.Kind.OTHER && kind() != UsageV2.Kind.GLOBAL_DECLARATION;
2929
}
3030

31-
enum Kind {
31+
public enum Kind {
3232
ASSIGNMENT_LHS,
3333
COMPOUND_ASSIGNMENT_LHS,
3434
IMPORT,

‎python-frontend/src/test/java/org/sonar/python/semantic/v2/TypeInferenceV2Test.java

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,10 @@
2828
import org.junit.jupiter.api.BeforeAll;
2929
import org.junit.jupiter.api.Test;
3030
import org.sonar.plugins.python.api.symbols.Symbol;
31+
import org.sonar.plugins.python.api.tree.AssignmentStatement;
3132
import org.sonar.plugins.python.api.tree.CallExpression;
3233
import org.sonar.plugins.python.api.tree.Expression;
34+
import org.sonar.plugins.python.api.tree.ExpressionStatement;
3335
import org.sonar.plugins.python.api.tree.FileInput;
3436
import org.sonar.plugins.python.api.tree.FunctionDef;
3537
import org.sonar.plugins.python.api.tree.ImportFrom;
@@ -228,6 +230,40 @@ def foo(a, b, c): ...
228230
assertThat(callExpression.callee().typeV2()).isInstanceOf(FunctionType.class);
229231
}
230232

233+
@Test
234+
void inferTypeForBuiltins() {
235+
FileInput root = inferTypes("""
236+
a = list
237+
""");
238+
239+
var assignmentStatement = (AssignmentStatement) root.statements().statements().get(0);
240+
var assignedType = assignmentStatement.assignedValue().typeV2();
241+
242+
assertThat(assignedType)
243+
.isNotNull()
244+
.isInstanceOf(ClassType.class);
245+
246+
assertThat(assignedType.resolveMember("append"))
247+
.isPresent()
248+
.get()
249+
.isInstanceOf(FunctionType.class);
250+
}
251+
252+
@Test
253+
void inferTypeForReassignedBuiltins() {
254+
FileInput root = inferTypes("""
255+
def foo():
256+
global list
257+
list = 42
258+
list = "hello"
259+
list
260+
""");
261+
262+
var functionDef = (FunctionDef) root.statements().statements().get(0);
263+
var expressionStatement = (ExpressionStatement) functionDef.body().statements().get(3);
264+
Assertions.assertThat(expressionStatement.expressions().get(0).typeV2()).isEqualTo(PythonType.UNKNOWN);
265+
}
266+
231267
private FileInput inferTypes(String lines) {
232268
return inferTypes(lines, new HashMap<>());
233269
}

‎python-frontend/src/test/java/org/sonar/python/types/v2/ClassTypeTest.java

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -157,8 +157,15 @@ void builtin_parent() {
157157
);
158158
ClassType classB = classTypes.get(1);
159159
assertThat(classB.superClasses()).hasSize(2);
160-
// FIXME: ensure builtin parent is resolved
161-
assertThat(classB.hasUnresolvedHierarchy()).isTrue();
160+
assertThat(classB.hasUnresolvedHierarchy()).isFalse();
161+
var baseExceptionType = classB.superClasses().get(1);
162+
assertThat(baseExceptionType)
163+
.isInstanceOf(ClassType.class)
164+
.extracting(PythonType::name)
165+
.isEqualTo("BaseException");
166+
167+
var baseExceptionClassType = (ClassType) baseExceptionType;
168+
assertThat(baseExceptionClassType.members()).hasSize(10);
162169
}
163170

164171
@Test

0 commit comments

Comments
 (0)