Skip to content

Commit b710d18

Browse files
maksim-grebeniuk-sonarsourceguillaume-dequenne
maksim-grebeniuk-sonarsource
authored and
guillaume-dequenne
committed
SONARPY-1760 Infer types of imported serialized module (#1765)
1 parent c59f45f commit b710d18

File tree

7 files changed

+498
-85
lines changed

7 files changed

+498
-85
lines changed

‎python-frontend/src/main/java/org/sonar/python/semantic/v2/ProjectLevelTypeTable.java

Lines changed: 22 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -19,87 +19,42 @@
1919
*/
2020
package org.sonar.python.semantic.v2;
2121

22-
import java.util.Collections;
23-
import java.util.HashMap;
2422
import java.util.List;
25-
import java.util.Map;
26-
import java.util.Set;
27-
import java.util.function.Function;
28-
import java.util.stream.Collectors;
29-
import org.sonar.plugins.python.api.symbols.AmbiguousSymbol;
30-
import org.sonar.plugins.python.api.symbols.ClassSymbol;
31-
import org.sonar.plugins.python.api.symbols.FunctionSymbol;
32-
import org.sonar.plugins.python.api.symbols.Symbol;
23+
import java.util.stream.IntStream;
3324
import org.sonar.python.semantic.ProjectLevelSymbolTable;
34-
import org.sonar.python.types.TypeShed;
35-
import org.sonar.python.types.v2.ClassType;
36-
import org.sonar.python.types.v2.FunctionType;
37-
import org.sonar.python.types.v2.Member;
3825
import org.sonar.python.types.v2.ModuleType;
39-
import org.sonar.python.types.v2.PythonType;
40-
import org.sonar.python.types.v2.UnionType;
4126

4227
public class ProjectLevelTypeTable {
4328

44-
private final ProjectLevelSymbolTable projectLevelSymbolTable;
45-
private final Map<String, PythonType> builtinTypes;
46-
private final Map<String, ModuleType> modules;
29+
private final SymbolsModuleTypeProvider symbolsModuleTypeProvider;
30+
private final ModuleType rootModule;
4731

4832
public ProjectLevelTypeTable(ProjectLevelSymbolTable projectLevelSymbolTable) {
49-
this.projectLevelSymbolTable = projectLevelSymbolTable;
50-
this.modules = new HashMap<>();
51-
Map<String, Symbol> builtinSymbols = TypeShed.builtinSymbols();
52-
this.builtinTypes = new HashMap<>();
53-
builtinSymbols.forEach((key, value) -> builtinTypes.put(key, convertToType(value)));
54-
modules.put("builtins", new ModuleType("builtins", builtinTypes));
33+
this.rootModule = new ModuleType(null);
34+
this.symbolsModuleTypeProvider = new SymbolsModuleTypeProvider(projectLevelSymbolTable);
35+
this.symbolsModuleTypeProvider.createBuiltinModule(rootModule);
5536
}
5637

57-
public ModuleType getModule(String moduleName) {
58-
if (modules.containsKey(moduleName)) {
59-
return modules.get(moduleName);
60-
}
61-
Set<Symbol> symbolsFromModule = projectLevelSymbolTable.getSymbolsFromModule(moduleName);
62-
Map<String, PythonType> children;
63-
if (symbolsFromModule != null) {
64-
children = symbolsFromModule.stream()
65-
.map(this::convertToType)
66-
.collect(Collectors.toMap(PythonType::name, Function.identity(), (a, b) -> PythonType.UNKNOWN));
67-
} else {
68-
// FIXME: what to do here?
69-
children = Collections.emptyMap();
70-
}
71-
72-
ModuleType moduleType = new ModuleType(moduleName, children);
73-
modules.put(moduleName, moduleType);
74-
return moduleType;
38+
public ModuleType getModule(String... moduleName) {
39+
return getModule(List.of(moduleName));
7540
}
7641

77-
private PythonType convertToType(Symbol symbol) {
78-
return switch (symbol.kind()) {
79-
case CLASS -> converToClassType((ClassSymbol) symbol);
80-
case FUNCTION -> convertToFunctionType((FunctionSymbol) symbol);
81-
case AMBIGUOUS -> convertToUnionType((AmbiguousSymbol) symbol);
82-
case OTHER -> converToObjectType(symbol);
83-
};
84-
}
42+
public ModuleType getModule(List<String> moduleNameParts) {
43+
var parent = rootModule;
44+
for (int i = 0; i < moduleNameParts.size(); i++) {
45+
var existing = parent.resolveMember(moduleNameParts.get(i));
8546

86-
private PythonType converToObjectType(Symbol symbol) {
87-
// What should we have here?
88-
return PythonType.UNKNOWN;
89-
}
47+
if (existing instanceof ModuleType existingModule) {
48+
parent = existingModule;
49+
continue;
50+
}
9051

91-
private PythonType convertToFunctionType(FunctionSymbol symbol) {
92-
return new FunctionType(symbol.name(), List.of(), List.of(), PythonType.UNKNOWN, false, false, false, false, null);
93-
}
94-
95-
private PythonType converToClassType(ClassSymbol symbol) {
96-
Set<Member> members = symbol.declaredMembers().stream().map(m -> new Member(m.name(), convertToType(m))).collect(Collectors.toSet());
97-
List<PythonType> superClasses = symbol.superClasses().stream().map(this::convertToType).toList();
98-
return new ClassType(symbol.name(), members, List.of(), superClasses);
99-
}
52+
var moduleFqn = IntStream.rangeClosed(0, i)
53+
.mapToObj(moduleNameParts::get)
54+
.toList();
10055

101-
private PythonType convertToUnionType(AmbiguousSymbol ambiguousSymbol) {
102-
List<PythonType> pythonTypes = ambiguousSymbol.alternatives().stream().map(this::convertToType).toList();
103-
return new UnionType(pythonTypes);
56+
parent = symbolsModuleTypeProvider.createModuleType(moduleFqn, parent);
57+
}
58+
return parent;
10459
}
10560
}

‎python-frontend/src/main/java/org/sonar/python/semantic/v2/SymbolsModuleTypeProvider.java

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
/*
2+
* SonarQube Python Plugin
3+
* Copyright (C) 2011-2024 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the GNU Lesser General Public
8+
* License as published by the Free Software Foundation; either
9+
* version 3 of the License, or (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
* Lesser General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Lesser General Public License
17+
* along with this program; if not, write to the Free Software Foundation,
18+
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19+
*/
20+
package org.sonar.python.semantic.v2;
21+
22+
import java.util.Collection;
23+
import java.util.HashMap;
24+
import java.util.List;
25+
import java.util.Map;
26+
import java.util.Optional;
27+
import java.util.Set;
28+
import java.util.function.Predicate;
29+
import java.util.stream.Collectors;
30+
import org.sonar.plugins.python.api.symbols.AmbiguousSymbol;
31+
import org.sonar.plugins.python.api.symbols.ClassSymbol;
32+
import org.sonar.plugins.python.api.symbols.FunctionSymbol;
33+
import org.sonar.plugins.python.api.symbols.Symbol;
34+
import org.sonar.python.semantic.ProjectLevelSymbolTable;
35+
import org.sonar.python.types.TypeShed;
36+
import org.sonar.python.types.v2.ClassType;
37+
import org.sonar.python.types.v2.FunctionType;
38+
import org.sonar.python.types.v2.Member;
39+
import org.sonar.python.types.v2.ModuleType;
40+
import org.sonar.python.types.v2.PythonType;
41+
import org.sonar.python.types.v2.UnionType;
42+
43+
public class SymbolsModuleTypeProvider {
44+
private final ProjectLevelSymbolTable projectLevelSymbolTable;
45+
46+
public SymbolsModuleTypeProvider(ProjectLevelSymbolTable projectLevelSymbolTable) {
47+
this.projectLevelSymbolTable = projectLevelSymbolTable;
48+
}
49+
50+
public void createBuiltinModule(ModuleType parent) {
51+
var name = "builtins";
52+
createModuleFromSymbols(name, parent, TypeShed.builtinSymbols().values());
53+
}
54+
55+
public ModuleType createModuleType(List<String> moduleFqn, ModuleType parent) {
56+
var moduleName = moduleFqn.get(moduleFqn.size() - 1);
57+
var moduleFqnString = getModuleFqnString(moduleFqn);
58+
return createModuleTypeFromProjectLevelSymbolTable(moduleName, moduleFqnString, parent)
59+
.or(() -> createModuleTypeFromTypeShed(moduleName, moduleFqnString, parent))
60+
.orElseGet(() -> createEmptyModule(moduleName, parent));
61+
}
62+
63+
private static String getModuleFqnString(List<String> moduleFqn) {
64+
return String.join(".", moduleFqn);
65+
}
66+
67+
private Optional<ModuleType> createModuleTypeFromProjectLevelSymbolTable(String moduleName, String moduleFqn, ModuleType parent) {
68+
return Optional.ofNullable(projectLevelSymbolTable.getSymbolsFromModule(moduleFqn))
69+
.map(projectModuleSymbols -> createModuleFromSymbols(moduleName, parent, projectModuleSymbols));
70+
}
71+
72+
private Optional<ModuleType> createModuleTypeFromTypeShed(String moduleName, String moduleFqn, ModuleType parent) {
73+
return Optional.ofNullable(TypeShed.symbolsForModule(moduleFqn))
74+
.filter(Predicate.not(Map::isEmpty))
75+
.map(typeShedModuleSymbols -> createModuleFromSymbols(moduleName, parent, typeShedModuleSymbols.values()));
76+
}
77+
78+
private static ModuleType createEmptyModule(String moduleName, ModuleType parent) {
79+
var emptyModule = new ModuleType(moduleName, parent);
80+
parent.members().put(moduleName, emptyModule);
81+
return emptyModule;
82+
}
83+
84+
private ModuleType createModuleFromSymbols(String name, ModuleType parent, Collection<Symbol> symbols) {
85+
var members = new HashMap<String, PythonType>();
86+
symbols.forEach(symbol -> {
87+
var type = convertToType(symbol);
88+
members.put(symbol.name(), type);
89+
});
90+
var module = new ModuleType(name, parent);
91+
module.members().putAll(members);
92+
93+
parent.members().put(name, module);
94+
return module;
95+
}
96+
97+
private static PythonType convertToObjectType(Symbol symbol) {
98+
// What should we have here?
99+
return PythonType.UNKNOWN;
100+
}
101+
102+
private static PythonType convertToFunctionType(FunctionSymbol symbol) {
103+
return new FunctionType(symbol.name(), List.of(), List.of(), PythonType.UNKNOWN, false, false, false, false, null);
104+
}
105+
106+
private PythonType convertToClassType(ClassSymbol symbol) {
107+
Set<Member> members = symbol.declaredMembers().stream().map(m -> new Member(m.name(), convertToType(m))).collect(Collectors.toSet());
108+
List<PythonType> superClasses = symbol.superClasses().stream().map(this::convertToType).toList();
109+
return new ClassType(symbol.name(), members, List.of(), superClasses);
110+
}
111+
112+
private PythonType convertToUnionType(AmbiguousSymbol ambiguousSymbol) {
113+
List<PythonType> pythonTypes = ambiguousSymbol.alternatives().stream().map(this::convertToType).toList();
114+
return new UnionType(pythonTypes);
115+
}
116+
117+
private PythonType convertToType(Symbol symbol) {
118+
return switch (symbol.kind()) {
119+
case CLASS -> convertToClassType((ClassSymbol) symbol);
120+
case FUNCTION -> convertToFunctionType((FunctionSymbol) symbol);
121+
case AMBIGUOUS -> convertToUnionType((AmbiguousSymbol) symbol);
122+
case OTHER -> convertToObjectType(symbol);
123+
};
124+
}
125+
126+
}

‎python-frontend/src/main/java/org/sonar/python/semantic/v2/TypeInferenceV2.java

Lines changed: 59 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,22 +22,25 @@
2222
import java.util.ArrayDeque;
2323
import java.util.Collection;
2424
import java.util.Deque;
25-
import java.util.HashMap;
2625
import java.util.List;
2726
import java.util.Optional;
27+
import javax.annotation.Nullable;
2828
import org.sonar.plugins.python.api.tree.AssignmentStatement;
2929
import org.sonar.plugins.python.api.tree.BaseTreeVisitor;
3030
import org.sonar.plugins.python.api.tree.ClassDef;
31+
import org.sonar.plugins.python.api.tree.DottedName;
3132
import org.sonar.plugins.python.api.tree.Expression;
3233
import org.sonar.plugins.python.api.tree.ExpressionList;
3334
import org.sonar.plugins.python.api.tree.FileInput;
3435
import org.sonar.plugins.python.api.tree.FunctionDef;
36+
import org.sonar.plugins.python.api.tree.ImportFrom;
3537
import org.sonar.plugins.python.api.tree.ImportName;
3638
import org.sonar.plugins.python.api.tree.ListLiteral;
3739
import org.sonar.plugins.python.api.tree.Name;
3840
import org.sonar.plugins.python.api.tree.NumericLiteral;
3941
import org.sonar.plugins.python.api.tree.QualifiedExpression;
4042
import org.sonar.plugins.python.api.tree.StringLiteral;
43+
import org.sonar.plugins.python.api.tree.Tree;
4144
import org.sonar.plugins.python.api.types.InferredType;
4245
import org.sonar.python.tree.ListLiteralImpl;
4346
import org.sonar.python.tree.NameImpl;
@@ -63,7 +66,7 @@ public TypeInferenceV2(ProjectLevelTypeTable projectLevelTypeTable) {
6366

6467
@Override
6568
public void visitFileInput(FileInput fileInput) {
66-
var type = new ModuleType("somehow get its name", new HashMap<>());
69+
var type = new ModuleType("somehow get its name");
6770
inTypeScope(type, () -> super.visitFileInput(fileInput));
6871
}
6972

@@ -141,8 +144,52 @@ private FunctionType buildFunctionType(FunctionDef functionDef) {
141144

142145
@Override
143146
public void visitImportName(ImportName importName) {
144-
//createImportedNames(importName.modules(), null, Collections.emptyList());
145-
super.visitImportName(importName);
147+
importName.modules()
148+
.forEach(aliasedName -> {
149+
var names = aliasedName.dottedName().names();
150+
var fqn = names
151+
.stream().map(Name::name)
152+
.toList();
153+
var module = projectLevelTypeTable.getModule(fqn);
154+
155+
if (aliasedName.alias() != null) {
156+
setTypeToName(aliasedName.alias(), module);
157+
} else {
158+
for (int i = names.size() - 1; i >= 0; i--) {
159+
setTypeToName(names.get(i), module);
160+
module = Optional.ofNullable(module)
161+
.map(ModuleType::parent)
162+
.orElse(null);
163+
}
164+
}
165+
});
166+
}
167+
168+
@Override
169+
public void visitImportFrom(ImportFrom importFrom) {
170+
Optional.of(importFrom)
171+
.map(ImportFrom::module)
172+
.map(DottedName::names)
173+
.ifPresent(names -> {
174+
var fqn = names
175+
.stream().map(Name::name)
176+
.toList();
177+
178+
var module = projectLevelTypeTable.getModule(fqn);
179+
importFrom.importedNames().forEach(aliasedName -> aliasedName
180+
.dottedName()
181+
.names()
182+
.stream()
183+
.findFirst()
184+
.ifPresent(name -> {
185+
var type = module.resolveMember(name.name());
186+
187+
var boundName = Optional.ofNullable(aliasedName.alias())
188+
.orElse(name);
189+
190+
setTypeToName(boundName, type);
191+
}));
192+
});
146193
}
147194

148195
@Override
@@ -186,8 +233,8 @@ public void visitName(Name name) {
186233
.map(Expression::typeV2)
187234
.toList();
188235

189-
if (type.size() == 1 && name instanceof NameImpl nameImpl) {
190-
nameImpl.typeV2(type.get(0));
236+
if (type.size() == 1) {
237+
setTypeToName(name, type.get(0));
191238
}
192239
}
193240

@@ -201,4 +248,10 @@ private void inTypeScope(PythonType pythonType, Runnable runnable) {
201248
this.typeStack.poll();
202249
}
203250

251+
private static void setTypeToName(@Nullable Tree tree, @Nullable PythonType type) {
252+
if (tree instanceof NameImpl name && type != null) {
253+
name.typeV2(type);
254+
}
255+
}
256+
204257
}

0 commit comments

Comments
 (0)