Skip to content

Commit c465221

Browse files
maksim-grebeniuk-sonarsourceguillaume-dequenne
maksim-grebeniuk-sonarsource
authored and
guillaume-dequenne
committed
SONARPY-2160 Implement TypeShedDescriptorsProvider
1 parent 158a80f commit c465221

File tree

2 files changed

+268
-0
lines changed

2 files changed

+268
-0
lines changed

‎python-frontend/src/main/java/org/sonar/python/semantic/v2/typeshed/TypeShedDescriptorsProvider.java

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
/*
2+
* SonarQube Python Plugin
3+
* Copyright (C) 2011-2024 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the GNU Lesser General Public
8+
* License as published by the Free Software Foundation; either
9+
* version 3 of the License, or (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
* Lesser General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Lesser General Public License
17+
* along with this program; if not, write to the Free Software Foundation,
18+
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19+
*/
20+
package org.sonar.python.semantic.v2.typeshed;
21+
22+
import java.io.IOException;
23+
import java.io.InputStream;
24+
import java.util.Collections;
25+
import java.util.HashMap;
26+
import java.util.Map;
27+
import java.util.Optional;
28+
import java.util.Set;
29+
import java.util.function.Predicate;
30+
import java.util.stream.Stream;
31+
import javax.annotation.CheckForNull;
32+
import org.slf4j.Logger;
33+
import org.slf4j.LoggerFactory;
34+
import org.sonar.python.index.ClassDescriptor;
35+
import org.sonar.python.index.Descriptor;
36+
import org.sonar.python.index.ModuleDescriptor;
37+
import org.sonar.python.types.protobuf.SymbolsProtos.ModuleSymbol;
38+
39+
import static org.sonar.plugins.python.api.types.BuiltinTypes.NONE_TYPE;
40+
41+
public class TypeShedDescriptorsProvider {
42+
private static final Logger LOG = LoggerFactory.getLogger(TypeShedDescriptorsProvider.class);
43+
44+
private static final String PROTOBUF_BASE_RESOURCE_PATH = "/org/sonar/python/types/";
45+
private static final String PROTOBUF_CUSTOM_STUBS = PROTOBUF_BASE_RESOURCE_PATH + "custom_protobuf/";
46+
private static final String PROTOBUF = PROTOBUF_BASE_RESOURCE_PATH + "stdlib_protobuf/";
47+
private static final String PROTOBUF_THIRD_PARTY = PROTOBUF_BASE_RESOURCE_PATH + "third_party_protobuf/";
48+
private static final String PROTOBUF_THIRD_PARTY_MYPY = PROTOBUF_BASE_RESOURCE_PATH + "third_party_protobuf_mypy/";
49+
public static final String BUILTINS_FQN = "builtins";
50+
// This is needed for some Python 2 modules whose name differ from their Python 3 counterpart by capitalization only.
51+
private static final Map<String, String> MODULES_TO_DISAMBIGUATE = Map.of(
52+
"ConfigParser", "2@ConfigParser",
53+
"Queue", "2@Queue",
54+
"SocketServer", "2@SocketServer"
55+
);
56+
private final ModuleSymbolToDescriptorConverter moduleConverter;
57+
58+
private Map<String, Descriptor> builtins;
59+
private final Set<String> projectBasePackages;
60+
private final Map<String, Map<String, Descriptor>> cachedDescriptors;
61+
62+
public TypeShedDescriptorsProvider(Set<String> projectBasePackages) {
63+
moduleConverter = new ModuleSymbolToDescriptorConverter();
64+
cachedDescriptors = new HashMap<>();
65+
this.projectBasePackages = projectBasePackages;
66+
}
67+
68+
//================================================================================
69+
// Public methods
70+
//================================================================================
71+
72+
public Map<String, Descriptor> builtinDescriptors() {
73+
if (builtins == null) {
74+
Map<String, Descriptor> symbols = getModuleDescriptors(BUILTINS_FQN, PROTOBUF);
75+
symbols.put(NONE_TYPE, new ClassDescriptor.ClassDescriptorBuilder().withName(NONE_TYPE).withFullyQualifiedName(NONE_TYPE).build());
76+
builtins = Collections.unmodifiableMap(symbols);
77+
}
78+
return builtins;
79+
}
80+
81+
/**
82+
* Returns map of exported symbols by name for a given module
83+
*/
84+
public Map<String, Descriptor> descriptorsForModule(String moduleName) {
85+
if (searchedModuleMatchesCurrentProject(moduleName)) {
86+
return Collections.emptyMap();
87+
}
88+
return cachedDescriptors.computeIfAbsent(moduleName, this::searchTypeShedForModule);
89+
}
90+
91+
//================================================================================
92+
// Private methods
93+
//================================================================================
94+
95+
private boolean searchedModuleMatchesCurrentProject(String searchedModule) {
96+
return projectBasePackages.contains(searchedModule.split("\\.", 2)[0]);
97+
}
98+
99+
private Map<String, Descriptor> searchTypeShedForModule(String moduleName) {
100+
return Stream.of(PROTOBUF_CUSTOM_STUBS, PROTOBUF, PROTOBUF_THIRD_PARTY_MYPY)
101+
.map(dirName -> getModuleDescriptors(moduleName, dirName))
102+
.filter(Predicate.not(Map::isEmpty))
103+
.findFirst()
104+
.orElseGet(() -> getModuleDescriptors(moduleName, PROTOBUF_THIRD_PARTY));
105+
}
106+
107+
private Map<String, Descriptor> getModuleDescriptors(String moduleName, String dirName) {
108+
String fileName = MODULES_TO_DISAMBIGUATE.getOrDefault(moduleName, moduleName);
109+
InputStream resource = this.getClass().getResourceAsStream(dirName + fileName + ".protobuf");
110+
if (resource == null) {
111+
return Collections.emptyMap();
112+
}
113+
var moduleSymbol = deserializedModule(moduleName, resource);
114+
var moduleDescriptor = moduleConverter.convert(moduleSymbol);
115+
return Optional.ofNullable(moduleDescriptor).map(ModuleDescriptor::members).orElseGet(Map::of);
116+
}
117+
118+
@CheckForNull
119+
static ModuleSymbol deserializedModule(String moduleName, InputStream resource) {
120+
try {
121+
return ModuleSymbol.parseFrom(resource);
122+
} catch (IOException e) {
123+
LOG.debug("Error while deserializing protobuf for module {}", moduleName, e);
124+
return null;
125+
}
126+
}
127+
128+
}

‎python-frontend/src/test/java/org/sonar/python/semantic/v2/typeshed/TypeShedDescriptorsProviderTest.java

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
/*
2+
* SonarQube Python Plugin
3+
* Copyright (C) 2011-2024 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the GNU Lesser General Public
8+
* License as published by the Free Software Foundation; either
9+
* version 3 of the License, or (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
* Lesser General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Lesser General Public License
17+
* along with this program; if not, write to the Free Software Foundation,
18+
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19+
*/
20+
package org.sonar.python.semantic.v2.typeshed;
21+
22+
import java.io.ByteArrayInputStream;
23+
import java.io.InputStream;
24+
import java.util.Set;
25+
import org.assertj.core.api.Assertions;
26+
import org.junit.jupiter.api.BeforeEach;
27+
import org.junit.jupiter.api.Test;
28+
import org.junit.jupiter.api.extension.RegisterExtension;
29+
import org.slf4j.event.Level;
30+
import org.sonar.api.testfixtures.log.LogTesterJUnit5;
31+
import org.sonar.plugins.python.api.ProjectPythonVersion;
32+
import org.sonar.plugins.python.api.PythonVersionUtils;
33+
import org.sonar.python.index.AmbiguousDescriptor;
34+
import org.sonar.python.index.Descriptor;
35+
import org.sonar.python.index.FunctionDescriptor;
36+
37+
import static org.assertj.core.api.Assertions.assertThat;
38+
39+
class TypeShedDescriptorsProviderTest {
40+
41+
@RegisterExtension
42+
public LogTesterJUnit5 logTester = new LogTesterJUnit5().setLevel(Level.DEBUG);
43+
44+
@BeforeEach
45+
void setup() {
46+
ProjectPythonVersion.setCurrentVersions(PythonVersionUtils.allVersions());
47+
}
48+
49+
@Test
50+
void builtinDescriptorsTest() {
51+
var provider = new TypeShedDescriptorsProvider(Set.of());
52+
var builtinDescriptors = provider.builtinDescriptors();
53+
Assertions.assertThat(builtinDescriptors).isNotEmpty();
54+
55+
var intDescriptor = builtinDescriptors.get("int");
56+
Assertions.assertThat(intDescriptor.fullyQualifiedName()).isEqualTo("int");
57+
58+
Assertions.assertThat(provider.builtinDescriptors()).isSameAs(builtinDescriptors);
59+
}
60+
61+
@Test
62+
void builtin312DescriptorsTest() {
63+
ProjectPythonVersion.setCurrentVersions(Set.of(PythonVersionUtils.Version.V_311));
64+
var provider = new TypeShedDescriptorsProvider(Set.of());
65+
var builtinDescriptors = provider.builtinDescriptors();
66+
67+
Assertions.assertThat(builtinDescriptors).isNotEmpty();
68+
}
69+
70+
@Test
71+
void typingDescriptorsTest() {
72+
var provider = new TypeShedDescriptorsProvider(Set.of());
73+
var typing = provider.descriptorsForModule("typing");
74+
Assertions.assertThat(typing).isNotEmpty();
75+
}
76+
77+
@Test
78+
void moduleMatchesCurrentProjectTest() {
79+
var provider = new TypeShedDescriptorsProvider(Set.of("typing"));
80+
var typing = provider.descriptorsForModule("typing");
81+
Assertions.assertThat(typing).isEmpty();
82+
}
83+
84+
@Test
85+
void cacheTest() {
86+
var provider = new TypeShedDescriptorsProvider(Set.of());
87+
var typing1 = provider.descriptorsForModule("typing");
88+
var typing2 = provider.descriptorsForModule("typing");
89+
Assertions.assertThat(typing1).isSameAs(typing2);
90+
}
91+
92+
93+
@Test
94+
void stdlibDescriptors() {
95+
var provider = new TypeShedDescriptorsProvider(Set.of());
96+
var mathDescriptors = provider.descriptorsForModule("math");
97+
var descriptor = mathDescriptors.get("acos");
98+
assertThat(descriptor.kind()).isEqualTo(Descriptor.Kind.AMBIGUOUS);
99+
var acosDescriptor = ((AmbiguousDescriptor) descriptor).alternatives().iterator().next();
100+
assertThat(acosDescriptor.kind()).isEqualTo(Descriptor.Kind.FUNCTION);
101+
assertThat(((FunctionDescriptor) acosDescriptor).parameters()).hasSize(1);
102+
assertThat(((FunctionDescriptor) acosDescriptor).annotatedReturnTypeName()).isEqualTo("float");
103+
104+
var threadingSymbols = provider.descriptorsForModule("threading");
105+
assertThat(threadingSymbols.get("Thread").kind()).isEqualTo(Descriptor.Kind.CLASS);
106+
107+
var imaplibSymbols = provider.descriptorsForModule("imaplib");
108+
assertThat(imaplibSymbols).isNotEmpty();
109+
}
110+
111+
@Test
112+
void shouldResolvePackages() {
113+
var provider = new TypeShedDescriptorsProvider(Set.of());
114+
assertThat(provider.descriptorsForModule("urllib")).isNotEmpty();
115+
assertThat(provider.descriptorsForModule("ctypes")).isNotEmpty();
116+
assertThat(provider.descriptorsForModule("email")).isNotEmpty();
117+
assertThat(provider.descriptorsForModule("json")).isNotEmpty();
118+
assertThat(provider.descriptorsForModule("docutils")).isNotEmpty();
119+
assertThat(provider.descriptorsForModule("ctypes.util")).isNotEmpty();
120+
assertThat(provider.descriptorsForModule("lib2to3.pgen2.grammar")).isNotEmpty();
121+
assertThat(provider.descriptorsForModule("cryptography")).isNotEmpty();
122+
// resolved but still empty
123+
assertThat(provider.descriptorsForModule("kazoo")).isEmpty();
124+
}
125+
126+
@Test
127+
void unknownModule() {
128+
var provider = new TypeShedDescriptorsProvider(Set.of());
129+
var unknownModule = provider.descriptorsForModule("unknown_module");
130+
assertThat(unknownModule).isEmpty();
131+
}
132+
133+
@Test
134+
void readExceptionTest() {
135+
InputStream targetStream = new ByteArrayInputStream("foo".getBytes());
136+
assertThat(TypeShedDescriptorsProvider.deserializedModule("mod", targetStream)).isNull();
137+
assertThat(logTester.logs(Level.DEBUG)).contains("Error while deserializing protobuf for module mod");
138+
}
139+
140+
}

0 commit comments

Comments
 (0)