SONARPY-2178 Add awareness of the Python version & disambiguation to the DescriptorToPythonTypeConverters (#2024)

ghislainpiot · guillaume-dequenne · commit 81be222cf84d · 2024-10-04T16:23:16.000+02:00
diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/typeshed/ClassSymbolToDescriptorConverter.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/typeshed/ClassSymbolToDescriptorConverter.java
@@ -19,7 +19,7 @@
  */
 package org.sonar.python.semantic.v2.typeshed;
 
-import java.util.function.Function;
+import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.sonar.python.index.ClassDescriptor;
@@ -31,13 +31,15 @@ public class ClassSymbolToDescriptorConverter {
   private final VarSymbolToDescriptorConverter varConverter;
   private final FunctionSymbolToDescriptorConverter functionConverter;
   private final OverloadedFunctionSymbolToDescriptorConverter overloadedFunctionConverter;
+  private final Set<String> projectPythonVersions;
 
   public ClassSymbolToDescriptorConverter(VarSymbolToDescriptorConverter varConverter,
     FunctionSymbolToDescriptorConverter functionConverter,
-    OverloadedFunctionSymbolToDescriptorConverter overloadedFunctionConverter) {
+    OverloadedFunctionSymbolToDescriptorConverter overloadedFunctionConverter, Set<String> projectPythonVersions) {
     this.varConverter = varConverter;
     this.functionConverter = functionConverter;
     this.overloadedFunctionConverter = overloadedFunctionConverter;
+    this.projectPythonVersions = projectPythonVersions;
   }
 
   public ClassDescriptor convert(SymbolsProtos.ClassSymbol classSymbol) {
@@ -47,20 +49,21 @@ public ClassDescriptor convert(SymbolsProtos.ClassSymbol classSymbol) {
 
     var variableDescriptors = classSymbol.getAttributesList()
       .stream()
+      .filter(d -> ProtoUtils.isValidForPythonVersion(d.getValidForList(), projectPythonVersions))
       .map(varConverter::convert);
 
     var functionDescriptors = classSymbol.getMethodsList()
       .stream()
+      .filter(d -> ProtoUtils.isValidForPythonVersion(d.getValidForList(), projectPythonVersions))
       .map(s -> functionConverter.convert(s, true));
 
     var overloadedFunctionDescriptors = classSymbol.getOverloadedMethodsList()
       .stream()
+      .filter(d -> ProtoUtils.isValidForPythonVersion(d.getValidForList(), projectPythonVersions))
       .map(s -> overloadedFunctionConverter.convert(s, true));
 
-    var members = Stream.of(variableDescriptors, functionDescriptors, overloadedFunctionDescriptors)
-      .flatMap(Function.identity())
-      .map(Descriptor.class::cast)
-      .collect(Collectors.toSet());
+    var members = ProtoUtils.disambiguateByName(Stream.of(variableDescriptors, functionDescriptors, overloadedFunctionDescriptors))
+      .values().stream().map(Descriptor.class::cast).collect(Collectors.toSet());
 
     return new ClassDescriptor.ClassDescriptorBuilder()
       .withName(classSymbol.getName())
diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/typeshed/ModuleSymbolToDescriptorConverter.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/typeshed/ModuleSymbolToDescriptorConverter.java
@@ -19,16 +19,13 @@
  */
 package org.sonar.python.semantic.v2.typeshed;
 
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
-import org.sonar.plugins.python.api.ProjectPythonVersion;
 import org.sonar.plugins.python.api.PythonVersionUtils;
-import org.sonar.python.index.AmbiguousDescriptor;
 import org.sonar.python.index.Descriptor;
 import org.sonar.python.index.ModuleDescriptor;
 import org.sonar.python.types.protobuf.SymbolsProtos;
@@ -38,14 +35,14 @@ public class ModuleSymbolToDescriptorConverter {
   private final FunctionSymbolToDescriptorConverter functionConverter;
   private final VarSymbolToDescriptorConverter variableConverter;
   private final OverloadedFunctionSymbolToDescriptorConverter overloadedFunctionConverter;
-  private final Set<String> supportedPythonVersions;
+  private final Set<String> projectPythonVersions;
 
-  public ModuleSymbolToDescriptorConverter() {
+  public ModuleSymbolToDescriptorConverter(Set<PythonVersionUtils.Version> projectPythonVersions) {
+    this.projectPythonVersions = projectPythonVersions.stream().map(PythonVersionUtils.Version::serializedValue).collect(Collectors.toSet());
     functionConverter = new FunctionSymbolToDescriptorConverter();
     variableConverter = new VarSymbolToDescriptorConverter();
     overloadedFunctionConverter = new OverloadedFunctionSymbolToDescriptorConverter(functionConverter);
-    classConverter = new ClassSymbolToDescriptorConverter(variableConverter, functionConverter, overloadedFunctionConverter);
-    supportedPythonVersions = ProjectPythonVersion.currentVersionValues();
+    classConverter = new ClassSymbolToDescriptorConverter(variableConverter, functionConverter, overloadedFunctionConverter, this.projectPythonVersions);
   }
 
   @CheckForNull
@@ -62,56 +59,28 @@ public ModuleDescriptor convert(@Nullable SymbolsProtos.ModuleSymbol moduleSymbo
   }
 
   private Map<String, Descriptor> getModuleDescriptors(SymbolsProtos.ModuleSymbol moduleSymbol) {
-    Map<String, Set<Descriptor>> protoSymbolsByName = new HashMap<>();
-    moduleSymbol.getClassesList()
+    var classesStream = moduleSymbol.getClassesList()
       .stream()
-      .filter(d -> isValidForProjectPythonVersion(d.getValidForList()))
+      .filter(d -> ProtoUtils.isValidForPythonVersion(d.getValidForList(), projectPythonVersions))
       .map(classConverter::convert)
-      .forEach(descriptor -> protoSymbolsByName.computeIfAbsent(descriptor.name(), d -> new HashSet<>()).add(descriptor));
-    moduleSymbol.getFunctionsList()
+      .map(Descriptor.class::cast);
+    var functionsStream = moduleSymbol.getFunctionsList()
       .stream()
-      .filter(d -> isValidForProjectPythonVersion(d.getValidForList()))
+      .filter(d -> ProtoUtils.isValidForPythonVersion(d.getValidForList(), projectPythonVersions))
       .map(functionConverter::convert)
-      .forEach(descriptor -> protoSymbolsByName.computeIfAbsent(descriptor.name(), d -> new HashSet<>()).add(descriptor));
-    moduleSymbol.getOverloadedFunctionsList()
+      .map(Descriptor.class::cast);
+    var overloadedFunctionsStream = moduleSymbol.getOverloadedFunctionsList()
       .stream()
-      .filter(d -> isValidForProjectPythonVersion(d.getValidForList()))
+      .filter(d -> ProtoUtils.isValidForPythonVersion(d.getValidForList(), projectPythonVersions))
       .map(overloadedFunctionConverter::convert)
-      .forEach(descriptor -> protoSymbolsByName.computeIfAbsent(descriptor.name(), d -> new HashSet<>()).add(descriptor));
-    moduleSymbol.getVarsList()
+      .map(Descriptor.class::cast);
+    var variablesStream = moduleSymbol.getVarsList()
       .stream()
-      .filter(d -> isValidForProjectPythonVersion(d.getValidForList()))
+      .filter(d -> ProtoUtils.isValidForPythonVersion(d.getValidForList(), projectPythonVersions))
       .map(variableConverter::convert)
-      .forEach(descriptor -> protoSymbolsByName.computeIfAbsent(descriptor.name(), d -> new HashSet<>()).add(descriptor));
+      .map(Descriptor.class::cast);
 
-    var descriptorsByName = new HashMap<String, Descriptor>();
-
-    protoSymbolsByName.forEach((name, descriptors) -> {
-      Descriptor disambiguatedDescriptor = disambiguateSymbolsWithSameName(descriptors);
-      descriptorsByName.put(name, disambiguatedDescriptor);
-    });
-
-    return descriptorsByName;
-  }
-
-  private static Descriptor disambiguateSymbolsWithSameName(Set<Descriptor> descriptors) {
-    if (descriptors.size() > 1) {
-      return AmbiguousDescriptor.create(descriptors);
-    }
-    return descriptors.iterator().next();
-  }
-
-  private boolean isValidForProjectPythonVersion(List<String> validForPythonVersions) {
-    if (validForPythonVersions.isEmpty()) {
-      return true;
-    }
-    if (supportedPythonVersions.stream().allMatch(PythonVersionUtils.Version.V_312.serializedValue()::equals)
-        && validForPythonVersions.contains(PythonVersionUtils.Version.V_311.serializedValue())) {
-      return true;
-    }
-    HashSet<String> intersection = new HashSet<>(validForPythonVersions);
-    intersection.retainAll(supportedPythonVersions);
-    return !intersection.isEmpty();
+    return ProtoUtils.disambiguateByName(Stream.of(classesStream, functionsStream, overloadedFunctionsStream, variablesStream));
   }
 
 }
diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/typeshed/OverloadedFunctionSymbolToDescriptorConverter.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/typeshed/OverloadedFunctionSymbolToDescriptorConverter.java
@@ -28,8 +28,8 @@ public class OverloadedFunctionSymbolToDescriptorConverter {
 
   private final FunctionSymbolToDescriptorConverter functionConverter;
 
-  public OverloadedFunctionSymbolToDescriptorConverter(FunctionSymbolToDescriptorConverter functionConverter2) {
-    functionConverter = functionConverter2;
+  public OverloadedFunctionSymbolToDescriptorConverter(FunctionSymbolToDescriptorConverter functionConverter) {
+    this.functionConverter = functionConverter;
   }
 
   public AmbiguousDescriptor convert(SymbolsProtos.OverloadedFunctionSymbol overloadedFunctionSymbol) {
diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/typeshed/ProtoUtils.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/typeshed/ProtoUtils.java
@@ -0,0 +1,65 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2011-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.python.semantic.v2.typeshed;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.sonar.plugins.python.api.PythonVersionUtils;
+import org.sonar.python.index.AmbiguousDescriptor;
+import org.sonar.python.index.Descriptor;
+
+public class ProtoUtils {
+  private ProtoUtils() {
+    // utility class
+  }
+
+  static Map<String, Descriptor> disambiguateByName(Stream<Stream<? extends Descriptor>> input) {
+    return input.flatMap(i -> i)
+      .collect(Collectors.groupingBy(
+        Descriptor::name, Collectors.collectingAndThen(
+          Collectors.toSet(),
+          ProtoUtils::disambiguateSymbolsWithSameName)));
+  }
+
+  private static Descriptor disambiguateSymbolsWithSameName(Set<Descriptor> descriptors) {
+    if (descriptors.size() > 1) {
+      return AmbiguousDescriptor.create(descriptors);
+    }
+    return descriptors.iterator().next();
+  }
+
+  static boolean isValidForPythonVersion(List<String> validForPythonVersions, Set<String> supportedPythonVersions) {
+    if (validForPythonVersions.isEmpty()) {
+      return true;
+    }
+    // TODO: SONARPY-1522 - remove this workaround when we will have all the stubs for Python 3.12.
+    if (supportedPythonVersions.stream().allMatch(PythonVersionUtils.Version.V_312.serializedValue()::equals)
+      && validForPythonVersions.contains(PythonVersionUtils.Version.V_311.serializedValue())) {
+      return true;
+    }
+    HashSet<String> intersection = new HashSet<>(validForPythonVersions);
+    intersection.retainAll(supportedPythonVersions);
+    return !intersection.isEmpty();
+  }
+}
diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/typeshed/TypeShedDescriptorsProvider.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/typeshed/TypeShedDescriptorsProvider.java
@@ -31,6 +31,8 @@
 import javax.annotation.CheckForNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.sonar.plugins.python.api.ProjectPythonVersion;
+import org.sonar.plugins.python.api.PythonVersionUtils;
 import org.sonar.python.index.ClassDescriptor;
 import org.sonar.python.index.Descriptor;
 import org.sonar.python.index.ModuleDescriptor;
@@ -60,7 +62,11 @@ public class TypeShedDescriptorsProvider {
   private final Map<String, Map<String, Descriptor>> cachedDescriptors;
 
   public TypeShedDescriptorsProvider(Set<String> projectBasePackages) {
-    moduleConverter = new ModuleSymbolToDescriptorConverter();
+    this(projectBasePackages, ProjectPythonVersion.currentVersions());
+  }
+
+  public TypeShedDescriptorsProvider(Set<String> projectBasePackages, Set<PythonVersionUtils.Version> projectPythonVersions) {
+    moduleConverter = new ModuleSymbolToDescriptorConverter(projectPythonVersions);
     cachedDescriptors = new HashMap<>();
     this.projectBasePackages = projectBasePackages;
   }
diff --git a/python-frontend/src/test/java/org/sonar/python/semantic/v2/typeshed/ClassSymbolToDescriptorConverterTest.java b/python-frontend/src/test/java/org/sonar/python/semantic/v2/typeshed/ClassSymbolToDescriptorConverterTest.java
@@ -23,6 +23,7 @@
 import java.util.stream.Collectors;
 import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.Test;
+import org.sonar.plugins.python.api.ProjectPythonVersion;
 import org.sonar.python.index.AmbiguousDescriptor;
 import org.sonar.python.index.Descriptor;
 import org.sonar.python.index.FunctionDescriptor;
@@ -36,7 +37,7 @@ void test() {
     var functionConverter = new FunctionSymbolToDescriptorConverter();
     var variableConverter = new VarSymbolToDescriptorConverter();
     var overloadedFunctionConverter = new OverloadedFunctionSymbolToDescriptorConverter(functionConverter);
-    var converter = new ClassSymbolToDescriptorConverter(variableConverter, functionConverter, overloadedFunctionConverter);
+    var converter = new ClassSymbolToDescriptorConverter(variableConverter, functionConverter, overloadedFunctionConverter, ProjectPythonVersion.currentVersionValues());
 
     var symbol = SymbolsProtos.ClassSymbol.newBuilder()
       .setName("MyClass")
@@ -102,7 +103,7 @@ void builtinsTest() {
     var functionConverter = new FunctionSymbolToDescriptorConverter();
     var variableConverter = new VarSymbolToDescriptorConverter();
     var overloadedFunctionConverter = new OverloadedFunctionSymbolToDescriptorConverter(functionConverter);
-    var converter = new ClassSymbolToDescriptorConverter(variableConverter, functionConverter, overloadedFunctionConverter);
+    var converter = new ClassSymbolToDescriptorConverter(variableConverter, functionConverter, overloadedFunctionConverter, ProjectPythonVersion.currentVersionValues());
 
     var symbol = SymbolsProtos.ClassSymbol.newBuilder()
       .setName("int")
diff --git a/python-frontend/src/test/java/org/sonar/python/semantic/v2/typeshed/ModuleSymbolToDescriptorConverterTest.java b/python-frontend/src/test/java/org/sonar/python/semantic/v2/typeshed/ModuleSymbolToDescriptorConverterTest.java
@@ -22,7 +22,6 @@
 import java.util.Set;
 import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.Test;
-import org.sonar.plugins.python.api.ProjectPythonVersion;
 import org.sonar.plugins.python.api.PythonVersionUtils;
 import org.sonar.python.index.AmbiguousDescriptor;
 import org.sonar.python.index.ClassDescriptor;
@@ -34,7 +33,7 @@ class ModuleSymbolToDescriptorConverterTest {
 
   @Test
   void test() {
-    var converter = new ModuleSymbolToDescriptorConverter();
+    var converter = new ModuleSymbolToDescriptorConverter(PythonVersionUtils.allVersions());
     var symbol = SymbolsProtos.ModuleSymbol.newBuilder()
       .setFullyQualifiedName("module")
       .addVars(SymbolsProtos.VarSymbol.newBuilder()
@@ -86,15 +85,14 @@ void test() {
 
   @Test
   void nullSymbolTest() {
-    var converter = new ModuleSymbolToDescriptorConverter();
+    var converter = new ModuleSymbolToDescriptorConverter(PythonVersionUtils.allVersions());
     var descriptor = converter.convert(null);
     Assertions.assertThat(descriptor).isNull();
   }
 
   @Test
   void validForPythonVersionsTest() {
-    ProjectPythonVersion.setCurrentVersions(Set.of(PythonVersionUtils.Version.V_312));
-    var converter = new ModuleSymbolToDescriptorConverter();
+    var converter = new ModuleSymbolToDescriptorConverter(Set.of(PythonVersionUtils.Version.V_312));
     var symbol = SymbolsProtos.ModuleSymbol.newBuilder()
       .setFullyQualifiedName("module")
       .addVars(SymbolsProtos.VarSymbol.newBuilder()
@@ -113,8 +111,7 @@ void validForPythonVersionsTest() {
     Assertions.assertThat(descriptor.members().get("v1")).isInstanceOf(VariableDescriptor.class);
     Assertions.assertThat(descriptor.members().get("v2")).isNull();
 
-    ProjectPythonVersion.setCurrentVersions(Set.of(PythonVersionUtils.Version.V_39));
-    converter = new ModuleSymbolToDescriptorConverter();
+    converter = new ModuleSymbolToDescriptorConverter(Set.of(PythonVersionUtils.Version.V_39));
     descriptor = converter.convert(symbol);
     Assertions.assertThat(descriptor).isNotNull();
     Assertions.assertThat(descriptor.members().get("v1")).isNull();
diff --git a/python-frontend/src/test/java/org/sonar/python/semantic/v2/typeshed/TypeShedDescriptorsProviderTest.java b/python-frontend/src/test/java/org/sonar/python/semantic/v2/typeshed/TypeShedDescriptorsProviderTest.java

Original file line number	Diff line number	Diff line change
`@@ -28,8 +28,8 @@ public class OverloadedFunctionSymbolToDescriptorConverter {`
`28`	`28`
`29`	`29`	`private final FunctionSymbolToDescriptorConverter functionConverter;`
`30`	`30`
`31`		`- public OverloadedFunctionSymbolToDescriptorConverter(FunctionSymbolToDescriptorConverter functionConverter2) {`
`32`		`- functionConverter = functionConverter2;`
	`31`	`+ public OverloadedFunctionSymbolToDescriptorConverter(FunctionSymbolToDescriptorConverter functionConverter) {`
	`32`	`+ this.functionConverter = functionConverter;`
`33`	`33`	`}`
`34`	`34`
`35`	`35`	`public AmbiguousDescriptor convert(SymbolsProtos.OverloadedFunctionSymbol overloadedFunctionSymbol) {`