SONARPY-961 Typeshed: serialize only public import

Author: guillaume-dequenne

python-frontend/src/test/java/org/sonar/python/types/TypeShedTest.java

@@ -195,14 +195,18 @@ public void package_inner_submodules_symbols() {
   @Test
   public void package_relative_import() {
     Map<String, Symbol> osSymbols = symbolsForModule("os");
-    Symbol sysSymbol = osSymbols.get("sys");
-    assertThat(sysSymbol.kind()).isEqualTo(Kind.OTHER);
-    Set<String> sysExportedSymbols = symbolsForModule("sys").keySet();
-    assertThat(((SymbolImpl) sysSymbol).getChildrenSymbolByName().values()).extracting(Symbol::name).containsAll(sysExportedSymbols);
-
-    Symbol timesResult = osSymbols.get("times_result");
-    assertThat(timesResult.kind()).isEqualTo(Kind.CLASS);
-    assertThat(timesResult.fullyQualifiedName()).isEqualTo("os.times_result");
+    // The "import sys" is not part of the exported API (private import) in Typeshed
+    // See: https://github.com/python/typeshed/blob/master/CONTRIBUTING.md#conventions
+    assertThat(osSymbols).doesNotContainKey("sys");
+
+    Map<String, Symbol> sqlite3Symbols = symbolsForModule("sqlite3");
+    Symbol completeStatementFunction = sqlite3Symbols.get("complete_statement");
+    assertThat(completeStatementFunction.kind()).isEqualTo(Kind.FUNCTION);
+    assertThat(completeStatementFunction.fullyQualifiedName()).isEqualTo("sqlite3.dbapi2.complete_statement");
+    Set<String> sqlite3Dbapi2Symbols = symbolsForModule("sqlite3.dbapi2").keySet();
+    // Python names with a leading underscore are not imported when using wildcard imports
+    sqlite3Dbapi2Symbols.removeIf(s -> s.startsWith("_"));
+    assertThat(sqlite3Symbols.keySet()).containsAll(sqlite3Dbapi2Symbols);
 
     Map<String, Symbol> requestsSymbols = symbolsForModule("requests");
     Symbol requestSymbol = requestsSymbols.get("request");
@@ -296,8 +300,7 @@ public void deserialize_annoy_protobuf() {
     Map<String, Symbol> deserializedAnnoySymbols = symbolsForModule("annoy").values().stream()
       .collect(Collectors.toMap(Symbol::fullyQualifiedName, s -> s));
     assertThat(deserializedAnnoySymbols.values()).extracting(Symbol::kind, Symbol::fullyQualifiedName)
-      .containsExactlyInAnyOrder(tuple(Kind.CLASS, "annoy._Vector"), tuple(Kind.CLASS, "annoy.AnnoyIndex"), tuple(Kind.OTHER, "annoy.Literal"),
-        tuple(Kind.CLASS, "annoy.Sized"), tuple(Kind.FUNCTION, "annoy.overload"), tuple(Kind.OTHER, "annoy.Protocol"));
+      .containsExactlyInAnyOrder(tuple(Kind.CLASS, "annoy._Vector"), tuple(Kind.CLASS, "annoy.AnnoyIndex"));
 
     ClassSymbol vector = (ClassSymbol) deserializedAnnoySymbols.get("annoy._Vector");
     assertThat(vector.superClasses()).extracting(Symbol::kind, Symbol::fullyQualifiedName)

python-frontend/typeshed_serializer/serializer/symbols.py

@@ -367,8 +367,21 @@ def __init__(self, mypy_file: mpn.MypyFile):
         self.functions = []
         self.overloaded_functions = []
         self.vars = []
+        private_imports = set()
+        for elem in mypy_file.imports:
+            # imports without aliases are considered private in Typeshed convention
+            if isinstance(elem, mpn.Import):
+                for _id, alias in elem.ids:
+                    if _id != alias:
+                        private_imports.add(_id)
+            if isinstance(elem, mpn.ImportFrom):
+                for _id, alias in elem.names:
+                    if _id != alias:
+                        private_imports.add(_id)
         for key in mypy_file.names:
             name = mypy_file.names.get(key)
+            if key in private_imports and not name.fullname.startswith(mypy_file.fullname):
+                continue
             if name.fullname == SONAR_CUSTOM_BASE_CLASS:
                 # Ignore custom stub name
                 continue

python-frontend/typeshed_serializer/serializer/typeshed_serializer.py

@@ -41,29 +41,6 @@ def get_options(python_version=(3, 8)):
     return opt
 
 
-def build_single_module(module_fqn: str, category="stdlib", python_version=(3, 8)):
-    opt = get_options(python_version)
-    module_source = load_single_module(module_fqn, category)
-    build_result = build.build([module_source], opt)
-    built_file = build_result.files.get(module_fqn)
-    return built_file
-
-
-def load_single_module(module_fqn: str, category="stdlib"):
-    module_path = module_fqn
-    if '.' in module_fqn:
-        module_path = module_fqn.replace('.', "/")
-    if os.path.isfile(path := os.path.join(CURRENT_PATH,
-                                           f"../resources/typeshed/{category}/{module_path}.pyi")):
-        module_source = build.BuildSource(path, module_fqn)
-    elif os.path.isfile(path := os.path.join(CURRENT_PATH,
-                                             f"../resources/typeshed/{category}/{module_path}/__init__.pyi")):
-        module_source = build.BuildSource(path, module_fqn)
-    else:
-        raise FileNotFoundError(f"No stub found for module {module_fqn}")
-    return module_source
-
-
 def walk_typeshed_stdlib(opt: options.Options = get_options()):
     generate_python2_stdlib = opt.python_version < (3, 0)
     relative_path = STDLIB_PATH if not generate_python2_stdlib else f"{STDLIB_PATH}/@python2"

python-frontend/typeshed_serializer/tests/conftest.py

@@ -19,12 +19,14 @@
 #
 
 import os
-from unittest.mock import Mock
 
 import pytest
 from mypy import build
 
 from serializer import typeshed_serializer, symbols_merger
+from serializer.typeshed_serializer import get_options
+
+CURRENT_PATH = os.path.dirname(__file__)
 
 
 @pytest.fixture(scope="session")
@@ -42,13 +44,29 @@ def typeshed_custom_stubs():
 
 @pytest.fixture(scope="session")
 def fake_module_36_38():
-    fake_module_path = os.path.join(os.path.dirname(__file__), "resources/fakemodule.pyi")
-    typeshed_serializer.load_single_module = Mock(return_value=build.BuildSource(fake_module_path, "fakemodule"))
-    fake_module_36 = typeshed_serializer.build_single_module('fakemodule', python_version=(3, 6))
-    fake_module_38 = typeshed_serializer.build_single_module('fakemodule', python_version=(3, 8))
-    return [fake_module_36, fake_module_38]
+    modules = {
+        "fakemodule": os.path.join(CURRENT_PATH, "resources/fakemodule.pyi"),
+        "fakemodule_imported": os.path.join(CURRENT_PATH, "resources/fakemodule_imported.pyi")
+    }
+    model_36 = build_modules(modules, python_version=(3, 6))
+    model_38 = build_modules(modules, python_version=(3, 8))
+    return [model_36.get("fakemodule"), model_38.get("fakemodule")]
 
 
 @pytest.fixture(scope="session")
 def typeshed_third_parties():
     return symbols_merger.merge_multiple_python_versions(is_third_parties=True)
+
+
+def build_modules(modules: dict[str, str], python_version=(3, 8)):
+    opt = get_options(python_version)
+    module_sources = []
+    for module_fqn in modules.keys():
+        module_sources.append(load_single_module(modules.get(module_fqn), module_fqn))
+    build_result = build.build(module_sources, opt)
+    return build_result.files
+
+
+def load_single_module(module_path: str, module_fqn):
+    module_source = build.BuildSource(module_path, module_fqn)
+    return module_source

python-frontend/typeshed_serializer/tests/resources/fakemodule.pyi

@@ -1,6 +1,8 @@
 import sys
+import math as math
 from typing import overload
 from sys import flags as my_flags
+from fakemodule_imported import *
 
 if sys.version_info >= (3, 8):
     class SomeClassUnique38:

python-frontend/typeshed_serializer/tests/resources/fakemodule_imported.pyi

@@ -0,0 +1,6 @@
+def common_imported_func():
+    ...
+
+
+def _private_func():
+    ...

python-frontend/typeshed_serializer/tests/test_symbols.py

@@ -30,12 +30,12 @@ def test_module_symbol(typeshed_stdlib):
     abc_module = typeshed_stdlib.files.get("abc")
     module_symbol = symbols.ModuleSymbol(abc_module)
     assert module_symbol.fullname == "abc"
-    assert len(module_symbol.classes) == 5
+    assert len(module_symbol.classes) == 3
     assert len(module_symbol.functions) == 4
 
     pb_module = module_symbol.to_proto()
     assert pb_module.fully_qualified_name == "abc"
-    assert len(pb_module.classes) == 5
+    assert len(pb_module.classes) == 3
     assert len(pb_module.functions) == 4
     imported_modules = [imported_module for imported_module in pb_module.vars if
                         imported_module.is_imported_module is True]
@@ -45,9 +45,8 @@ def test_module_symbol(typeshed_stdlib):
     pb_module = symbols.ModuleSymbol(os_module).to_proto()
     imported_modules = [imported_module for imported_module in pb_module.vars if
                         imported_module.is_imported_module is True]
-    assert len(imported_modules) == 3
+    assert len(imported_modules) == 2
     imported_modules = map(lambda m: (m.fully_qualified_name, m.name), imported_modules)
-    assert ("sys", "sys") in imported_modules
     assert ("os.path", "_path") in imported_modules
     assert ("os.path", "path") in imported_modules
 

python-frontend/typeshed_serializer/tests/test_symbols_merger.py

@@ -168,10 +168,17 @@ def test_actual_module_merge(fake_module_36_38):
 
     functions_dict = merged_fakemodule_module.functions
     assert len(functions_dict) == 5
+
     common_function_symbols = functions_dict['fakemodule.common_function']
     assert len(common_function_symbols) == 1
     assert common_function_symbols[0].valid_for == ["36", "38"]
 
+    common_function_wildcard_imported = functions_dict['fakemodule_imported.common_imported_func']
+    assert len(common_function_wildcard_imported) == 1
+    assert common_function_wildcard_imported[0].valid_for == ["36", "38"]
+
+    assert 'fakemodule_imported._private_func' not in functions_dict
+
     function_unique_36 = functions_dict['fakemodule.function_unique_36']
     assert len(function_unique_36) == 1
     assert function_unique_36[0].valid_for == ["36"]
@@ -239,9 +246,9 @@ def test_actual_module_merge(fake_module_36_38):
     assert alias_symbol.type.kind == TypeKind.CALLABLE
     assert alias_symbol.type.pretty_printed_name == "CallableType[builtins.function]"
 
-    imported_sys = all_vars['sys']
-    assert len(imported_sys) == 1
-    assert imported_sys[0].var_symbol.is_imported_module is True
+    imported_math = all_vars['math']
+    assert len(imported_math) == 1
+    assert imported_math[0].var_symbol.is_imported_module is True
 
     fakemodule_class_with_fields_symbols = classes_dict['fakemodule.ClassWithFields']
     assert len(fakemodule_class_with_fields_symbols) == 1
@@ -317,7 +324,7 @@ def assert_abc_merged_module(merged_modules, expected_valid_for):
     assert isinstance(abc_merged_symbol, MergedModuleSymbol)
     assert abc_merged_symbol.fullname == "abc"
     assert ([c for c in abc_merged_symbol.classes]
-            == ['_typeshed.SupportsWrite', 'typing.TypeVar', 'abc.ABCMeta', 'abc.abstractproperty', 'abc.ABC'])
+            == ['abc.ABCMeta', 'abc.abstractproperty', 'abc.ABC'])
     for merged_class_proto in abc_merged_symbol.classes.values():
         assert len(merged_class_proto) == 1
         assert merged_class_proto[0].valid_for == expected_valid_for