SONARPY-873 Run Typeshed serializer on Cirrus (#943)

Author: guillaume-dequenne

.cirrus.yml

@@ -35,7 +35,12 @@ win_vm_definition: &WINDOWS_VM_DEFINITION
 
 build_task:
   gke_container:
-    <<: *CONTAINER_DEFINITION
+    dockerfile: .cirrus/python-env.Dockerfile
+    builder_image_project: language-team
+    builder_image_name: docker-builder-lt-v1
+    cluster_name: cirrus-ci-lt-cluster
+    zone: us-central1-a
+    namespace: default
     cpu: 2
     memory: 2G
   env:
@@ -65,7 +70,7 @@ build_win_task:
     folder: ~/.m2/repository
   build_script:
     - git submodule update --init
-    - mvn.cmd -s /c/buildTools-docker/settings-public.xml test
+    - mvn.cmd -s /c/buildTools-docker/settings-public.xml test -DskipTypeshed=true
   cleanup_before_cache_script: cleanup_maven_repository
 
 plugin_qa_task:

.cirrus/python-env.Dockerfile

@@ -0,0 +1,16 @@
+FROM gcr.io/language-team/base:latest
+USER root
+ENV PYTHON_VERSION=3.9.5
+RUN apt-get update && apt-get install -y build-essential zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libsqlite3-dev libreadline-dev libffi-dev curl libbz2-dev
+RUN curl -O https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz
+RUN tar -xf Python-${PYTHON_VERSION}.tar.xz
+RUN cd Python-${PYTHON_VERSION} && ./configure --enable-optimizations && make -j 4 && make altinstall
+RUN cd /usr/local/bin \
+    && ln -s python3.9 python \
+    && ln -s python3.9 python3 \
+    && ln -s pip3.9 pip \
+    && ln -s pip3.9 pip3
+RUN python3.9 -m pip install --upgrade pip
+USER sonarsource
+RUN pip install tox
+ENV PATH=${PATH}:/home/sonarsource/.local/bin

python-frontend/pom.xml

@@ -11,7 +11,10 @@
   <artifactId>python-frontend</artifactId>
 
   <name>Python :: Frontend</name>
-
+  <properties>
+    <sonar.sources>pom.xml,src/main/java,typeshed_serializer/serializer</sonar.sources>
+    <sonar.python.coverage.reportPaths>typeshed_serializer/cov.xml</sonar.python.coverage.reportPaths>
+  </properties>
   <dependencies>
     <dependency>
       <groupId>org.sonarsource.sslr</groupId>
@@ -122,7 +125,7 @@
       <activation>
         <property>
           <name>
-            generateTypeshed
+            !skipTypeshed
           </name>
         </property>
       </activation>

python-frontend/typeshed_serializer/requirements.txt

@@ -4,3 +4,5 @@ protobuf==3.17.3
 six==1.16.0
 toml==0.10.2
 typing-extensions==3.10.0.0
+pytest==6.2.4
+pytest-cov==2.12.1

python-frontend/typeshed_serializer/tests/test_typeshed_serializer.py

@@ -1,52 +1,53 @@
-import unittest
-
+import pytest
 import mypy.nodes as mpn
 
 from serializer import typeshed_serializer, symbols
 
 
-class TypeshedSerializerTest(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls) -> None:
-        cls.build_result = typeshed_serializer.walk_typeshed_stdlib()
-
-    def test_build_mypy_model(self):
-        self.assertIsNotNone(self.build_result)
-
-    def test_module_symbol(self):
-        abc_module = self.build_result.files.get("abc")
-        module_symbol = symbols.ModuleSymbol(abc_module)
-        self.assertEqual(module_symbol.fullname, "abc")
-        self.assertEqual(len(module_symbol.classes), 3)
-        self.assertEqual(len(module_symbol.functions), 4)
-
-    def test_class_symbol(self):
-        mypy_cmd_module = self.build_result.files.get("cmd")
-        mypy_cmd_class = mypy_cmd_module.names.get("Cmd")
-        cmd_class_symbol = symbols.ClassSymbol(mypy_cmd_class.node)
-        self.assertEqual(cmd_class_symbol.fullname, "cmd.Cmd")
-        self.assertEqual(cmd_class_symbol.name, "Cmd")
-        self.assertEqual(cmd_class_symbol.super_classes, ["builtins.object"])
-        self.assertEqual(len(cmd_class_symbol.mro), 0)
-
-    def test_function_symbol(self):
-        mypy_cmd_module = self.build_result.files.get("cmd")
-        mypy_cmd_class_node = mypy_cmd_module.names.get("Cmd").node
-        self.assertIsInstance(mypy_cmd_class_node, mpn.TypeInfo)
-        mypy_completenames_method_node = mypy_cmd_class_node.names.get("completenames").node
-        self.assertIsInstance(mypy_completenames_method_node, mpn.FuncDef)
-        completenames_method_symbol = symbols.FunctionSymbol(mypy_completenames_method_node)
-        self.assertEqual(completenames_method_symbol.name, "completenames")
-        self.assertEqual(completenames_method_symbol.fullname, "cmd.Cmd.completenames")
-        self.assertFalse(completenames_method_symbol.has_decorators)
-        self.assertFalse(completenames_method_symbol.is_asynchronous)
-        self.assertEqual(completenames_method_symbol.return_type.pretty_printed_name, "builtins.list[builtins.str]")
-        self.assertEqual(len(completenames_method_symbol.resolved_decorator_names), 0)
-        self.assertEqual(completenames_method_symbol.fullname, "cmd.Cmd.completenames")
-
-        args = completenames_method_symbol.parameters
-        self.assertEqual(len(args), 3)
-
-
-if __name__ == '__main__':
-    unittest.main()
+@pytest.fixture(scope="module")
+def typeshed_stdlib():
+    return typeshed_serializer.walk_typeshed_stdlib()
+
+
+def test_build_mypy_model(typeshed_stdlib):
+    assert typeshed_stdlib is not None
+
+
+def test_module_symbol(typeshed_stdlib):
+    abc_module = typeshed_stdlib.files.get("abc")
+    module_symbol = symbols.ModuleSymbol(abc_module)
+    assert module_symbol.fullname == "abc"
+    assert len(module_symbol.classes) == 3
+    assert len(module_symbol.functions) == 4
+
+
+def test_class_symbol(typeshed_stdlib):
+    mypy_cmd_module = typeshed_stdlib.files.get("cmd")
+    mypy_cmd_class = mypy_cmd_module.names.get("Cmd")
+    cmd_class_symbol = symbols.ClassSymbol(mypy_cmd_class.node)
+    assert cmd_class_symbol.fullname == "cmd.Cmd"
+    assert cmd_class_symbol.name == "Cmd"
+    assert cmd_class_symbol.super_classes == ["builtins.object"]
+    assert len(cmd_class_symbol.mro) == 0
+
+
+def test_function_symbol(typeshed_stdlib):
+    mypy_cmd_module = typeshed_stdlib.files.get("cmd")
+
+    mypy_cmd_class_node = mypy_cmd_module.names.get("Cmd").node
+    assert isinstance(mypy_cmd_class_node, mpn.TypeInfo)
+
+    mypy_completenames_method_node = mypy_cmd_class_node.names.get("completenames").node
+    assert isinstance(mypy_completenames_method_node, mpn.FuncDef)
+
+    completenames_method_symbol = symbols.FunctionSymbol(mypy_completenames_method_node)
+    assert completenames_method_symbol.name == "completenames"
+    assert completenames_method_symbol.fullname == "cmd.Cmd.completenames"
+    assert not completenames_method_symbol.has_decorators
+    assert not completenames_method_symbol.is_asynchronous
+    assert completenames_method_symbol.return_type.pretty_printed_name == "builtins.list[builtins.str]"
+    assert len(completenames_method_symbol.resolved_decorator_names) == 0
+    assert completenames_method_symbol.fullname == "cmd.Cmd.completenames"
+
+    args = completenames_method_symbol.parameters
+    assert len(args) == 3

python-frontend/typeshed_serializer/tox.ini

@@ -1,10 +1,11 @@
 [tox]
-envlist = py38, serialize
+envlist = py39, serialize
 skipsdist = True
 
 [testenv]
 deps = -rrequirements.txt
-commands = python -m unittest
+whitelist_externals = pytest
+commands = pytest --cov=serializer --cov-report xml:cov.xml --ignore=resources
 
 [testenv:serialize]
 deps = -rrequirements.txt