Parallelize integration tests (#411)

* start making int tests independent"

* int tests should calc their own expected result

* make dependency testing work

* correctly teardown files

* ouput file always created

* handle django files and unique dirs

* requirements.txt should be within tmp code dir

* initial batch of fixed integration tests

* allow for providing expected_new_code instead of replacement_lines

* second batch of converted int tests

* next fixed batch of integration tests

* sonar integration tests can still use samples file

* make test dependency manager run in parallel

* update makefile

* sonar int tests should use 1-idx replacements

* cleanup utils

* remove unnecessary samples files

* fix pre-commit

* fix unit tests to use tmpdir

* lower coverage

* move to cls setup

Author: clavedeluna

Makefile

@@ -1,12 +1,12 @@
 PYTEST = pytest -v
-COV_FLAGS = --cov-fail-under=94 --cov=codemodder --cov=core_codemods
+COV_FLAGS = --cov-fail-under=93.5 --cov=codemodder --cov=core_codemods
 XDIST_FLAGS = --numprocesses auto
 
 test:
 	COVERAGE_CORE=sysmon ${PYTEST} ${COV_FLAGS} tests ${XDIST_FLAGS}
 
 integration-test:
-	${PYTEST} integration_tests
+	${PYTEST} integration_tests ${XDIST_FLAGS}
 
 pygoat-test:
 	${PYTEST} -v ci_tests/test_pygoat_findings.py

integration_tests/test_add_requests_timeout.py

@@ -1,7 +1,6 @@
-from codemodder.codemods.test import (
-    BaseIntegrationTest,
-    original_and_expected_from_code_path,
-)
+from requests.exceptions import ConnectionError
+
+from codemodder.codemods.test import BaseIntegrationTest
 from core_codemods.add_requests_timeouts import (
     AddRequestsTimeouts,
     TransformAddRequestsTimeouts,
@@ -10,15 +9,19 @@
 
 class TestAddRequestsTimeouts(BaseIntegrationTest):
     codemod = AddRequestsTimeouts
-    code_path = "tests/samples/requests_timeout.py"
+    original_code = """
+    import requests
+    
+    requests.get("https://example.com")
+    requests.get("https://example.com", timeout=1)
+    requests.get("https://example.com", timeout=(1, 10), verify=False)
+    requests.post("https://example.com", verify=False)
+    """
 
-    original_code, expected_new_code = original_and_expected_from_code_path(
-        code_path,
-        [
-            (2, 'requests.get("https://example.com", timeout=60)\n'),
-            (5, 'requests.post("https://example.com", verify=False, timeout=60)\n'),
-        ],
-    )
+    replacement_lines = [
+        (3, 'requests.get("https://example.com", timeout=60)\n'),
+        (6, 'requests.post("https://example.com", verify=False, timeout=60)\n'),
+    ]
 
     expected_diff = """\
 --- 
@@ -37,3 +40,5 @@ class TestAddRequestsTimeouts(BaseIntegrationTest):
     num_changes = 2
     expected_line_change = "3"
     change_description = TransformAddRequestsTimeouts.change_description
+    # expected because requests are made
+    allowed_exceptions = (ConnectionError,)

integration_tests/test_combine_startswith_endswith.py

@@ -1,16 +1,16 @@
-from codemodder.codemods.test import (
-    BaseIntegrationTest,
-    original_and_expected_from_code_path,
-)
+from codemodder.codemods.test import BaseIntegrationTest
 from core_codemods.combine_startswith_endswith import CombineStartswithEndswith
 
 
 class TestCombineStartswithEndswith(BaseIntegrationTest):
     codemod = CombineStartswithEndswith
-    code_path = "tests/samples/combine_startswith_endswith.py"
-    original_code, expected_new_code = original_and_expected_from_code_path(
-        code_path, [(1, 'if x.startswith(("foo", "bar")):\n')]
-    )
+    original_code = """
+    x = 'foo'
+    if x.startswith("foo") or x.startswith("bar"):
+        print("Yes")
+    """
+    replacement_lines = [(2, 'if x.startswith(("foo", "bar")):\n')]
+
     expected_diff = '--- \n+++ \n@@ -1,3 +1,3 @@\n x = \'foo\'\n-if x.startswith("foo") or x.startswith("bar"):\n+if x.startswith(("foo", "bar")):\n     print("Yes")\n'
     expected_line_change = "2"
     change_description = CombineStartswithEndswith.change_description

integration_tests/test_dependency_manager.py

@@ -1,19 +1,27 @@
 import os
+import pathlib
 import shutil
 import subprocess
+import tempfile
 from textwrap import dedent
 
 import pytest
 
-from codemodder.codemods.test.integration_utils import SAMPLES_DIR, CleanRepoMixin
+from codemodder.codemods.test.integration_utils import SAMPLES_DIR
 
 
-class TestDependencyManager(CleanRepoMixin):
-    output_path = "test-codetf.txt"
+class TestDependencyManager:
+    @classmethod
+    def setup_class(cls):
+        cls.output_path = tempfile.mkstemp()[1]
+        cls.toml_file = "pyproject.toml"
+        cls.requirements_file = "requirements.txt"
+        cls.setup_file = "setup.py"
 
-    toml_file = "pyproject.toml"
-    requirements_file = "requirements.txt"
-    setup_file = "setup.py"
+    @classmethod
+    def teardown_class(cls):
+        """Ensure any re-written file is undone after integration test class"""
+        pathlib.Path(cls.output_path).unlink(missing_ok=True)
 
     @pytest.fixture
     def tmp_repo(self, tmp_path):

integration_tests/test_django_debug_flag_on.py

@@ -1,17 +1,24 @@
-from codemodder.codemods.test import (
-    BaseIntegrationTest,
-    original_and_expected_from_code_path,
-)
+from codemodder.codemods.test import BaseIntegrationTest
 from core_codemods.django_debug_flag_on import DjangoDebugFlagOn
 
 
 class TestDjangoDebugFlagFlip(BaseIntegrationTest):
     codemod = DjangoDebugFlagOn
-    code_path = "tests/samples/django-project/mysite/mysite/settings.py"
-
-    original_code, expected_new_code = original_and_expected_from_code_path(
-        code_path, [(25, "DEBUG = False\n")]
+    code_filename = "settings.py"
+    original_code = """
+    # SECURITY WARNING: don't run with debug turned on in production!
+    DEBUG = True
+    """
+    replacement_lines = [(2, "DEBUG = False\n")]
+    # fmt: off
+    expected_diff = (
+    """--- \n"""
+    """+++ \n"""
+    """@@ -1,2 +1,2 @@\n"""
+    """ # SECURITY WARNING: don't run with debug turned on in production!\n"""
+    """-DEBUG = True\n"""
+    """+DEBUG = False\n"""
     )
-    expected_diff = '--- \n+++ \n@@ -23,7 +23,7 @@\n SECRET_KEY = "django-insecure-t*rrda&qd4^#q+50^%q^rrsp-t$##&u5_#=9)&@ei^ppl6$*c*"\n \n # SECURITY WARNING: don\'t run with debug turned on in production!\n-DEBUG = True\n+DEBUG = False\n \n ALLOWED_HOSTS = []\n \n'
-    expected_line_change = "26"
+    # fmt: on
+    expected_line_change = "2"
     change_description = DjangoDebugFlagOn.change_description

integration_tests/test_django_json_response_type.py

@@ -1,7 +1,4 @@
-from codemodder.codemods.test import (
-    BaseIntegrationTest,
-    original_and_expected_from_code_path,
-)
+from codemodder.codemods.test import BaseIntegrationTest
 from core_codemods.django_json_response_type import (
     DjangoJsonResponseType,
     DjangoJsonResponseTypeTransformer,
@@ -10,19 +7,23 @@
 
 class TestDjangoJsonResponseType(BaseIntegrationTest):
     codemod = DjangoJsonResponseType
-    code_path = "tests/samples/django_json_response_type.py"
-    original_code, expected_new_code = original_and_expected_from_code_path(
-        code_path,
-        [
-            (
-                5,
-                """    return HttpResponse(json_response, content_type="application/json")\n""",
-            ),
-        ],
-    )
+    original_code = """
+    from django.http import HttpResponse
+    import json
+    
+    def foo(request):
+        json_response = json.dumps({ "user_input": request.GET.get("input") })
+        return HttpResponse(json_response)
+    """
+    replacement_lines = [
+        (
+            6,
+            """    return HttpResponse(json_response, content_type="application/json")\n""",
+        ),
+    ]
 
     # fmt: off
-    expected_diff =(
+    expected_diff = (
     """--- \n"""
     """+++ \n"""
     """@@ -3,4 +3,4 @@\n"""

integration_tests/test_django_model_without_dunder_str.py

@@ -1,7 +1,4 @@
-from codemodder.codemods.test import (
-    BaseIntegrationTest,
-    original_and_expected_from_code_path,
-)
+from codemodder.codemods.test import BaseIntegrationTest
 from core_codemods.django_model_without_dunder_str import (
     DjangoModelWithoutDunderStr,
     DjangoModelWithoutDunderStrTransformer,
@@ -10,23 +7,35 @@
 
 class TestDjangoModelWithoutDunderStr(BaseIntegrationTest):
     codemod = DjangoModelWithoutDunderStr
-    code_path = "tests/samples/django-project/mysite/mysite/models.py"
-    original_code, expected_new_code = original_and_expected_from_code_path(
-        code_path,
-        [
-            (15, """\n"""),
-            (16, """    def __str__(self):\n"""),
-            (17, """        model_name = self.__class__.__name__\n"""),
-            (
-                18,
-                """        fields_str = ", ".join((f"{field.name}={getattr(self, field.name)}" for field in self._meta.fields))\n""",
-            ),
-            (19, """        return f"{model_name}({fields_str})"\n"""),
-        ],
-    )
+    original_code = """
+    import django
+    from django.conf import settings
+    from django.db import models
+    # required to run this module standalone for testing
+    settings.configure()
+    django.setup()
+    
+    
+    class Message(models.Model):
+        author = models.CharField(max_length=100)
+        content = models.CharField(max_length=200)
+        class Meta:
+            app_label = 'myapp'
+
+    """
+    replacement_lines = [
+        (15, """\n"""),
+        (16, """    def __str__(self):\n"""),
+        (17, """        model_name = self.__class__.__name__\n"""),
+        (
+            18,
+            """        fields_str = ", ".join((f"{field.name}={getattr(self, field.name)}" for field in self._meta.fields))\n""",
+        ),
+        (19, """        return f"{model_name}({fields_str})"\n"""),
+    ]
 
     # fmt: off
-    expected_diff =(
+    expected_diff = (
     """--- \n"""
     """+++ \n"""
     """@@ -11,3 +11,8 @@\n"""

integration_tests/test_django_receiver_on_top.py

@@ -1,7 +1,4 @@
-from codemodder.codemods.test import (
-    BaseIntegrationTest,
-    original_and_expected_from_code_path,
-)
+from codemodder.codemods.test import BaseIntegrationTest
 from core_codemods.django_receiver_on_top import (
     DjangoReceiverOnTop,
     DjangoReceiverOnTopTransformer,
@@ -10,17 +7,23 @@
 
 class TestDjangoReceiverOnTop(BaseIntegrationTest):
     codemod = DjangoReceiverOnTop
-    code_path = "tests/samples/django_receiver_on_top.py"
-    original_code, expected_new_code = original_and_expected_from_code_path(
-        code_path,
-        [
-            (4, """@receiver(request_finished)\n"""),
-            (5, """@csrf_exempt\n"""),
-        ],
-    )
+    original_code = """
+    from django.dispatch import receiver
+    from django.views.decorators.csrf import csrf_exempt
+    from django.core.signals import request_finished
+    
+    @csrf_exempt
+    @receiver(request_finished)
+    def foo():
+        pass
+    """
+    replacement_lines = [
+        (5, """@receiver(request_finished)\n"""),
+        (6, """@csrf_exempt\n"""),
+    ]
 
     # fmt: off
-    expected_diff =(
+    expected_diff = (
     """--- \n"""
     """+++ \n"""
     """@@ -2,7 +2,7 @@\n"""

integration_tests/test_django_session_cookie_secure_off.py

@@ -1,17 +1,26 @@
-from codemodder.codemods.test import (
-    BaseIntegrationTest,
-    original_and_expected_from_code_path,
-)
+from codemodder.codemods.test import BaseIntegrationTest
 from core_codemods.django_session_cookie_secure_off import DjangoSessionCookieSecureOff
 
 
 class TestDjangoSessionCookieSecureOff(BaseIntegrationTest):
     codemod = DjangoSessionCookieSecureOff
-    code_path = "tests/samples/django-project/mysite/mysite/settings.py"
+    code_filename = "settings.py"
 
-    original_code, expected_new_code = original_and_expected_from_code_path(
-        code_path, [(124, "SESSION_COOKIE_SECURE = True\n")]
+    original_code = """
+    # django settings
+    # SESSION_COOKIE_SECURE is not defined
+    """
+    replacement_lines = [(3, "SESSION_COOKIE_SECURE = True\n")]
+
+    # fmt: off
+    expected_diff = (
+        """--- \n"""
+        """+++ \n"""
+        """@@ -1,2 +1,3 @@\n"""
+        """ # django settings\n"""
+        """ # SESSION_COOKIE_SECURE is not defined\n"""
+        """+SESSION_COOKIE_SECURE = True\n"""
     )
-    expected_diff = '--- \n+++ \n@@ -121,3 +121,4 @@\n # https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field\n \n DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"\n+SESSION_COOKIE_SECURE = True\n'
-    expected_line_change = "124"
+    # fmt: on
+    expected_line_change = "3"
     change_description = DjangoSessionCookieSecureOff.change_description

integration_tests/test_exception_without_raise.py

@@ -1,7 +1,4 @@
-from codemodder.codemods.test import (
-    BaseIntegrationTest,
-    original_and_expected_from_code_path,
-)
+from codemodder.codemods.test import BaseIntegrationTest
 from core_codemods.exception_without_raise import (
     ExceptionWithoutRaise,
     ExceptionWithoutRaiseTransformer,
@@ -10,16 +7,15 @@
 
 class TestExceptionWithoutRaise(BaseIntegrationTest):
     codemod = ExceptionWithoutRaise
-    code_path = "tests/samples/exception_without_raise.py"
-    original_code, expected_new_code = original_and_expected_from_code_path(
-        code_path,
-        [
-            (1, """    raise ValueError\n"""),
-        ],
-    )
-
+    original_code = """
+    try:
+        ValueError
+    except:
+        pass
+    """
+    replacement_lines = [(2, """    raise ValueError\n""")]
     # fmt: off
-    expected_diff =(
+    expected_diff = (
     """--- \n"""
     """+++ \n"""
     """@@ -1,4 +1,4 @@\n"""