refactor: make coherent lib module for integration testing

Author: ytausch

tests_integration/README.md

@@ -81,7 +81,7 @@ A test scenario assigns one test case to every feedstock and runs them in parall
 Thus, test cases of different feedstocks can run simultaneously, but the different test cases for the same feedstock
 are always run sequentially.
 
-The generation of test scenarios is done in [collect_test_scenarios.py](collect_test_scenarios.py). It is pseudo-random,
+The generation of test scenarios is done in [_collect_test_scenarios.py](lib/_collect_test_scenarios.py). It is pseudo-random,
 ensuring that faulty interactions between test cases are detected eventually.
 
 In detail, the process of collecting test scenarios is as follows:

tests_integration/lib/__init__.py

@@ -0,0 +1,36 @@
+from ._collect_test_scenarios import get_all_test_scenario_ids, get_test_scenario
+from ._integration_test_helper import IntegrationTestHelper
+from ._run_test_cases import (
+    close_all_open_pull_requests,
+    reset_cf_graph,
+    run_all_prepare_functions,
+    run_all_validate_functions,
+)
+from ._setup_repositories import prepare_all_accounts
+from ._shared import (
+    ENV_TEST_SCENARIO_ID,
+    VIRTUAL_PROXY_HOSTNAME,
+    VIRTUAL_PROXY_PORT,
+    get_global_router,
+    get_transparent_urls,
+    setup_logging,
+)
+from ._test_case import TestCase
+
+__all__ = [
+    "get_all_test_scenario_ids",
+    "get_test_scenario",
+    "IntegrationTestHelper",
+    "close_all_open_pull_requests",
+    "reset_cf_graph",
+    "run_all_prepare_functions",
+    "run_all_validate_functions",
+    "prepare_all_accounts",
+    "get_global_router",
+    "get_transparent_urls",
+    "setup_logging",
+    "ENV_TEST_SCENARIO_ID",
+    "VIRTUAL_PROXY_HOSTNAME",
+    "VIRTUAL_PROXY_PORT",
+    "TestCase",
+]

tests_integration/collect_test_scenarios.py renamed to tests_integration/lib/_collect_test_scenarios.py

@@ -2,8 +2,9 @@
 import random
 
 from tests_integration.definitions import TEST_CASE_MAPPING
-from tests_integration.lib.shared import ENV_GITHUB_RUN_ID
-from tests_integration.lib.test_case import TestCase
+
+from ._shared import ENV_GITHUB_RUN_ID
+from ._test_case import TestCase
 
 
 def get_number_of_test_scenarios() -> int:

tests_integration/lib/integration_test_helper.py renamed to tests_integration/lib/_integration_test_helper.py

@@ -11,7 +11,8 @@
 from conda_forge_tick.utils import (
     run_command_hiding_token,
 )
-from tests_integration.lib.shared import (
+
+from ._shared import (
     FEEDSTOCK_SUFFIX,
     GitHubAccount,
     get_github_token,

tests_integration/step_prepare.py renamed to tests_integration/lib/_run_test_cases.py

@@ -1,23 +1,13 @@
-"""
-After closing all open Pull Requests in the conda-forge staging organization,
-runs the prepare() function of all test cases of the current test scenario to prepare the test environment.
-
-Expects the scenario ID to be present in the environment variable named SCENARIO_ID.
-"""
-
 import logging
 from pathlib import Path
 
 from github import Github
 
 from conda_forge_tick.settings import settings
-from tests_integration.lib.integration_test_helper import IntegrationTestHelper
-from tests_integration.lib.shared import (
-    FEEDSTOCK_SUFFIX,
-    GitHubAccount,
-    get_github_token,
-)
-from tests_integration.lib.test_case import TestCase
+from tests_integration.lib import IntegrationTestHelper
+
+from ._shared import FEEDSTOCK_SUFFIX, GitHubAccount, get_github_token
+from ._test_case import TestCase
 
 LOGGER = logging.getLogger(__name__)
 
@@ -50,3 +40,10 @@ def run_all_prepare_functions(scenario: dict[str, TestCase]):
     for feedstock_name, test_case in scenario.items():
         LOGGER.info("Preparing %s...", feedstock_name)
         test_case.prepare(test_helper)
+
+
+def run_all_validate_functions(scenario: dict[str, TestCase]):
+    test_helper = IntegrationTestHelper()
+    for feedstock_name, test_case in scenario.items():
+        LOGGER.info("Validating %s...", feedstock_name)
+        test_case.validate(test_helper)

tests_integration/setup_repositories.py renamed to tests_integration/lib/_setup_repositories.py

@@ -6,8 +6,6 @@
 test cases themselves because tests could purposefully rely on the actual bot itself to create repositories.
 
 However, we do delete unnecessary feedstocks from the bot's user account.
-
-After the repositories are set up, we write a list of all test scenario ids to be run to $GITHUB_OUTPUT.
 """
 
 import logging
@@ -19,7 +17,8 @@
 from github.Repository import Repository
 
 from tests_integration.definitions import TEST_CASE_MAPPING
-from tests_integration.lib.shared import (
+
+from ._shared import (
     FEEDSTOCK_SUFFIX,
     REGRO_ACCOUNT_REPOS,
     GitHubAccount,
@@ -29,10 +28,6 @@
 
 LOGGER = logging.getLogger(__name__)
 
-IGNORE_FEEDSTOCK_NAMES = {
-    "__pycache__",
-}
-
 
 @dataclass(frozen=True)
 class GitHubAccountSetup:

tests_integration/lib/shared.py renamed to tests_integration/lib/_shared.py

@@ -1,7 +1,6 @@
 import logging
 import os
 from enum import StrEnum
-from pathlib import Path
 
 from fastapi import APIRouter
 
@@ -28,20 +27,12 @@ class GitHubAccount(StrEnum):
 
 REGRO_ACCOUNT_REPOS = {"cf-graph-countyfair"}
 
-ENV_GITHUB_OUTPUT = "GITHUB_OUTPUT"
 ENV_GITHUB_RUN_ID = "GITHUB_RUN_ID"
 """
 Used as a random seed for the integration tests.
 """
 ENV_TEST_SCENARIO_ID = "SCENARIO_ID"
 
-GITHUB_OUTPUT_KEY_SCENARIO_IDS = "scenario_ids"
-
-TESTS_INTEGRATION_DIR_NAME = "tests_integration"
-DEFINITIONS_DIR_NAME = "definitions"
-
-DEFINITIONS_DIR = Path(__file__).parents[1] / DEFINITIONS_DIR_NAME
-
 FEEDSTOCK_SUFFIX = "-feedstock"
 
 

tests_integration/lib/test_case.py renamed to tests_integration/lib/_test_case.py

@@ -2,7 +2,7 @@
 
 from fastapi import APIRouter
 
-from tests_integration.lib.integration_test_helper import IntegrationTestHelper
+from ._integration_test_helper import IntegrationTestHelper
 
 
 class TestCase(ABC):

tests_integration/mock_server_addon.py

@@ -16,12 +16,12 @@
 from mitmproxy.addons import asgiapp
 from mitmproxy.http import HTTPFlow
 
-from tests_integration.collect_test_scenarios import get_test_scenario
-from tests_integration.lib.shared import (
+from tests_integration.lib import (
     ENV_TEST_SCENARIO_ID,
     VIRTUAL_PROXY_HOSTNAME,
     VIRTUAL_PROXY_PORT,
     get_global_router,
+    get_test_scenario,
     get_transparent_urls,
 )