♻️ Split director-v2 02 into 02 and 03

.github/workflows/ci-testing-deploy.yml

@@ -1614,6 +1614,52 @@ jobs:
         with:
           flags: integrationtests #optional
 
+  integration-test-director-v2-03:
+    needs: [changes, build-test-images]
+    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.director-v2 == 'true' || github.event_name == 'push' }}
+    timeout-minutes: 30 # if this timeout gets too small, then split the tests
+    name: "[int] director-v2 03"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: ["3.11"]
+        os: [ubuntu-24.04]
+      fail-fast: false
+    env:
+      # NOTE: DIRECTOR_DEFAULT_MAX_* used for integration-tests that include `director` service
+      DIRECTOR_DEFAULT_MAX_MEMORY: 268435456
+      DIRECTOR_DEFAULT_MAX_NANO_CPUS: 10000000
+    steps:
+      - uses: actions/checkout@v5
+      - name: Setup SimCore environment
+        uses: ./.github/actions/setup-simcore-env
+        with:
+          python-version: ${{ matrix.python }}
+          cache-dependency-glob: "**/director-v2/requirements/ci.txt"
+      - name: setup rclone docker volume plugin
+        run: sudo ./ci/github/helpers/install_rclone_docker_volume_plugin.bash
+      - name: Download and load Docker images
+        uses: ./.github/actions/download-load-docker-images
+        with:
+          artifact-name-pattern: 'backend'
+      - name: install rclone
+        run: sudo ./ci/github/helpers/install_rclone.bash
+      - name: install
+        run: ./ci/github/integration-testing/director-v2.bash install
+      - name: test
+        run: ./ci/github/integration-testing/director-v2.bash test 03
+      - name: upload failed tests logs
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ github.job }}_docker_logs
+          path: ./services/director-v2/test_failures
+      - uses: codecov/codecov-action@v5
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        with:
+          flags: integrationtests #optional
+
   integration-test-dynamic-sidecar:
     needs: [changes, build-test-images]
     if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.dynamic-sidecar == 'true' || github.event_name == 'push'}}
@@ -1740,6 +1786,7 @@ jobs:
       [
         integration-test-director-v2-01,
         integration-test-director-v2-02,
+        integration-test-director-v2-03,
         integration-test-dynamic-sidecar,
         integration-test-docker-api-proxy,
         integration-test-simcore-sdk,

services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py

@@ -29,12 +29,7 @@
 from models_library.api_schemas_directorv2.computations import ComputationGet
 from models_library.clusters import ClusterAuthentication
 from models_library.products import ProductName
-from models_library.projects import (
-    Node,
-    NodesDict,
-    ProjectAtDB,
-    ProjectID,
-)
+from models_library.projects import Node, NodesDict, ProjectAtDB, ProjectID
 from models_library.projects_networks import (
     PROJECT_NETWORK_PREFIX,
     ContainerAliases,
@@ -47,6 +42,19 @@
 from models_library.users import UserID
 from pydantic import AnyHttpUrl, TypeAdapter
 from pytest_mock.plugin import MockerFixture
+from pytest_simcore.directorv2_integration_utils import (
+    assert_all_services_running,
+    assert_retrieve_service,
+    assert_services_reply_200,
+    assert_start_service,
+    assert_stop_service,
+    ensure_network_cleanup,
+    ensure_volume_cleanup,
+    is_legacy,
+    patch_dynamic_service_url,
+    run_command,
+    sleep_for,
+)
 from pytest_simcore.helpers.host import get_localhost_ip
 from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict
 from pytest_simcore.helpers.typing_env import EnvVarsDict
@@ -86,19 +94,6 @@
 from tenacity.retry import retry_if_exception_type
 from tenacity.stop import stop_after_attempt, stop_after_delay
 from tenacity.wait import wait_fixed
-from utils import (
-    assert_all_services_running,
-    assert_retrieve_service,
-    assert_services_reply_200,
-    assert_start_service,
-    assert_stop_service,
-    ensure_network_cleanup,
-    ensure_volume_cleanup,
-    is_legacy,
-    patch_dynamic_service_url,
-    run_command,
-    sleep_for,
-)
 from yarl import URL
 
 pytest_simcore_core_services_selection = [

services/director-v2/tests/integration/03/conftest.py

@@ -0,0 +1,91 @@
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+
+from collections.abc import AsyncIterator
+from uuid import uuid4
+
+import aiodocker
+import pytest
+from models_library.api_schemas_resource_usage_tracker.pricing_plans import (
+    RutPricingPlanGet,
+)
+from models_library.projects_networks import ProjectsNetworks
+from models_library.services_resources import (
+    ServiceResourcesDict,
+    ServiceResourcesDictHelpers,
+)
+from pydantic import TypeAdapter
+from pytest_mock.plugin import MockerFixture
+
+
+@pytest.fixture(scope="session")
+def network_name() -> str:
+    return "pytest-simcore_interactive_services_subnet"
+
+
+@pytest.fixture
+async def ensure_swarm_and_networks(
+    network_name: str, docker_swarm: None
+) -> AsyncIterator[None]:
+    """
+    Make sure to always have a docker swarm network.
+    If one is not present crete one. There can not be more then one.
+    """
+
+    async with aiodocker.Docker() as docker_client:
+        # if network dose not exist create and remove it
+        create_and_remove_network = True
+        for network_data in await docker_client.networks.list():
+            if network_data["Name"] == network_name:
+                create_and_remove_network = False
+                break
+        docker_network = None
+        if create_and_remove_network:
+            network_config = {
+                "Name": network_name,
+                "Driver": "overlay",
+                "Attachable": True,
+                "Internal": False,
+                "Scope": "swarm",
+            }
+            docker_network = await docker_client.networks.create(network_config)
+
+        yield
+
+        if create_and_remove_network and docker_network:
+            network = await docker_client.networks.get(docker_network.id)
+            assert await network.delete() is True
+
+
+@pytest.fixture
+def mock_projects_networks_repository(mocker: MockerFixture) -> None:
+    mocker.patch(
+        (
+            "simcore_service_director_v2.modules.db.repositories."
+            "projects_networks.ProjectsNetworksRepository.get_projects_networks"
+        ),
+        return_value=ProjectsNetworks.model_validate(
+            {"project_uuid": uuid4(), "networks_with_aliases": {}}
+        ),
+    )
+
+
+@pytest.fixture
+def service_resources() -> ServiceResourcesDict:
+    return TypeAdapter(ServiceResourcesDict).validate_python(
+        ServiceResourcesDictHelpers.model_config["json_schema_extra"]["examples"][0],
+    )
+
+
+@pytest.fixture
+def mock_resource_usage_tracker(mocker: MockerFixture) -> None:
+    base_module = "simcore_service_director_v2.modules.resource_usage_tracker_client"
+    service_pricing_plan = RutPricingPlanGet.model_validate(
+        RutPricingPlanGet.model_config["json_schema_extra"]["examples"][1]
+    )
+    for unit in service_pricing_plan.pricing_units:
+        unit.specific_info.aws_ec2_instances.clear()
+    mocker.patch(
+        f"{base_module}.ResourceUsageTrackerClient.get_default_service_pricing_plan",
+        return_value=service_pricing_plan,
+    )

services/director-v2/tests/integration/02/test_dynamic_services_routes.py → services/director-v2/tests/integration/03/test_dynamic_services_routes.py

@@ -26,6 +26,10 @@
 )
 from models_library.users import UserID
 from pytest_mock.plugin import MockerFixture
+from pytest_simcore.directorv2_integration_utils import (
+    ensure_network_cleanup,
+    patch_dynamic_service_url,
+)
 from pytest_simcore.helpers.host import get_localhost_ip
 from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict
 from pytest_simcore.helpers.typing_env import EnvVarsDict
@@ -42,7 +46,6 @@
 from tenacity.retry import retry_if_exception_type
 from tenacity.stop import stop_after_delay
 from tenacity.wait import wait_fixed
-from utils import ensure_network_cleanup, patch_dynamic_service_url
 from yarl import URL
 
 SERVICE_IS_READY_TIMEOUT = 2 * 60

services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py → services/director-v2/tests/integration/03/test_mixed_dynamic_sidecar_and_legacy_project.py

@@ -21,12 +21,7 @@
 from models_library.services_resources import ServiceResourcesDict
 from models_library.users import UserID
 from pytest_mock.plugin import MockerFixture
-from pytest_simcore.helpers.host import get_localhost_ip
-from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict
-from pytest_simcore.helpers.typing_env import EnvVarsDict
-from settings_library.rabbit import RabbitSettings
-from settings_library.redis import RedisSettings
-from utils import (
+from pytest_simcore.directorv2_integration_utils import (
     assert_all_services_running,
     assert_services_reply_200,
     assert_start_service,
@@ -35,6 +30,11 @@
     is_legacy,
     patch_dynamic_service_url,
 )
+from pytest_simcore.helpers.host import get_localhost_ip
+from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict
+from pytest_simcore.helpers.typing_env import EnvVarsDict
+from settings_library.rabbit import RabbitSettings
+from settings_library.redis import RedisSettings
 from yarl import URL
 
 logger = logging.getLogger(__name__)