diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index 678ce3905413..613698469a72 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -1614,6 +1614,52 @@ jobs: with: flags: integrationtests #optional + integration-test-director-v2-03: + needs: [changes, build-test-images] + if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.director-v2 == 'true' || github.event_name == 'push' }} + timeout-minutes: 30 # if this timeout gets too small, then split the tests + name: "[int] director-v2 03" + runs-on: ${{ matrix.os }} + strategy: + matrix: + python: ["3.11"] + os: [ubuntu-24.04] + fail-fast: false + env: + # NOTE: DIRECTOR_DEFAULT_MAX_* used for integration-tests that include `director` service + DIRECTOR_DEFAULT_MAX_MEMORY: 268435456 + DIRECTOR_DEFAULT_MAX_NANO_CPUS: 10000000 + steps: + - uses: actions/checkout@v5 + - name: Setup SimCore environment + uses: ./.github/actions/setup-simcore-env + with: + python-version: ${{ matrix.python }} + cache-dependency-glob: "**/director-v2/requirements/ci.txt" + - name: setup rclone docker volume plugin + run: sudo ./ci/github/helpers/install_rclone_docker_volume_plugin.bash + - name: Download and load Docker images + uses: ./.github/actions/download-load-docker-images + with: + artifact-name-pattern: 'backend' + - name: install rclone + run: sudo ./ci/github/helpers/install_rclone.bash + - name: install + run: ./ci/github/integration-testing/director-v2.bash install + - name: test + run: ./ci/github/integration-testing/director-v2.bash test 03 + - name: upload failed tests logs + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: ${{ github.job }}_docker_logs + path: ./services/director-v2/test_failures + - uses: codecov/codecov-action@v5 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: integrationtests #optional + integration-test-dynamic-sidecar: needs: [changes, build-test-images] if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.dynamic-sidecar == 'true' || github.event_name == 'push'}} @@ -1740,6 +1786,7 @@ jobs: [ integration-test-director-v2-01, integration-test-director-v2-02, + integration-test-director-v2-03, integration-test-dynamic-sidecar, integration-test-docker-api-proxy, integration-test-simcore-sdk, diff --git a/services/director-v2/tests/integration/02/utils.py b/packages/pytest-simcore/src/pytest_simcore/directorv2_integration_utils.py similarity index 100% rename from services/director-v2/tests/integration/02/utils.py rename to packages/pytest-simcore/src/pytest_simcore/directorv2_integration_utils.py diff --git a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py index aa70662048c9..2d276c52ac37 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py +++ b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py @@ -29,12 +29,7 @@ from models_library.api_schemas_directorv2.computations import ComputationGet from models_library.clusters import ClusterAuthentication from models_library.products import ProductName -from models_library.projects import ( - Node, - NodesDict, - ProjectAtDB, - ProjectID, -) +from models_library.projects import Node, NodesDict, ProjectAtDB, ProjectID from models_library.projects_networks import ( PROJECT_NETWORK_PREFIX, ContainerAliases, @@ -47,6 +42,19 @@ from models_library.users import UserID from pydantic import AnyHttpUrl, TypeAdapter from pytest_mock.plugin import MockerFixture +from pytest_simcore.directorv2_integration_utils import ( + assert_all_services_running, + assert_retrieve_service, + assert_services_reply_200, + assert_start_service, + assert_stop_service, + ensure_network_cleanup, + ensure_volume_cleanup, + is_legacy, + patch_dynamic_service_url, + run_command, + sleep_for, +) from pytest_simcore.helpers.host import get_localhost_ip from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict @@ -86,19 +94,6 @@ from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_attempt, stop_after_delay from tenacity.wait import wait_fixed -from utils import ( - assert_all_services_running, - assert_retrieve_service, - assert_services_reply_200, - assert_start_service, - assert_stop_service, - ensure_network_cleanup, - ensure_volume_cleanup, - is_legacy, - patch_dynamic_service_url, - run_command, - sleep_for, -) from yarl import URL pytest_simcore_core_services_selection = [ diff --git a/services/director-v2/tests/integration/03/conftest.py b/services/director-v2/tests/integration/03/conftest.py new file mode 100644 index 000000000000..4e0b6a5b31f4 --- /dev/null +++ b/services/director-v2/tests/integration/03/conftest.py @@ -0,0 +1,91 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument + +from collections.abc import AsyncIterator +from uuid import uuid4 + +import aiodocker +import pytest +from models_library.api_schemas_resource_usage_tracker.pricing_plans import ( + RutPricingPlanGet, +) +from models_library.projects_networks import ProjectsNetworks +from models_library.services_resources import ( + ServiceResourcesDict, + ServiceResourcesDictHelpers, +) +from pydantic import TypeAdapter +from pytest_mock.plugin import MockerFixture + + +@pytest.fixture(scope="session") +def network_name() -> str: + return "pytest-simcore_interactive_services_subnet" + + +@pytest.fixture +async def ensure_swarm_and_networks( + network_name: str, docker_swarm: None +) -> AsyncIterator[None]: + """ + Make sure to always have a docker swarm network. + If one is not present crete one. There can not be more then one. + """ + + async with aiodocker.Docker() as docker_client: + # if network dose not exist create and remove it + create_and_remove_network = True + for network_data in await docker_client.networks.list(): + if network_data["Name"] == network_name: + create_and_remove_network = False + break + docker_network = None + if create_and_remove_network: + network_config = { + "Name": network_name, + "Driver": "overlay", + "Attachable": True, + "Internal": False, + "Scope": "swarm", + } + docker_network = await docker_client.networks.create(network_config) + + yield + + if create_and_remove_network and docker_network: + network = await docker_client.networks.get(docker_network.id) + assert await network.delete() is True + + +@pytest.fixture +def mock_projects_networks_repository(mocker: MockerFixture) -> None: + mocker.patch( + ( + "simcore_service_director_v2.modules.db.repositories." + "projects_networks.ProjectsNetworksRepository.get_projects_networks" + ), + return_value=ProjectsNetworks.model_validate( + {"project_uuid": uuid4(), "networks_with_aliases": {}} + ), + ) + + +@pytest.fixture +def service_resources() -> ServiceResourcesDict: + return TypeAdapter(ServiceResourcesDict).validate_python( + ServiceResourcesDictHelpers.model_config["json_schema_extra"]["examples"][0], + ) + + +@pytest.fixture +def mock_resource_usage_tracker(mocker: MockerFixture) -> None: + base_module = "simcore_service_director_v2.modules.resource_usage_tracker_client" + service_pricing_plan = RutPricingPlanGet.model_validate( + RutPricingPlanGet.model_config["json_schema_extra"]["examples"][1] + ) + for unit in service_pricing_plan.pricing_units: + unit.specific_info.aws_ec2_instances.clear() + mocker.patch( + f"{base_module}.ResourceUsageTrackerClient.get_default_service_pricing_plan", + return_value=service_pricing_plan, + ) diff --git a/services/director-v2/tests/integration/02/test_dynamic_services_routes.py b/services/director-v2/tests/integration/03/test_dynamic_services_routes.py similarity index 99% rename from services/director-v2/tests/integration/02/test_dynamic_services_routes.py rename to services/director-v2/tests/integration/03/test_dynamic_services_routes.py index dd1ebb2e2f73..27c6286e2620 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_services_routes.py +++ b/services/director-v2/tests/integration/03/test_dynamic_services_routes.py @@ -26,6 +26,10 @@ ) from models_library.users import UserID from pytest_mock.plugin import MockerFixture +from pytest_simcore.directorv2_integration_utils import ( + ensure_network_cleanup, + patch_dynamic_service_url, +) from pytest_simcore.helpers.host import get_localhost_ip from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict @@ -42,7 +46,6 @@ from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed -from utils import ensure_network_cleanup, patch_dynamic_service_url from yarl import URL SERVICE_IS_READY_TIMEOUT = 2 * 60 diff --git a/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py b/services/director-v2/tests/integration/03/test_mixed_dynamic_sidecar_and_legacy_project.py similarity index 99% rename from services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py rename to services/director-v2/tests/integration/03/test_mixed_dynamic_sidecar_and_legacy_project.py index 2d0abc8e74f9..3c09d518bb41 100644 --- a/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py +++ b/services/director-v2/tests/integration/03/test_mixed_dynamic_sidecar_and_legacy_project.py @@ -21,12 +21,7 @@ from models_library.services_resources import ServiceResourcesDict from models_library.users import UserID from pytest_mock.plugin import MockerFixture -from pytest_simcore.helpers.host import get_localhost_ip -from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict -from pytest_simcore.helpers.typing_env import EnvVarsDict -from settings_library.rabbit import RabbitSettings -from settings_library.redis import RedisSettings -from utils import ( +from pytest_simcore.directorv2_integration_utils import ( assert_all_services_running, assert_services_reply_200, assert_start_service, @@ -35,6 +30,11 @@ is_legacy, patch_dynamic_service_url, ) +from pytest_simcore.helpers.host import get_localhost_ip +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict +from settings_library.rabbit import RabbitSettings +from settings_library.redis import RedisSettings from yarl import URL logger = logging.getLogger(__name__)