♻️ Maintenance: improve CI reliability + add healthcheck in migration service (ITISFoundation#3365)

Author: sanderegg

packages/pytest-simcore/src/pytest_simcore/docker_compose.py

@@ -66,6 +66,13 @@ def testing_environ_vars(env_devel_file: Path) -> EnvVarsDict:
         "SWARM_STACK_NAME_NO_HYPHEN", env_devel["SWARM_STACK_NAME"].replace("-", "_")
     )
 
+    env_devel[
+        "AIOCACHE_DISABLE"
+    ] = "1"  # ensure that aio-caches are disabled for testing [https://aiocache.readthedocs.io/en/latest/testing.html]
+    env_devel[
+        "CATALOG_BACKGROUND_TASK_REST_TIME"
+    ] = "1"  # ensure catalog refreshes services access rights fast
+
     env_devel["DIRECTOR_REGISTRY_CACHING"] = "False"
     env_devel.setdefault("DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS", "")
     env_devel.setdefault("DIRECTOR_SELF_SIGNED_SSL_SECRET_ID", "")

packages/simcore-sdk/Makefile

@@ -80,10 +80,11 @@ tests-integration-ci: ## runs integration tests against local+production images
 		--cov-report=xml \
 		--cov=simcore_sdk \
 		--durations=10 \
+		--keep-docker-up \
 		--log-date-format="%Y-%m-%d %H:%M:%S" \
-    --log-format="%(asctime)s %(levelname)s %(message)s" \
 		--verbose \
 		-m "not heavy_load" \
+		--log-format="%(asctime)s %(levelname)s %(message)s" \
 		$(CURDIR)/tests/integration
 
 tests: tests-unit tests-integration ## runs all tests

services/catalog/tests/unit/with_dbs/conftest.py

@@ -6,7 +6,7 @@
 import itertools
 import random
 from random import randint
-from typing import Any, AsyncIterator, Callable, Iterable, Iterator
+from typing import Any, AsyncIterator, Awaitable, Callable, Iterable, Iterator
 
 import pytest
 import respx
@@ -24,6 +24,7 @@
     services_access_rights,
     services_meta_data,
 )
+from sqlalchemy import tuple_
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.ext.asyncio import AsyncEngine
 from starlette.testclient import TestClient
@@ -178,7 +179,7 @@ async def user_groups_ids(
 @pytest.fixture()
 async def services_db_tables_injector(
     sqlalchemy_async_engine: AsyncEngine,
-) -> AsyncIterator[Callable]:
+) -> AsyncIterator[Callable[[list[tuple]], Awaitable[None]]]:
     """Returns a helper function to init
     services_meta_data and services_access_rights tables
 
@@ -205,6 +206,7 @@ async def services_db_tables_injector(
         )
     """
     # pylint: disable=no-value-for-parameter
+    inserted_services: set[tuple[str, str]] = set()
 
     async def inject_in_db(fake_catalog: list[tuple]):
         # [(service, ar1, ...), (service2, ar1, ...) ]
@@ -221,6 +223,7 @@ async def inject_in_db(fake_catalog: list[tuple]):
                     set_=service,
                 )
                 await conn.execute(upsert_meta)
+                inserted_services.add((service["key"], service["version"]))
 
             for access_rights in itertools.chain(items[1:] for items in fake_catalog):
                 stmt_access = services_access_rights.insert().values(access_rights)
@@ -229,8 +232,13 @@ async def inject_in_db(fake_catalog: list[tuple]):
     yield inject_in_db
 
     async with sqlalchemy_async_engine.begin() as conn:
-        await conn.execute(services_access_rights.delete())
-        await conn.execute(services_meta_data.delete())
+        await conn.execute(
+            services_meta_data.delete().where(
+                tuple_(services_meta_data.c.key, services_meta_data.c.version).in_(
+                    inserted_services
+                )
+            )
+        )
 
 
 @pytest.fixture()

services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py

@@ -164,7 +164,7 @@ async def director_v2_client(
     monkeypatch.setenv("S3_ACCESS_KEY", minio_config["client"]["access_key"])
     monkeypatch.setenv("S3_SECRET_KEY", minio_config["client"]["secret_key"])
     monkeypatch.setenv("S3_BUCKET_NAME", minio_config["bucket_name"])
-    monkeypatch.setenv("S3_SECURE", minio_config["client"]["secure"])
+    monkeypatch.setenv("S3_SECURE", f"{minio_config['client']['secure']}")
 
     # patch host for dynamic-sidecar, not reachable via localhost
     # the dynamic-sidecar (running inside a container) will use

services/migration/Dockerfile

@@ -74,5 +74,7 @@ USER ${SC_USER_NAME}
 # bring installed package without build tools
 COPY --from=build ${VIRTUAL_ENV} ${VIRTUAL_ENV}
 COPY --chown=scu:scu services/migration/docker services/migration/docker
+#, "| grep --regexp='^Rev: .* (head)$'" ]
+HEALTHCHECK --interval=10s --timeout=10s --start-period=1ms --retries=3 CMD [ "/bin/sh", "/home/scu/services/migration/docker/healthcheck.sh"]
 
 ENTRYPOINT [ "/bin/sh", "services/migration/docker/entrypoint.sh" ]

services/migration/docker/healthcheck.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec sc-pg info | grep --regexp='^Rev: .* (head)$'

services/web/server/tests/unit/with_dbs/01/conftest.py

@@ -0,0 +1,16 @@
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+# pylint: disable=unused-variable
+
+
+import pytest
+from pytest import MonkeyPatch
+
+
+@pytest.fixture
+def app_environment(
+    app_environment: dict[str, str], monkeypatch: MonkeyPatch
+) -> dict[str, str]:
+    # NOTE: overrides app_environment
+    monkeypatch.setenv("WEBSERVER_GARBAGE_COLLECTOR", "null")
+    return app_environment | {"WEBSERVER_GARBAGE_COLLECTOR": "null"}

services/web/server/tests/unit/with_dbs/02/conftest.py

@@ -16,11 +16,21 @@
     ServiceResourcesDictHelpers,
 )
 from pydantic import parse_obj_as
+from pytest import MonkeyPatch
 from pytest_simcore.helpers.utils_assert import assert_status
 from pytest_simcore.helpers.utils_projects import NewProject, delete_all_projects
 from simcore_service_webserver import catalog
 
 
+@pytest.fixture
+def app_environment(
+    app_environment: dict[str, str], monkeypatch: MonkeyPatch
+) -> dict[str, str]:
+    # NOTE: overrides app_environment
+    monkeypatch.setenv("WEBSERVER_GARBAGE_COLLECTOR", "null")
+    return app_environment | {"WEBSERVER_GARBAGE_COLLECTOR": "null"}
+
+
 @pytest.fixture
 def mock_service_resources() -> ServiceResourcesDict:
     return parse_obj_as(

services/web/server/tests/unit/with_dbs/03/test_resource_manager.py renamed to services/web/server/tests/unit/with_dbs/03/garbage_collector/test_resource_manager.py

@@ -30,6 +30,7 @@
 from aiohttp import web
 from aiohttp.test_utils import TestClient, TestServer
 from pydantic import ByteSize, parse_obj_as
+from pytest_mock.plugin import MockerFixture
 from pytest_simcore.helpers.utils_dict import ConfigDict
 from pytest_simcore.helpers.utils_login import NewUser
 from servicelib.aiohttp.application_keys import APP_DB_ENGINE_KEY
@@ -254,7 +255,7 @@ def asyncpg_storage_system_mock(mocker):
 
 
 @pytest.fixture
-async def mocked_director_v2_api(mocker) -> dict[str, MagicMock]:
+async def mocked_director_v2_api(mocker: MockerFixture) -> dict[str, MagicMock]:
     mock = {}
 
     #