♻️ Cleanup of setup and tests tooling (#5446)

Author: pcrespov

api/tests/test_individual_openapi_schemas.py

@@ -7,8 +7,6 @@
 from pathlib import Path
 
 import pytest
-from openapi_spec_validator import validate_spec
-from openapi_spec_validator.exceptions import OpenAPISpecValidatorError
 from utils import dump_specs, is_json_schema, is_openapi_schema, load_specs
 
 # Conventions
@@ -132,15 +130,3 @@ def converted_specs_testdir(api_specs_dir, all_api_specs_tails, tmpdir_factory):
             shutil.copy2(basedir / tail, testdir / tail)
 
     return testdir
-
-
-@pytest.mark.skip(reason="Implementing in PR 324")
-def test_valid_individual_openapi_specs(api_specs_tail, converted_specs_testdir):
-    # NOTE: api_specs_tail is a parametrized **fixture**
-    #
-    api_specs_path = converted_specs_testdir / api_specs_tail
-    try:
-        specs = load_specs(api_specs_path)
-        validate_spec(specs, spec_url=api_specs_path.as_uri())
-    except OpenAPISpecValidatorError as err:
-        pytest.fail(f"Failed validating {api_specs_path}:\n{err.message}")

packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py

@@ -9,7 +9,6 @@
 
 import docker
 import yaml
-from pytest_simcore.helpers.typing_env import EnvVarsDict
 from tenacity import retry
 from tenacity.after import after_log
 from tenacity.stop import stop_after_attempt
@@ -61,17 +60,19 @@ def get_service_published_port(
 
     services = [s for s in client.services.list() if str(s.name).endswith(service_name)]
     if not services:
-        raise RuntimeError(
+        msg = (
             f"Cannot find published port for service '{service_name}'."
             "Probably services still not started."
         )
+        raise RuntimeError(msg)
 
     service_ports = services[0].attrs["Endpoint"].get("Ports")
     if not service_ports:
-        raise RuntimeError(
+        msg = (
             f"Cannot find published port for service '{service_name}' in endpoint."
             "Probably services still not started."
         )
+        raise RuntimeError(msg)
 
     published_port = None
     msg = ", ".join(
@@ -89,7 +90,7 @@ def get_service_published_port(
 
     else:
         ports_to_look_for: list = (
-            [target_ports] if isinstance(target_ports, (int, str)) else target_ports
+            [target_ports] if isinstance(target_ports, int | str) else target_ports
         )
 
         for target_port in ports_to_look_for:
@@ -100,7 +101,8 @@ def get_service_published_port(
                     break
 
     if published_port is None:
-        raise RuntimeError(f"Cannot find published port for {target_ports}. Got {msg}")
+        msg = f"Cannot find published port for {target_ports}. Got {msg}"
+        raise RuntimeError(msg)
 
     return str(published_port)
 
@@ -111,7 +113,6 @@ def run_docker_compose_config(
     project_dir: Path,
     env_file_path: Path,
     destination_path: Path | None = None,
-    additional_envs: EnvVarsDict | None = None,
 ) -> dict:
     """Runs docker compose config to validate and resolve a compose file configuration
 
@@ -140,13 +141,12 @@ def run_docker_compose_config(
         ], "Expected yaml/yml file as destination path"
 
     # SEE https://docs.docker.com/compose/reference/
-
-    global_options = [
+    bash_options = [
         "-p",
         str(project_dir),  # Specify an alternate working directory
     ]
     # https://docs.docker.com/compose/environment-variables/#using-the---env-file--option
-    global_options += [
+    bash_options += [
         "-e",
         str(env_file_path),  # Custom environment variables
     ]
@@ -155,26 +155,22 @@ def run_docker_compose_config(
     #  - When you use multiple Compose files, all paths in the files are relative to the first configuration file specified with -f.
     #    You can use the --project-directory option to override this base path.
     for docker_compose_path in docker_compose_paths:
-        global_options += [os.path.relpath(docker_compose_path, project_dir)]
+        bash_options += [os.path.relpath(docker_compose_path, project_dir)]
 
     # SEE https://docs.docker.com/compose/reference/config/
     docker_compose_path = scripts_dir / "docker" / "docker-compose-config.bash"
     assert docker_compose_path.exists()
 
-    cmd = [f"{docker_compose_path}"] + global_options
-    print(" ".join(cmd))
-
-    process_environment_variables = dict(os.environ)
-    if additional_envs:
-        process_environment_variables |= additional_envs
+    args = [f"{docker_compose_path}", *bash_options]
+    print(" ".join(args))
 
     process = subprocess.run(
-        cmd,
+        args,
         shell=False,
-        check=True,
         cwd=project_dir,
         capture_output=True,
-        env=process_environment_variables,
+        check=True,
+        env=None,  # NOTE: Do not use since since we pass all necessary env vars via --env-file option of  docker compose
     )
 
     compose_file_str = process.stdout.decode("utf-8")

packages/pytest-simcore/src/pytest_simcore/helpers/utils_postgres.py

@@ -1,6 +1,7 @@
 import logging
+from collections.abc import Iterator
 from contextlib import contextmanager
-from typing import Iterator, TypedDict
+from typing import TypedDict
 
 import simcore_postgres_database.cli
 import sqlalchemy as sa

packages/service-library/setup.py

@@ -30,23 +30,23 @@ def read_reqs(reqs_path: Path) -> set[str]:
 
 
 SETUP = {
+    "name": "simcore-service-library",
+    "version": Path(CURRENT_DIR / "VERSION").read_text().strip(),
     "author": "Pedro Crespo-Valero (pcrespov)",
     "description": "Core service library for simcore (or servicelib)",
+    "license": "MIT license",
+    "python_requires": "~=3.10",
+    "install_requires": tuple(PROD_REQUIREMENTS),
+    "packages": find_packages(where="src"),
+    "package_dir": {"": "src"},
+    "test_suite": "tests",
+    "tests_require": tuple(TEST_REQUIREMENTS),
     "extras_require": {
         "aiohttp": tuple(AIOHTTP_REQUIREMENTS),
-        "all": tuple(AIOHTTP_REQUIREMENTS | FASTAPI_REQUIREMENTS),
         "fastapi": tuple(FASTAPI_REQUIREMENTS),
+        "all": tuple(AIOHTTP_REQUIREMENTS | FASTAPI_REQUIREMENTS),
         "test": tuple(TEST_REQUIREMENTS),
     },
-    "install_requires": tuple(PROD_REQUIREMENTS),
-    "license": "MIT license",
-    "name": "simcore-service-library",
-    "package_dir": {"": "src"},
-    "packages": find_packages(where="src"),
-    "python_requires": "~=3.10",
-    "test_suite": "tests",
-    "tests_require": tuple(TEST_REQUIREMENTS),
-    "version": Path(CURRENT_DIR / "VERSION").read_text().strip(),
 }
 
 

packages/service-library/tests/fastapi/test_httpx_utils.py

@@ -62,7 +62,7 @@ async def test_to_curl_command(client: AsyncClient):
 
     assert (
         cmd_short
-        == f'curl -X POST -H "host: test_base_http_api" -H "accept: */*" -H "accept-encoding: gzip, deflate" -H "connection: keep-alive" -H "user-agent: python-httpx/0.25.0" -H "x-secret: {_PLACEHOLDER}" -H "content-length: 9" -H "content-type: application/json" -d \'{{"y": 12}}\' https://test_base_http_api/foo?x=3'
+        == f'curl -X POST -H "host: test_base_http_api" -H "accept: */*" -H "accept-encoding: gzip, deflate" -H "connection: keep-alive" -H "user-agent: python-httpx/{httpx.__version__}" -H "x-secret: {_PLACEHOLDER}" -H "content-length: 9" -H "content-type: application/json" -d \'{{"y": 12}}\' https://test_base_http_api/foo?x=3'
     )
 
     cmd_long = to_curl_command(response.request, use_short_options=False)
@@ -81,14 +81,14 @@ async def test_to_curl_command(client: AsyncClient):
     assert (
         cmd_multiline
         == textwrap.dedent(
-            """\
+            f"""\
         curl \\
         -X GET \\
         -H "host: test_base_http_api" \\
         -H "accept: */*" \\
         -H "accept-encoding: gzip, deflate" \\
         -H "connection: keep-alive" \\
-        -H "user-agent: python-httpx/0.25.0" \\
+        -H "user-agent: python-httpx/{httpx.__version__}" \\
         https://test_base_http_api/foo?x=3
         """
         ).strip()
@@ -115,5 +115,5 @@ async def test_to_httpx_command(client: AsyncClient):
     print(cmd_short)
     assert (
         cmd_short
-        == f'httpx -m POST -c \'{{"y": 12}}\' -h "host" "test_base_http_api" -h "accept" "*/*" -h "accept-encoding" "gzip, deflate" -h "connection" "keep-alive" -h "user-agent" "python-httpx/0.25.0" -h "x-secret" "{_PLACEHOLDER}" -h "content-length" "9" -h "content-type" "application/json" https://test_base_http_api/foo?x=3'
+        == f'httpx -m POST -c \'{{"y": 12}}\' -h "host" "test_base_http_api" -h "accept" "*/*" -h "accept-encoding" "gzip, deflate" -h "connection" "keep-alive" -h "user-agent" "python-httpx/{httpx.__version__}" -h "x-secret" "{_PLACEHOLDER}" -h "content-length" "9" -h "content-type" "application/json" https://test_base_http_api/foo?x=3'
     )

packages/simcore-sdk/setup.py

@@ -1,12 +1,11 @@
 import re
 import sys
 from pathlib import Path
-from typing import Set
 
 from setuptools import find_packages, setup
 
 
-def read_reqs(reqs_path: Path) -> Set[str]:
+def read_reqs(reqs_path: Path) -> set[str]:
     return {
         r
         for r in re.findall(
@@ -31,17 +30,17 @@ def read_reqs(reqs_path: Path) -> Set[str]:
     }
 )
 
-SETUP = dict(
-    name="simcore-sdk",
-    version=Path(CURRENT_DIR / "VERSION").read_text().strip(),
-    packages=find_packages(where="src"),
-    package_dir={"": "src"},
-    python_requires=">=3.6",
-    install_requires=INSTALL_REQUIREMENTS,
-    tests_require=TEST_REQUIREMENTS,
-    extras_require={"test": TEST_REQUIREMENTS},
-    test_suite="tests",
-)
+SETUP = {
+    "name": "simcore-sdk",
+    "version": Path(CURRENT_DIR / "VERSION").read_text().strip(),
+    "packages": find_packages(where="src"),
+    "package_dir": {"": "src"},
+    "python_requires": ">=3.6",
+    "install_requires": INSTALL_REQUIREMENTS,
+    "tests_require": TEST_REQUIREMENTS,
+    "extras_require": {"test": TEST_REQUIREMENTS},
+    "test_suite": "tests",
+}
 
 
 if __name__ == "__main__":

packages/simcore-sdk/tests/conftest.py

@@ -19,8 +19,8 @@
 
 
 pytest_plugins = [
-    "pytest_simcore.aws_server",
     "pytest_simcore.aws_s3_service",
+    "pytest_simcore.aws_server",
     "pytest_simcore.docker_compose",
     "pytest_simcore.docker_swarm",
     "pytest_simcore.file_extra",
@@ -42,30 +42,6 @@ def package_dir():
     return pdir
 
 
-@pytest.fixture(scope="session")
-def osparc_simcore_root_dir() -> Path:
-    """osparc-simcore repo root dir"""
-    WILDCARD = "packages/simcore-sdk"
-
-    root_dir = Path(current_dir)
-    while not any(root_dir.glob(WILDCARD)) and root_dir != Path("/"):
-        root_dir = root_dir.parent
-
-    msg = f"'{root_dir}' does not look like the git root directory of osparc-simcore"
-    assert root_dir.exists(), msg
-    assert any(root_dir.glob(WILDCARD)), msg
-    assert any(root_dir.glob(".git")), msg
-
-    return root_dir
-
-
-@pytest.fixture(scope="session")
-def env_devel_file(osparc_simcore_root_dir) -> Path:
-    env_devel_fpath = osparc_simcore_root_dir / ".env-devel"
-    assert env_devel_fpath.exists()
-    return env_devel_fpath
-
-
 @pytest.fixture(scope="session")
 def default_configuration_file() -> Path:
     path = current_dir / "mock" / "default_config.json"
@@ -75,8 +51,7 @@ def default_configuration_file() -> Path:
 
 @pytest.fixture(scope="session")
 def default_configuration(default_configuration_file: Path) -> dict[str, Any]:
-    config = json.loads(default_configuration_file.read_text())
-    return config
+    return json.loads(default_configuration_file.read_text())
 
 
 @pytest.fixture(scope="session")

packages/simcore-sdk/tests/integration/conftest.py

@@ -39,10 +39,10 @@ def user_id(postgres_db: sa.engine.Engine) -> Iterable[UserID]:
     # which would turn this test too complex.
 
     # pylint: disable=no-value-for-parameter
-    stmt = users.insert().values(**random_user(name="test")).returning(users.c.id)
-    print(f"{stmt}")
     with postgres_db.connect() as conn:
-        result = conn.execute(stmt)
+        result = conn.execute(
+            users.insert().values(**random_user(name="test")).returning(users.c.id)
+        )
         row = result.first()
         assert row
         usr_id = row[users.c.id]