✨ Capture osparc envs automatically using a pydantic model (#170)

* cleanup

* overrides Config with new defaults

* testing config

* upgrades pre-commit

* mv pytest.ini

* monkey patch

* update workflow before publishing python package

fix dependency issue and bump version

point to website in project description

fix broken dependency

improve doc

add github token to download artifacts

ensure only read-access @wvangeit

yet another attempt at downloading artifacts

make sure to use repo that ran the trigger wf

another attempt at fixing

change owner

allow publishing to testpypi also when pr

minor change

revert minor (but breaking) change

minor fix

add debug messages

another debug message

hopefully the final version

final fix

minor fix

move master and tag to individual jobs

add debug messages

add python script for determining semantic version

minor changes

minor changes

improve error handling and add version file to artifacts

check if release

minor fix

ensure to enter venv

also when tagging

source venv in publishin workflow

ensure only master

add script for testing 'pure' semver

adapt workflows to new python script

minor change

attempt to evaluate expressions correctly

several fixes to fix tests

ensure repo is checked out in publish workflow

several small fixes

cleanup

debug

minor cleanup

mionr changes

add debug message

minor change

minor change

yet another try

minor change

minor change

minor change

mionr change

minor changes

correct workflow run id

cosmetic change

avoid using gh

change to a single job for publishing

minor cleanup

swap loops in clean up jobs

correction

get correct versions of github workflow files

update a couple of other files

update a few more tests

update yet another file

yet another file

* wrap api_client

* add type

* remove _configuration.py

* avoid bad import

* fix import

* fix e2e tests

* add unit test

* require api_client in wrapped apis

* fix unit tests

---------

Co-authored-by: Pedro Crespo-Valero <[email protected]>

Author: bisgaard-itis

clients/python/client/osparc/__init__.py

@@ -4,7 +4,6 @@
 
 import nest_asyncio
 from osparc_client import (  # APIs; API client; models
-    ApiClient,
     ApiException,
     ApiKeyError,
     ApiTypeError,
@@ -38,6 +37,7 @@
 )
 from packaging.version import Version
 
+from ._api_client import ApiClient
 from ._exceptions import RequestError, VisibleDeprecationWarning
 from ._files_api import FilesApi
 from ._info import openapi
@@ -114,4 +114,4 @@
     "UsersApi",
     "UsersGroup",
     "ValidationError",
-)
+)  # type: ignore

clients/python/client/osparc/_api_client.py

@@ -0,0 +1,37 @@
+from typing import Optional
+
+from osparc_client import ApiClient as _ApiClient
+from osparc_client import Configuration
+from pydantic import ValidationError
+
+from ._models import ConfigurationModel
+
+
+class ApiClient(_ApiClient):
+    def __init__(
+        self,
+        configuration: Optional[Configuration] = None,
+        header_name=None,
+        header_value=None,
+        cookie=None,
+        pool_threads=1,
+    ):
+        if configuration is None:
+            try:
+                env_vars = ConfigurationModel()
+                configuration = Configuration(
+                    host=f"{env_vars.OSPARC_API_HOST}".rstrip(
+                        "/"
+                    ),  # https://github.com/pydantic/pydantic/issues/7186
+                    username=env_vars.OSPARC_API_KEY,
+                    password=env_vars.OSPARC_API_SECRET,
+                )
+            except ValidationError as exc:
+                raise RuntimeError(
+                    "Could not initialize configuration from environment. "
+                    "If your osparc host, key and secret are not exposed as "
+                    "environment variables you must construct the "
+                    "Configuration object explicitly"
+                ) from exc
+
+        super().__init__(configuration, header_name, header_value, cookie, pool_threads)

clients/python/client/osparc/_files_api.py

@@ -21,7 +21,8 @@
 from tqdm.asyncio import tqdm
 from tqdm.contrib.logging import logging_redirect_tqdm
 
-from . import ApiClient, File
+from . import File
+from ._api_client import ApiClient
 from ._http_client import AsyncHttpClient
 from ._utils import (
     DEFAULT_TIMEOUT_SECONDS,
@@ -36,7 +37,7 @@
 class FilesApi(_FilesApi):
     """Class for interacting with files"""
 
-    def __init__(self, api_client: Optional[ApiClient] = None):
+    def __init__(self, api_client: ApiClient):
         """Construct object
 
         Args:

clients/python/client/osparc/_models.py

@@ -1,7 +1,7 @@
 from typing import Optional
 from uuid import UUID
 
-from pydantic import Field, field_validator
+from pydantic import AnyHttpUrl, Field, field_validator
 from pydantic_settings import BaseSettings
 
 
@@ -18,7 +18,19 @@ class ParentProjectInfo(BaseSettings):
 
     @field_validator("x_simcore_parent_project_uuid", "x_simcore_parent_node_id")
     @classmethod
-    def _validate_uuids(cls, v: Optional[str]) -> str:
+    def _validate_uuids(cls, v: Optional[str]) -> Optional[str]:
         if v is not None:
             _ = UUID(v)
         return v
+
+
+class ConfigurationModel(BaseSettings):
+    """Model for capturing env vars which should go into the Configuration"""
+
+    OSPARC_API_HOST: AnyHttpUrl = Field(
+        default=...,
+        description="OSPARC api url",
+        examples=["https://api.osparc-master.speag.com/"],
+    )
+    OSPARC_API_KEY: str = Field(default=..., description="OSPARC api key")
+    OSPARC_API_SECRET: str = Field(default=..., description="OSPARC api secret")

clients/python/client/osparc/_solvers_api.py

@@ -4,7 +4,7 @@
 from osparc_client import JobInputs, OnePageSolverPort, SolverPort
 from osparc_client import SolversApi as _SolversApi
 
-from . import ApiClient
+from ._api_client import ApiClient
 from ._models import ParentProjectInfo
 from ._utils import (
     _DEFAULT_PAGINATION_LIMIT,
@@ -27,7 +27,7 @@ def __getattr__(self, name: str) -> Any:
             raise NotImplementedError(f"SolversApi.{name} is still under development")
         return super().__getattribute__(name)
 
-    def __init__(self, api_client: Optional[ApiClient] = None):
+    def __init__(self, api_client: ApiClient):
         """Construct object
 
         Args:

clients/python/client/osparc/_studies_api.py

@@ -5,10 +5,11 @@
 from typing import Any, Optional
 
 import httpx
-from osparc_client import ApiClient, JobInputs, JobLogsMap, PageStudy
+from osparc_client import JobInputs, JobLogsMap, PageStudy
 from osparc_client import StudiesApi as _StudiesApi
 from tqdm.asyncio import tqdm_asyncio
 
+from ._api_client import ApiClient
 from ._http_client import AsyncHttpClient
 from ._models import ParentProjectInfo
 from ._utils import (
@@ -39,7 +40,7 @@ class StudiesApi(_StudiesApi):
         "stop_study_job",
     ]
 
-    def __init__(self, api_client: Optional[ApiClient] = None):
+    def __init__(self, api_client: ApiClient):
         """Construct object
 
         Args:

clients/python/test/e2e/conftest.py

@@ -4,6 +4,7 @@
 # pylint: disable=unused-argument
 # pylint: disable=unused-variable
 
+import datetime
 import logging
 import os
 from pathlib import Path
@@ -14,37 +15,93 @@
 import pytest
 from httpx import AsyncClient, BasicAuth
 from numpy import random
+from osparc._models import ConfigurationModel
 from pydantic import ByteSize
 
 _KB: ByteSize = ByteSize(1024)  # in bytes
 _MB: ByteSize = ByteSize(_KB * 1024)  # in bytes
 _GB: ByteSize = ByteSize(_MB * 1024)  # in bytes
 
 
-@pytest.fixture
-def configuration() -> osparc.Configuration:
-    assert (host := os.environ.get("OSPARC_API_HOST"))
-    assert (username := os.environ.get("OSPARC_API_KEY"))
-    assert (password := os.environ.get("OSPARC_API_SECRET"))
-    return osparc.Configuration(
-        host=host,
-        username=username,
-        password=password,
-    )
+# Dictionary to store start times of tests
+_test_start_times = {}
+
+
+def _utc_now():
+    return datetime.datetime.now(tz=datetime.timezone.utc)
+
+
+def _construct_graylog_url(api_host, start_time, end_time):
+    """
+    Construct a Graylog URL for the given time interval.
+    """
+    base_url = api_host.replace("api.", "monitoring.", 1).rstrip("/")
+    url = f"{base_url}/graylog/search"
+    start_time_str = start_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+    end_time_str = end_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+    query = f"from={start_time_str}&to={end_time_str}"
+    return f"{url}?{query}"
+
+
+def pytest_runtest_setup(item):
+    """
+    Hook to capture the start time of each test.
+    """
+    _test_start_times[item.name] = _utc_now()
+
+
+def pytest_runtest_makereport(item, call):
+    """
+    Hook to add extra information when a test fails.
+    """
+    if call.when == "call":
+        # Check if the test failed
+        if call.excinfo is not None:
+            test_name = item.name
+            test_location = item.location
+            api_host = os.environ.get("OSPARC_API_HOST", "")
+
+            diagnostics = {
+                "test_name": test_name,
+                "test_location": test_location,
+                "api_host": api_host,
+            }
+
+            # Get the start and end times of the test
+            start_time = _test_start_times.get(test_name)
+            end_time = _utc_now()
+
+            if start_time:
+                diagnostics["graylog_url"] = _construct_graylog_url(
+                    api_host, start_time, end_time
+                )
+
+            # Print the diagnostics
+            print(f"\nDiagnostics for {test_name}:")
+            for key, value in diagnostics.items():
+                print("  ", key, ":", value)
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_configure(config):
+    config.pluginmanager.register(pytest_runtest_setup, "osparc_test_times_plugin")
+    config.pluginmanager.register(pytest_runtest_makereport, "osparc_makereport_plugin")
 
 
 @pytest.fixture
-def api_client(configuration: osparc.Configuration) -> Iterable[osparc.ApiClient]:
-    with osparc.ApiClient(configuration=configuration) as _api_client:
-        yield _api_client
+def api_client() -> Iterable[osparc.ApiClient]:
+    with osparc.ApiClient() as api_client:
+        yield api_client
 
 
 @pytest.fixture
-def async_client(configuration: osparc.Configuration) -> AsyncClient:
-    return AsyncClient(
-        base_url=configuration.host,
+def async_client() -> Iterable[AsyncClient]:
+    configuration = ConfigurationModel()
+    yield AsyncClient(
+        base_url=f"{configuration.OSPARC_API_HOST}".rstrip("/"),
         auth=BasicAuth(
-            username=configuration.username, password=configuration.password
+            username=configuration.OSPARC_API_KEY,
+            password=configuration.OSPARC_API_SECRET,
         ),
     )  # type: ignore
 

clients/python/test/test_osparc/conftest.py

@@ -2,6 +2,7 @@
 import pytest
 from faker import Faker
 
+
 @pytest.fixture
 def cfg(faker: Faker) -> osparc.Configuration:
     return osparc.Configuration(
@@ -12,5 +13,10 @@ def cfg(faker: Faker) -> osparc.Configuration:
 
 
 @pytest.fixture
-def dev_mode_enabled(monkeypatch:pytest.MonkeyPatch):
+def api_client(cfg: osparc.Configuration) -> osparc.ApiClient:
+    return osparc.ApiClient(configuration=cfg)
+
+
+@pytest.fixture
+def dev_mode_enabled(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setenv("OSPARC_DEV_FEATURES_ENABLED", "1")

clients/python/test/test_osparc/test_apis.py

@@ -4,11 +4,11 @@
 # pylint: disable=too-many-arguments
 
 import os
-from typing import Callable
+from typing import Callable, Optional
 
 import pytest
 from faker import Faker
-from osparc import SolversApi, StudiesApi
+from osparc import ApiClient, SolversApi, StudiesApi
 from pytest_mock import MockerFixture
 
 
@@ -31,6 +31,7 @@ def test_create_jobs_parent_headers(
     create_parent_env: Callable,
     dev_mode_enabled: None,
     parent_env: bool,
+    api_client: ApiClient,
 ):
     create_parent_env(parent_env)
 
@@ -58,9 +59,43 @@ def check_headers(**kwargs):
         side_effect=lambda study_id, **kwargs: check_headers(**kwargs),
     )
 
-    solvers_api = SolversApi()
+    solvers_api = SolversApi(api_client=api_client)
     solvers_api.create_job(solver_key="mysolver", version="1.2.3", job_inputs={})
 
-    studies_api = StudiesApi()
+    studies_api = StudiesApi(api_client=api_client)
     studies_api.create_study_job(study_id=faker.uuid4(), job_inputs={})
     studies_api.clone_study(study_id=faker.uuid4())
+
+
+@pytest.mark.parametrize(
+    "OSPARC_API_HOST", ["https://api.foo.com", "https://api.bar.com/", None]
+)
+@pytest.mark.parametrize("OSPARC_API_KEY", ["key", None])
+@pytest.mark.parametrize("OSPARC_API_SECRET", ["secret", None])
+def test_api_client_constructor(
+    monkeypatch: pytest.MonkeyPatch,
+    OSPARC_API_HOST: Optional[str],
+    OSPARC_API_KEY: Optional[str],
+    OSPARC_API_SECRET: Optional[str],
+):
+    with monkeypatch.context() as patch:
+        patch.delenv("OSPARC_API_HOST", raising=False)
+        patch.delenv("OSPARC_API_KEY", raising=False)
+        patch.delenv("OSPARC_API_SECRET", raising=False)
+
+        if OSPARC_API_HOST is not None:
+            patch.setenv("OSPARC_API_HOST", OSPARC_API_HOST)
+        if OSPARC_API_KEY is not None:
+            patch.setenv("OSPARC_API_KEY", OSPARC_API_KEY)
+        if OSPARC_API_SECRET is not None:
+            patch.setenv("OSPARC_API_SECRET", OSPARC_API_SECRET)
+
+        if OSPARC_API_HOST and OSPARC_API_KEY and OSPARC_API_SECRET:
+            api = ApiClient()
+            assert api.configuration.host == OSPARC_API_HOST.rstrip("/")
+            assert api.configuration.username == OSPARC_API_KEY
+            assert api.configuration.password == OSPARC_API_SECRET
+
+        else:
+            with pytest.raises(RuntimeError):
+                ApiClient()

clients/python/test/test_osparc/test_osparc_client/test_files_api.py

@@ -14,14 +14,16 @@
 
 import unittest
 
-from osparc import FilesApi  # noqa: E501
+from osparc import ApiClient, Configuration, FilesApi  # noqa: E501
 
 
 class TestFilesApi(unittest.TestCase):
     """FilesApi unit test stubs"""
 
     def setUp(self):
-        self.api = FilesApi()  # noqa: E501
+        self.api = FilesApi(
+            api_client=ApiClient(configuration=Configuration())
+        )  # noqa: E501
 
     def tearDown(self):
         pass