♻️ Replace auto-generated storage python client (ITISFoundation#2578)

* removed storage auto-generated client
* created aiohttp-based simple client
* wait for director-v0 to be up
* improve wait time and logging
* added docker registry
* @GitHK review: wait max 1hour

Author: sanderegg

api/specs/storage/openapi.yaml

@@ -240,7 +240,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/FileMetaDataEnveloped"
+                $ref: "#/components/schemas/FileMetaEnvelope"
         default:
           $ref: "#/components/responses/DefaultErrorResponse"
     patch:
@@ -268,7 +268,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/FileMetaDataEnveloped"
+                $ref: "#/components/schemas/FileMetaEnvelope"
         default:
           $ref: "#/components/responses/DefaultErrorResponse"
 
@@ -480,7 +480,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/FileMetaDataEnveloped"
+                $ref: "#/components/schemas/FileMetaEnvelope"
         default:
           $ref: "#/components/responses/DefaultErrorResponse"
 components:
@@ -738,7 +738,7 @@ components:
           nullable: true
           default: null
 
-    DatasetMetaDataEnveloped:
+    DatasetMetaEnvelope:
       type: object
       required:
         - data
@@ -766,7 +766,7 @@ components:
       items:
         $ref: "#/components/schemas/DatasetMetaData"
 
-    FileMetaDataEnveloped:
+    FileMetaEnvelope:
       type: object
       required:
         - data

api/specs/webserver/components/schemas/datasets.yaml

@@ -1,10 +1,10 @@
-DatasetMetaDataEnveloped:
+DatasetMetaEnvelope:
   type: object
   required:
     - data
   properties:
     data:
-      $ref: '#DatasetMetaData'
+      $ref: "#DatasetMetaData"
     error:
       nullable: true
       default: null
@@ -17,10 +17,10 @@ DatasetMetaData:
     display_name:
       type: string
   example:
-    dataset_uuid: 'N:id-aaaa'
+    dataset_uuid: "N:id-aaaa"
     display_name: "simcore-testing"
 
 DatasetMetaDataArray:
   type: array
   items:
-    $ref: '#/DatasetMetaData'
+    $ref: "#/DatasetMetaData"

api/specs/webserver/components/schemas/files.yaml

@@ -1,10 +1,10 @@
-FileMetaDataEnveloped:
+FileMetaEnvelope:
   type: object
   required:
     - data
   properties:
     data:
-      $ref: '#/FileMetaData'
+      $ref: "#/FileMetaData"
     error:
       nullable: true
       default: null
@@ -50,9 +50,9 @@ FileMetaData:
     file_size:
       type: integer
     parent_id:
-          type: string
+      type: string
   example:
-    file_uuid: 'simcore-testing/105/1000/3'
+    file_uuid: "simcore-testing/105/1000/3"
     location_id: "0"
     location_name: "simcore.s3"
     bucket_name: "simcore-testing"
@@ -72,8 +72,7 @@ FileMetaData:
     file_size: 73
     parent_id: "N:collection:e263da07-2d89-45a6-8b0f-61061b913873"
 
-
 FileMetaDataArray:
   type: array
   items:
-    $ref: '#/FileMetaData'
+    $ref: "#/FileMetaData"

ci/github/integration-testing/simcore-sdk.bash

@@ -17,7 +17,6 @@ install() {
   # pull the test images if registry is set up, else build the images
   make pull-version || (make build tag-version)
   make info-images
-  # pip3 install services/storage/client-sdk/python
 }
 
 test() {

packages/models-library/src/models_library/api_schemas_storage.py

@@ -12,8 +12,10 @@
 from uuid import UUID
 
 from pydantic import BaseModel, Field, constr
+from pydantic.networks import AnyUrl
 
 from .basic_regex import UUID_RE
+from .generics import ListModel
 
 
 # /
@@ -24,11 +26,6 @@ class HealthCheck(BaseModel):
     version: Optional[str]
 
 
-class HealthCheckEnveloped(BaseModel):
-    data: HealthCheck
-    error: Any
-
-
 # /check/{action}:
 class Fake(BaseModel):
     path_value: str
@@ -49,19 +46,7 @@ class Config:
         }
 
 
-class FileLocationArray(BaseModel):
-    __root__: List[FileLocation]
-
-
-class FileLocationEnveloped(BaseModel):
-    data: FileLocation
-    error: Any
-
-
-class FileLocationArrayEnveloped(BaseModel):
-    data: FileLocationArray
-    error: Any
-
+FileLocationArray = ListModel[FileLocation]
 
 # /locations/{location_id}/datasets
 
@@ -101,18 +86,7 @@ class Config:
         }
 
 
-class DatasetMetaDataArray(BaseModel):
-    __root__: List[DatasetMetaData] = []
-
-
-class DatasetMetaDataEnveloped(BaseModel):
-    data: DatasetMetaData
-    error: Any
-
-
-class DatasetMetaDataArrayEnveloped(BaseModel):
-    data: DatasetMetaDataArray
-    error: Any
+DatasetMetaDataArray = ListModel[DatasetMetaData]
 
 
 # /locations/{location_id}/files/metadata:
@@ -215,26 +189,11 @@ class FileMetaDataArray(BaseModel):
     __root__: List[FileMetaData] = []
 
 
-class FileMetaDataEnveloped(BaseModel):
-    data: FileMetaData
-    error: Any
-
-
-class FileMetaDataArrayEnveloped(BaseModel):
-    data: FileMetaDataArray
-    error: Any
-
-
 # /locations/{location_id}/files/{fileId}
 
 
 class PresignedLink(BaseModel):
-    link: str
-
-
-class PresignedLinkEnveloped(BaseModel):
-    data: PresignedLink
-    error: Any
+    link: AnyUrl
 
 
 # /simcore-s3/
@@ -279,18 +238,3 @@ class Error(BaseModel):
     logs: Optional[List[LogMessage]] = Field(description="Log messages")
     errors: Optional[List[ErrorItem]] = Field(description="Errors metadata")
     status: Optional[int] = Field(description="HTTP error code")
-
-
-class LogMessageEnveloped(BaseModel):
-    data: LogMessage
-    error: Any
-
-
-class FakeEnveloped(BaseModel):
-    data: Fake
-    error: Any
-
-
-class ErrorEnveloped(BaseModel):
-    data: Any
-    error: Error

packages/models-library/src/models_library/generics.py

@@ -0,0 +1,76 @@
+from typing import (
+    Any,
+    Dict,
+    Generic,
+    ItemsView,
+    Iterator,
+    KeysView,
+    List,
+    Optional,
+    TypeVar,
+    ValuesView,
+)
+
+from pydantic import validator
+from pydantic.generics import GenericModel
+
+DictKey = TypeVar("DictKey")
+DictValue = TypeVar("DictValue")
+
+
+class DictModel(GenericModel, Generic[DictKey, DictValue]):
+    __root__: Dict[DictKey, DictValue]
+
+    def __getitem__(self, k: DictKey) -> DictValue:
+        return self.__root__.__getitem__(k)
+
+    def __setitem__(self, k: DictKey, v: DictValue) -> None:
+        self.__root__.__setitem__(k, v)
+
+    def items(self) -> ItemsView[DictKey, DictValue]:
+        return self.__root__.items()
+
+    def keys(self) -> KeysView[DictKey]:
+        return self.__root__.keys()
+
+    def values(self) -> ValuesView[DictValue]:
+        return self.__root__.values()
+
+    def __iter__(self) -> Iterator[DictKey]:
+        return self.__root__.__iter__()
+
+    def get(self, key: DictKey, default: Optional[DictValue] = None):
+        return self.__root__.get(key, default)
+
+    def __len__(self) -> int:
+        return self.__root__.__len__()
+
+
+DataT = TypeVar("DataT")
+
+
+class ListModel(GenericModel, Generic[DataT]):
+    __root__: List[DataT]
+
+    def __iter__(self):
+        return iter(self.__root__)
+
+    def __getitem__(self, item):
+        return self.__root__[item]
+
+    def __len__(self):
+        return len(self.__root__)
+
+
+class Envelope(GenericModel, Generic[DataT]):
+    data: Optional[DataT]
+    error: Optional[Any]
+
+    @validator("error")
+    @classmethod
+    def data_and_error_cannot_be_populated_together(cls, v, values):
+        if v is not None and values.get("data") is not None:
+            raise ValueError(
+                "both data and error cannot contain values at the same time"
+            )
+        return v

packages/models-library/tests/test_generics.py

@@ -0,0 +1,61 @@
+from pathlib import Path
+from typing import Any
+
+import pytest
+from faker import Faker
+from models_library.generics import DictModel, Envelope
+
+
+def test_dict_base_model():
+    some_dict = {
+        "a key": 123,
+        "another key": "a string value",
+        "yet another key": Path("some_path"),
+    }
+    some_instance = DictModel[str, Any].parse_obj(some_dict)
+    assert some_instance
+
+    # test some typical dict methods
+    assert len(some_instance) == 3
+    for k, k2 in zip(some_dict, some_instance):
+        assert k == k2
+
+    for k, k2 in zip(some_dict.keys(), some_instance.keys()):
+        assert k == k2
+
+    for v, v2 in zip(some_dict.values(), some_instance.values()):
+        assert v == v2
+
+    for i, i2 in zip(some_dict.items(), some_instance.items()):
+        assert i == i2
+
+    assert some_instance.get("a key") == 123
+    assert some_instance.get("a non existing key") is None
+
+    assert some_instance["a key"] == 123
+    with pytest.raises(KeyError):
+        some_instance["a non existing key"]  # pylint: disable=pointless-statement
+    some_instance["a new key"] = 23
+    assert some_instance["a new key"] == 23
+
+
+def test_data_enveloped(faker: Faker):
+    some_enveloped_string = Envelope[str]()
+    assert some_enveloped_string
+    assert not some_enveloped_string.data
+    assert not some_enveloped_string.error
+
+    random_float = faker.pyfloat()
+    some_enveloped_float = Envelope[float](data=random_float)
+    assert some_enveloped_float
+    assert some_enveloped_float.data == random_float
+    assert not some_enveloped_float.error
+
+    random_text = faker.text()
+    some_enveloped_bool = Envelope[bool](error=random_text)
+    assert some_enveloped_bool
+    assert not some_enveloped_bool.data
+    assert some_enveloped_bool.error == random_text
+
+    with pytest.raises(ValueError):
+        Envelope[int](data=213, error="some error message")

packages/pytest-simcore/src/pytest_simcore/environment_configs.py

@@ -5,22 +5,26 @@
 
 from copy import deepcopy
 from pathlib import Path
-from typing import Dict, Union
+from typing import Dict
 
 import dotenv
 import pytest
+from _pytest.monkeypatch import MonkeyPatch
 
 
 @pytest.fixture(scope="session")
-def env_devel_dict(env_devel_file: Path) -> Dict[str, Union[str, None]]:
+def env_devel_dict(env_devel_file: Path) -> Dict[str, str]:
     assert env_devel_file.exists()
     assert env_devel_file.name == ".env-devel"
     environ = dotenv.dotenv_values(env_devel_file, verbose=True, interpolate=True)
-    return environ
+    assert all(v is not None for v in environ.values())
+    return environ  # type: ignore
 
 
 @pytest.fixture(scope="function")
-def mock_env_devel_environment(env_devel_dict, monkeypatch):
+def mock_env_devel_environment(
+    env_devel_dict: Dict[str, str], monkeypatch: MonkeyPatch
+) -> Dict[str, str]:
     for key, value in env_devel_dict.items():
-        monkeypatch.setenv(key, value)
+        monkeypatch.setenv(key, str(value))
     return deepcopy(env_devel_dict)