Is621/director2 adds routes to access logfile (ITISFoundation#3067)

Author: pcrespov

packages/pytest-simcore/src/pytest_simcore/docker_compose.py

@@ -145,7 +145,7 @@ def simcore_docker_compose(
         docker_compose_path.exists() for docker_compose_path in docker_compose_paths
     )
 
-    config = run_docker_compose_config(
+    compose_specs = run_docker_compose_config(
         project_dir=osparc_simcore_root_dir / "services",
         docker_compose_paths=docker_compose_paths,
         env_file_path=env_file_for_testing,
@@ -154,10 +154,10 @@ def simcore_docker_compose(
     # NOTE: do not add indent. Copy&Paste log into editor instead
     print(
         HEADER_STR.format("simcore docker-compose"),
-        json.dumps(config),
+        json.dumps(compose_specs),
         HEADER_STR.format("-"),
     )
-    return config
+    return compose_specs
 
 
 @pytest.fixture(scope="module")
@@ -214,7 +214,7 @@ def ops_docker_compose(
     )
     assert docker_compose_path.exists()
 
-    config = run_docker_compose_config(
+    compose_specs = run_docker_compose_config(
         project_dir=osparc_simcore_root_dir / "services",
         docker_compose_paths=docker_compose_path,
         env_file_path=env_file_for_testing,
@@ -223,10 +223,10 @@ def ops_docker_compose(
     # NOTE: do not add indent. Copy&Paste log into editor instead
     print(
         HEADER_STR.format("ops docker-compose"),
-        json.dumps(config),
+        json.dumps(compose_specs),
         HEADER_STR.format("-"),
     )
-    return config
+    return compose_specs
 
 
 @pytest.fixture(scope="module")

packages/pytest-simcore/src/pytest_simcore/docker_swarm.py

@@ -242,6 +242,7 @@ def docker_stack(
                 f"returncode={err.returncode}",
                 f"stdout={err.stdout}",
                 f"stderr={err.stderr}",
+                "\nTIP: frequent failure is due to a corrupt .env file: Delete .env and .env.bak",
             )
             raise
 

packages/pytest-simcore/src/pytest_simcore/postgres_service.py

@@ -1,7 +1,8 @@
-# pylint:disable=redefined-outer-name
-# pylint:disable=unused-argument
-# pylint:disable=unused-variable
-from typing import AsyncIterator, Dict, Final, Iterator, List
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+# pylint: disable=unused-variable
+
+from typing import AsyncIterator, Final, Iterator, TypedDict
 
 import pytest
 import sqlalchemy as sa
@@ -17,9 +18,17 @@
 TEMPLATE_DB_TO_RESTORE = "template_simcore_db"
 
 
+class PostgresTestConfig(TypedDict):
+    user: str
+    password: str
+    database: str
+    host: str
+    port: str
+
+
 def execute_queries(
     postgres_engine: sa.engine.Engine,
-    sql_statements: List[str],
+    sql_statements: list[str],
     ignore_errors: bool = False,
 ) -> None:
     """runs the queries in the list in order"""
@@ -34,7 +43,9 @@ def execute_queries(
                 print(f"SQL error which can be ignored {e}")
 
 
-def create_template_db(postgres_dsn: Dict, postgres_engine: sa.engine.Engine) -> None:
+def create_template_db(
+    postgres_dsn: PostgresTestConfig, postgres_engine: sa.engine.Engine
+) -> None:
     # create a template db, the removal is necessary to allow for the usage of --keep-docker-up
     queries = [
         # disconnect existing users
@@ -69,15 +80,17 @@ def drop_template_db(postgres_engine: sa.engine.Engine) -> None:
 
 @pytest.fixture(scope="module")
 def postgres_with_template_db(
-    postgres_db: sa.engine.Engine, postgres_dsn: Dict, postgres_engine: sa.engine.Engine
+    postgres_db: sa.engine.Engine,
+    postgres_dsn: PostgresTestConfig,
+    postgres_engine: sa.engine.Engine,
 ) -> Iterator[sa.engine.Engine]:
     create_template_db(postgres_dsn, postgres_engine)
     yield postgres_engine
     drop_template_db(postgres_engine)
 
 
 @pytest.fixture
-def drop_db_engine(postgres_dsn: Dict) -> sa.engine.Engine:
+def drop_db_engine(postgres_dsn: PostgresTestConfig) -> sa.engine.Engine:
     postgres_dsn_copy = postgres_dsn.copy()  # make a copy to change these parameters
     postgres_dsn_copy["database"] = "postgres"
     dsn = "postgresql://{user}:{password}@{host}:{port}/{database}".format(
@@ -88,7 +101,7 @@ def drop_db_engine(postgres_dsn: Dict) -> sa.engine.Engine:
 
 @pytest.fixture
 def database_from_template_before_each_function(
-    postgres_dsn: Dict, drop_db_engine: sa.engine.Engine, postgres_db
+    postgres_dsn: PostgresTestConfig, drop_db_engine: sa.engine.Engine, postgres_db
 ) -> None:
     """
     Will recrate the db before running each test.
@@ -118,10 +131,10 @@ def database_from_template_before_each_function(
 
 
 @pytest.fixture(scope="module")
-def postgres_dsn(docker_stack: Dict, testing_environ_vars: Dict) -> Dict[str, str]:
+def postgres_dsn(docker_stack: dict, testing_environ_vars: dict) -> PostgresTestConfig:
     assert "pytest-simcore_postgres" in docker_stack["services"]
 
-    pg_config = {
+    pg_config: PostgresTestConfig = {
         "user": testing_environ_vars["POSTGRES_USER"],
         "password": testing_environ_vars["POSTGRES_PASSWORD"],
         "database": testing_environ_vars["POSTGRES_DB"],
@@ -138,7 +151,7 @@ def postgres_dsn(docker_stack: Dict, testing_environ_vars: Dict) -> Dict[str, st
 
 
 @pytest.fixture(scope="module")
-def postgres_engine(postgres_dsn: Dict[str, str]) -> Iterator[sa.engine.Engine]:
+def postgres_engine(postgres_dsn: PostgresTestConfig) -> Iterator[sa.engine.Engine]:
     dsn = "postgresql://{user}:{password}@{host}:{port}/{database}".format(
         **postgres_dsn
     )
@@ -165,15 +178,15 @@ def postgres_engine(postgres_dsn: Dict[str, str]) -> Iterator[sa.engine.Engine]:
 
 
 @pytest.fixture(scope="module")
-def postgres_dsn_url(postgres_dsn: Dict[str, str]) -> str:
+def postgres_dsn_url(postgres_dsn: PostgresTestConfig) -> str:
     return "postgresql://{user}:{password}@{host}:{port}/{database}".format(
         **postgres_dsn
     )
 
 
 @pytest.fixture(scope="module")
 def postgres_db(
-    postgres_dsn: Dict[str, str],
+    postgres_dsn: PostgresTestConfig[str, str],
     postgres_engine: sa.engine.Engine,
 ) -> Iterator[sa.engine.Engine]:
     """An postgres database init with empty tables and an sqlalchemy engine connected to it"""
@@ -215,7 +228,10 @@ async def sqlalchemy_async_engine(
 
 
 @pytest.fixture(scope="function")
-def postgres_host_config(postgres_dsn: Dict[str, str], monkeypatch) -> Dict[str, str]:
+def postgres_host_config(
+    postgres_dsn: PostgresTestConfig, monkeypatch
+) -> PostgresTestConfig:
+    """sets postgres env vars and returns config"""
     monkeypatch.setenv("POSTGRES_USER", postgres_dsn["user"])
     monkeypatch.setenv("POSTGRES_PASSWORD", postgres_dsn["password"])
     monkeypatch.setenv("POSTGRES_DB", postgres_dsn["database"])

packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py

@@ -43,6 +43,11 @@ async def _get_download_link(
     session: ClientSession,
     link_type: storage_client.LinkType,
 ) -> URL:
+    """
+    :raises exceptions.S3InvalidPathError
+    :raises exceptions.StorageInvalidCall
+    :raises exceptions.StorageServerIssue
+    """
     link: AnyUrl = await storage_client.get_download_file_link(
         session, file_id, store_id, user_id, link_type
     )
@@ -145,7 +150,13 @@ async def get_download_link_from_s3(
     s3_object: str,
     link_type: storage_client.LinkType,
     client_session: Optional[ClientSession] = None,
-) -> Optional[URL]:
+) -> URL:
+    """
+    :raises exceptions.NodeportsException
+    :raises exceptions.S3InvalidPathError
+    :raises exceptions.StorageInvalidCall
+    :raises exceptions.StorageServerIssue
+    """
     if store_name is None and store_id is None:
         raise exceptions.NodeportsException(msg="both store name and store id are None")
 

packages/simcore-sdk/src/simcore_sdk/node_ports_common/storage_client.py

@@ -76,6 +76,10 @@ async def get_download_file_link(
     user_id: UserID,
     link_type: LinkType,
 ) -> AnyUrl:
+    """
+    :raises exceptions.StorageInvalidCall
+    :raises exceptions.StorageServerIssue
+    """
     if (
         not isinstance(file_id, str)
         or not isinstance(location_id, str)

packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py

@@ -68,16 +68,45 @@ async def get_value_from_link(
 
 async def get_download_link_from_storage(
     user_id: int, value: FileLink, link_type: LinkType
-) -> Optional[AnyUrl]:
+) -> AnyUrl:
+    """
+    :raises exceptions.NodeportsException
+    :raises exceptions.S3InvalidPathError
+    :raises exceptions.StorageInvalidCall
+    :raises exceptions.StorageServerIssue
+    """
     log.debug("getting link to file from storage %s", value)
+
     link = await filemanager.get_download_link_from_s3(
         user_id=user_id,
         store_id=f"{value.store}",
         store_name=None,
         s3_object=value.path,
         link_type=link_type,
     )
-    return parse_obj_as(AnyUrl, f"{link}") if link else None
+
+    # could raise ValidationError but will never do it since
+    assert isinstance(link, URL)  # nosec
+    return parse_obj_as(AnyUrl, f"{link}")
+
+
+async def get_download_link_from_storage_overload(
+    user_id: int, project_id: str, node_id: str, file_name: str, link_type: LinkType
+) -> AnyUrl:
+    """Overloads get_download_link_from_storage with arguments that match those in
+    get_upload_link_from_storage
+
+    """
+    # NOTE: might consider using https://github.com/mrocklin/multipledispatch/ in the future?
+    s3_object = data_items_utils.encode_file_id(Path(file_name), project_id, node_id)
+    link = await filemanager.get_download_link_from_s3(
+        user_id=user_id,
+        store_name=config.STORE,
+        store_id=None,
+        s3_object=s3_object,
+        link_type=link_type,
+    )
+    return parse_obj_as(AnyUrl, f"{link}")
 
 
 async def get_upload_link_from_storage(

scripts/common.Makefile

@@ -37,6 +37,7 @@ REPO_BASE_DIR := $(shell git rev-parse --show-toplevel)
 
 # relevant repo folders
 SCRIPTS_DIR := $(abspath $(REPO_BASE_DIR)/scripts)
+PACKAGES_DIR := $(abspath $(REPO_BASE_DIR)/packages)
 
 # virtual env
 EXPECTED_PYTHON_VERSION := $(shell cat $(REPO_BASE_DIR)/requirements/PYTHON_VERSION)

services/director-v2/Makefile

@@ -21,6 +21,22 @@ openapi.json: .env
 	python3 -c "import json; from $(APP_PACKAGE_NAME).main import *; print( json.dumps(the_app.openapi(), indent=2) )" > $@
 
 
+DOCKER_API_VERSION ?= 1.41
+.PHONY: directorv2_rest_api_schemas.py
+directorv2_rest_api_schemas.py: openapi.json ## auto-generates pydantic models for Docker REST API models
+	# auto-generates $@ from $<
+	$(SCRIPTS_DIR)/openapi-pydantic-models-generator.bash \
+			--input $< \
+			--output $@ \
+			--field-constraints \
+			--reuse-model \
+			--enum-field-as-literal=all
+
+	# formats
+	@black $@
+	# copy output to src/models_library/generated_models...
+	@mv $@ $(PACKAGES_DIR)/models-library/src/models_library/generated_models/$@
+	# done
 
 # DEVELOPMENT #############
 include .env

services/director-v2/VERSION

@@ -1 +1 @@
-2.0.0
+2.1.0