Merge branch 'is1988/upgrading-director-to-fastapi' of github.com:pcrespov/osparc-simcore into is1988/upgrading-director-to-fastapi

Author: sanderegg

.env-devel

@@ -65,11 +65,20 @@ DASK_TLS_CA_FILE=/home/scu/.dask/dask-crt.pem
 DASK_TLS_CERT=/home/scu/.dask/dask-crt.pem
 DASK_TLS_KEY=/home/scu/.dask/dask-key.pem
 
+DIRECTOR_DEFAULT_MAX_MEMORY=2_147_483_648
+DIRECTOR_DEFAULT_MAX_NANO_CPUS=1_000_000_000
 DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}'
 DIRECTOR_HOST=director
+DIRECTOR_LOGLEVEL=INFO
+DIRECTOR_MONITORING_ENABLED=True
 DIRECTOR_PORT=8000
+DIRECTOR_PUBLISHED_HOST_NAME=
 DIRECTOR_REGISTRY_CACHING_TTL=900
 DIRECTOR_REGISTRY_CACHING=True
+DIRECTOR_SELF_SIGNED_SSL_FILENAME=
+DIRECTOR_SELF_SIGNED_SSL_SECRET_ID=
+DIRECTOR_SELF_SIGNED_SSL_SECRET_NAME=
+DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS=
 
 EFS_USER_ID=8006
 EFS_USER_NAME=efs
@@ -88,7 +97,7 @@ COMPUTATIONAL_BACKEND_DEFAULT_FILE_LINK_TYPE=PRESIGNED
 COMPUTATIONAL_BACKEND_ON_DEMAND_CLUSTERS_FILE_LINK_TYPE=PRESIGNED
 DIRECTOR_V2_DEV_FEATURES_ENABLED=0
 DIRECTOR_V2_DYNAMIC_SCHEDULER_CLOSE_SERVICES_VIA_FRONTEND_WHEN_CREDITS_LIMIT_REACHED=1
-DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL=0
+DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL=PT0S
 DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}'
 DIRECTOR_V2_HOST=director-v2
 DIRECTOR_V2_LOGLEVEL=INFO
@@ -108,7 +117,7 @@ DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT=3600
 # DIRECTOR_V2 ----
 DYNAMIC_SCHEDULER_LOGLEVEL=DEBUG
 DYNAMIC_SCHEDULER_PROFILING=1
-DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=3600
+DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=PT1H
 
 FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "[email protected]", "affiliation": "unknown"}}'
 
@@ -171,6 +180,7 @@ REDIS_SECURE=false
 REDIS_USER=null
 
 REGISTRY_AUTH=True
+REGISTRY_PATH=""
 REGISTRY_PW=adminadminadmin
 REGISTRY_SSL=True
 REGISTRY_URL=registry.osparc-master.speag.com

scripts/maintenance/computational-clusters/autoscaled_monitor/cli.py

@@ -13,6 +13,7 @@
     DEFAULT_COMPUTATIONAL_EC2_FORMAT_WORKERS,
     DEFAULT_DYNAMIC_EC2_FORMAT,
     DEPLOY_SSH_KEY_PARSER,
+    UNIFIED_SSH_KEY_PARSE,
     wallet_id_spec,
 )
 from .ec2 import autoscaling_ec2_client, cluster_keeper_ec2_client
@@ -78,16 +79,12 @@ def main(
 
     # locate ssh key path
     for file_path in deploy_config.glob("**/*.pem"):
-        if any(_ in file_path.name for _ in ["license", "pkcs8"]):
-            continue
-        # very bad HACK where the license file contain openssh in the name
-        if (
-            any(_ in f"{file_path}" for _ in ("sim4life.io", "osparc-master"))
-            and "openssh" not in f"{file_path}"
-        ):
+        if any(_ in file_path.name for _ in ["license", "pkcs8", "dask"]):
             continue
 
-        if DEPLOY_SSH_KEY_PARSER.parse(f"{file_path.name}") is not None:
+        if DEPLOY_SSH_KEY_PARSER.parse(
+            f"{file_path.name}"
+        ) is not None or UNIFIED_SSH_KEY_PARSE.parse(f"{file_path.name}"):
             rich.print(
                 f"will be using following ssh_key_path: {file_path}. "
                 "TIP: if wrong adapt the code or manually remove some of them."

scripts/maintenance/computational-clusters/autoscaled_monitor/constants.py

@@ -22,6 +22,7 @@ def wallet_id_spec(text) -> None | int:
 DEPLOY_SSH_KEY_PARSER: Final[parse.Parser] = parse.compile(
     r"{prefix}-{random_name}.pem"
 )
+UNIFIED_SSH_KEY_PARSE: Final[parse.Parser] = parse.compile("sshkey.pem")
 
 MINUTE: Final[int] = 60
 HOUR: Final[int] = 60 * MINUTE

services/agent/src/simcore_service_agent/core/settings.py

@@ -53,14 +53,14 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings):
     AGENT_VOLUMES_CLEANUP_INTERVAL: timedelta = Field(
         timedelta(minutes=1), description="interval for running volumes removal"
     )
-    AGENT_VOLUMES_CLENUP_BOOK_KEEPING_INTERVAL: timedelta = Field(
+    AGENT_VOLUMES_CLEANUP_BOOK_KEEPING_INTERVAL: timedelta = Field(
         timedelta(minutes=1),
         description=(
             "interval at which to scan for unsued volumes and keep track since "
             "they were detected as being unused"
         ),
     )
-    AGENT_VOLUMES_CLENUP_REMOVE_VOLUMES_INACTIVE_FOR: timedelta = Field(
+    AGENT_VOLUMES_CLEANUP_REMOVE_VOLUMES_INACTIVE_FOR: timedelta = Field(
         timedelta(minutes=65),
         description=(
             "if a volume is unused for more than this interval it can be removed. "

services/agent/src/simcore_service_agent/services/volumes_manager.py

@@ -174,9 +174,9 @@ async def _on_startup() -> None:
 
         volumes_manager = VolumesManager(
             app=app,
-            book_keeping_interval=settings.AGENT_VOLUMES_CLENUP_BOOK_KEEPING_INTERVAL,
+            book_keeping_interval=settings.AGENT_VOLUMES_CLEANUP_BOOK_KEEPING_INTERVAL,
             volume_cleanup_interval=settings.AGENT_VOLUMES_CLEANUP_INTERVAL,
-            remove_volumes_inactive_for=settings.AGENT_VOLUMES_CLENUP_REMOVE_VOLUMES_INACTIVE_FOR.total_seconds(),
+            remove_volumes_inactive_for=settings.AGENT_VOLUMES_CLEANUP_REMOVE_VOLUMES_INACTIVE_FOR.total_seconds(),
         )
         volumes_manager.set_to_app_state(app)
         await volumes_manager.setup()

services/director/src/simcore_service_director/core/settings.py

@@ -73,11 +73,11 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings):
     )
 
     # migrated settings
-    DIRECTOR_DEFAULT_MAX_NANO_CPUS: int = Field(
+    DIRECTOR_DEFAULT_MAX_NANO_CPUS: PositiveInt = Field(
         default=1 * pow(10, 9),
         env=["DIRECTOR_DEFAULT_MAX_NANO_CPUS", "DEFAULT_MAX_NANO_CPUS"],
     )
-    DIRECTOR_DEFAULT_MAX_MEMORY: int = Field(
+    DIRECTOR_DEFAULT_MAX_MEMORY: PositiveInt = Field(
         default=parse_obj_as(ByteSize, "2GiB"),
         env=["DIRECTOR_DEFAULT_MAX_MEMORY", "DEFAULT_MAX_MEMORY"],
     )

services/director/tests/conftest.py

@@ -135,9 +135,9 @@ def app_settings(app_environment: EnvVarsDict) -> ApplicationSettings:
 
 @pytest.fixture
 async def app(
-    app_setting: ApplicationSettings, is_pdb_enabled: bool
+    app_settings: ApplicationSettings, is_pdb_enabled: bool
 ) -> AsyncIterator[FastAPI]:
-    the_test_app = create_app(settings=app_setting)
+    the_test_app = create_app(settings=app_settings)
     async with LifespanManager(
         the_test_app,
         startup_timeout=None if is_pdb_enabled else MAX_TIME_FOR_APP_TO_STARTUP,

services/director/tests/test_core_settings.py

@@ -21,4 +21,7 @@ def test_valid_web_application_settings(app_environment: EnvVarsDict):
 
     assert settings == ApplicationSettings.create_from_envs()
 
-    assert app_environment["DIRECTOR_DEBUG"] == settings.DIRECTOR_DEBUG
+    assert (
+        app_environment["DIRECTOR_DEFAULT_MAX_MEMORY"]
+        == settings.DIRECTOR_DEFAULT_MAX_MEMORY
+    )

services/docker-compose.yml

@@ -245,8 +245,8 @@ services:
     init: true
     hostname: "{{.Node.Hostname}}-{{.Task.Slot}}"
     environment:
-      DEFAULT_MAX_MEMORY: ${DIRECTOR_DEFAULT_MAX_MEMORY:-0}
-      DEFAULT_MAX_NANO_CPUS: ${DIRECTOR_DEFAULT_MAX_NANO_CPUS:-0}
+      DIRECTOR_DEFAULT_MAX_MEMORY: ${DIRECTOR_DEFAULT_MAX_MEMORY}
+      DIRECTOR_DEFAULT_MAX_NANO_CPUS: ${DIRECTOR_DEFAULT_MAX_NANO_CPUS}
       DIRECTOR_REGISTRY_CACHING_TTL: ${DIRECTOR_REGISTRY_CACHING_TTL}
       DIRECTOR_REGISTRY_CACHING: ${DIRECTOR_REGISTRY_CACHING}
       DIRECTOR_SELF_SIGNED_SSL_FILENAME: ${DIRECTOR_SELF_SIGNED_SSL_FILENAME}
@@ -256,9 +256,9 @@ services:
       DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS: ${DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS}
       DIRECTOR_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED}
       EXTRA_HOSTS_SUFFIX: undefined
-      LOGLEVEL: ${LOG_LEVEL:-WARNING}
-      MONITORING_ENABLED: ${MONITORING_ENABLED:-True}
-      PUBLISHED_HOST_NAME: ${MACHINE_FQDN}
+      DIRECTOR_LOGLEVEL: ${DIRECTOR_LOGLEVEL}
+      DIRECTOR_MONITORING_ENABLED: ${DIRECTOR_MONITORING_ENABLED}
+      DIRECTOR_PUBLISHED_HOST_NAME: ${DIRECTOR_PUBLISHED_HOST_NAME}
       POSTGRES_DB: ${POSTGRES_DB}
       POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT}
       POSTGRES_HOST: ${POSTGRES_HOST}

services/static-webserver/client/source/class/osparc/file/FileDrop.js

@@ -102,7 +102,9 @@ qx.Class.define("osparc.file.FileDrop", {
         }
       }
       return files;
-    }
+    },
+
+    ONE_FILE_ONLY: qx.locale.Manager.tr("Only one file at a time is accepted.") + "<br>" + qx.locale.Manager.tr("Please zip all files together."),
   },
 
   events: {
@@ -299,7 +301,7 @@ qx.Class.define("osparc.file.FileDrop", {
               pos: this.__pointerFileEventToScreenPos(e)
             });
           } else {
-            osparc.FlashMessenger.getInstance().logAs(this.tr("Only one file at a time is accepted."), "ERROR");
+            osparc.FlashMessenger.getInstance().logAs(osparc.file.FileDrop.ONE_FILE_ONLY, "ERROR");
           }
         } else {
           osparc.FlashMessenger.getInstance().logAs(this.tr("Folders are not accepted. You might want to upload a zip file."), "ERROR");