Merge branch 'master' into introduce-vip-models-pricing-3-part

Author: matusdrobuliak66

.env-devel

@@ -17,12 +17,12 @@ AGENT_VOLUMES_CLEANUP_S3_ENDPOINT=http://172.17.0.1:9001
 AGENT_VOLUMES_CLEANUP_S3_PROVIDER=MINIO
 AGENT_VOLUMES_CLEANUP_S3_REGION=us-east-1
 AGENT_VOLUMES_CLEANUP_S3_SECRET_KEY=12345678
-AGENT_TRACING={}
+AGENT_TRACING=null
 
 API_SERVER_DEV_FEATURES_ENABLED=0
 API_SERVER_LOGLEVEL=INFO
 API_SERVER_PROFILING=1
-API_SERVER_TRACING={}
+API_SERVER_TRACING=null
 TRAEFIK_API_SERVER_INFLIGHTREQ_AMOUNT=25
 
 AUTOSCALING_DASK=null
@@ -35,7 +35,7 @@ AUTOSCALING_LOGLEVEL=INFO
 AUTOSCALING_NODES_MONITORING=null
 AUTOSCALING_POLL_INTERVAL="00:00:10"
 AUTOSCALING_SSM_ACCESS=null
-AUTOSCALING_TRACING={}
+AUTOSCALING_TRACING=null
 
 AWS_S3_CLI_S3=null
 
@@ -47,7 +47,7 @@ CATALOG_PORT=8000
 CATALOG_PROFILING=1
 CATALOG_SERVICES_DEFAULT_RESOURCES='{"CPU": {"limit": 0.1, "reservation": 0.1}, "RAM": {"limit": 2147483648, "reservation": 2147483648}}'
 CATALOG_SERVICES_DEFAULT_SPECIFICATIONS='{}'
-CATALOG_TRACING={}
+CATALOG_TRACING=null
 
 CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}'
 CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG=master-github-latest
@@ -61,7 +61,7 @@ CLUSTERS_KEEPER_MAX_MISSED_HEARTBEATS_BEFORE_CLUSTER_TERMINATION=5
 CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES=null
 CLUSTERS_KEEPER_TASK_INTERVAL=00:00:30
 CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES=null
-CLUSTERS_KEEPER_TRACING={}
+CLUSTERS_KEEPER_TRACING=null
 
 DASK_SCHEDULER_HOST=dask-scheduler
 DASK_SCHEDULER_PORT=8786
@@ -81,7 +81,7 @@ DIRECTOR_PUBLISHED_HOST_NAME="127.0.0.1:9081"
 DIRECTOR_REGISTRY_CACHING_TTL=00:15:00
 DIRECTOR_REGISTRY_CACHING=True
 DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS=null
-DIRECTOR_TRACING={}
+DIRECTOR_TRACING=null
 
 EFS_USER_ID=8006
 EFS_USER_NAME=efs
@@ -90,11 +90,11 @@ EFS_GROUP_NAME=efs-group
 EFS_DNS_NAME=fs-xxx.efs.us-east-1.amazonaws.com
 EFS_MOUNTED_PATH=/tmp/efs
 EFS_PROJECT_SPECIFIC_DATA_DIRECTORY=project-specific-data
-EFS_GUARDIAN_TRACING={}
+EFS_GUARDIAN_TRACING=null
 EFS_DEFAULT_USER_SERVICE_SIZE_BYTES=10000
 
 # DATCORE_ADAPTER
-DATCORE_ADAPTER_TRACING={}
+DATCORE_ADAPTER_TRACING=null
 
 # DIRECTOR_V2 ----
 COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}'
@@ -121,14 +121,14 @@ DYNAMIC_SIDECAR_LOG_LEVEL=DEBUG
 DYNAMIC_SIDECAR_PROMETHEUS_MONITORING_NETWORKS=[]
 DYNAMIC_SIDECAR_PROMETHEUS_SERVICE_LABELS={}
 DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT=01:00:00
-DIRECTOR_V2_TRACING={}
+DIRECTOR_V2_TRACING=null
 
 # DYNAMIC_SCHEDULER ----
 DYNAMIC_SCHEDULER_LOGLEVEL=DEBUG
 DYNAMIC_SCHEDULER_PROFILING=1
 DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER=0
 DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=01:00:00
-DYNAMIC_SCHEDULER_TRACING={}
+DYNAMIC_SCHEDULER_TRACING=null
 DYNAMIC_SCHEDULER_UI_STORAGE_SECRET=adminadmin
 
 FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "[email protected]", "affiliation": "unknown"}}'
@@ -143,7 +143,7 @@ INVITATIONS_PORT=8000
 INVITATIONS_SECRET_KEY='REPLACE_ME_with_result__Fernet_generate_key='
 INVITATIONS_SWAGGER_API_DOC_ENABLED=1
 INVITATIONS_USERNAME=admin
-INVITATIONS_TRACING={}
+INVITATIONS_TRACING=null
 
 LOG_FORMAT_LOCAL_DEV_ENABLED=1
 LOG_FILTER_MAPPING='{}'
@@ -168,7 +168,7 @@ PAYMENTS_STRIPE_API_SECRET='REPLACE_ME_with_api_secret'
 PAYMENTS_STRIPE_URL=https://api.stripe.com
 PAYMENTS_SWAGGER_API_DOC_ENABLED=1
 PAYMENTS_USERNAME=admin
-PAYMENTS_TRACING={}
+PAYMENTS_TRACING=null
 
 POSTGRES_DB=simcoredb
 POSTGRES_ENDPOINT=postgres:5432
@@ -209,7 +209,7 @@ RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_CHECK_ENABLED=1
 RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_COUNTER_FAIL=6
 RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC=300
 RESOURCE_USAGE_TRACKER_S3=null
-RESOURCE_USAGE_TRACKER_TRACING={}
+RESOURCE_USAGE_TRACKER_TRACING=null
 
 # NOTE: 172.17.0.1 is the docker0 interface, which redirect from inside a container onto the host network interface.
 R_CLONE_OPTION_BUFFER_SIZE=16M
@@ -243,7 +243,7 @@ STORAGE_HOST=storage
 STORAGE_LOGLEVEL=INFO
 STORAGE_PORT=8080
 STORAGE_PROFILING=1
-STORAGE_TRACING={}
+STORAGE_TRACING=null
 # STORAGE ----
 
 SWARM_STACK_NAME=master-simcore
@@ -389,6 +389,6 @@ WEBSERVER_SOCKETIO=1
 WEBSERVER_STATICWEB={}
 WEBSERVER_STUDIES_DISPATCHER={}
 WEBSERVER_TAGS=1
-WEBSERVER_TRACING={}
+WEBSERVER_TRACING=null
 WEBSERVER_USERS={}
 WEBSERVER_VERSION_CONTROL=1

.github/workflows/ci-testing-deploy.yml

@@ -2664,11 +2664,14 @@ jobs:
 
   system-api-specs:
     needs: [changes]
-    if: ${{ needs.changes.outputs.anything-py == 'true' && github.event_name == 'push' && github.event.pull_request != null }}
+    if: ${{ needs.changes.outputs.anything-py == 'true' || github.event_name == 'push' }}
     timeout-minutes: 10
     name: "[sys] check api-specs are up to date"
     runs-on: ubuntu-latest
     steps:
+      - name: Ensure job passes if not PR # ensure pass so upstream jobs which depend on this will run (dockerhub deployment)
+        if: ${{ github.event.pull_request == null }}
+        run: echo "::notice Passing job because not in PR"; exit 0
       - name: setup python environment
         uses: actions/setup-python@v5
         with:
@@ -2690,11 +2693,14 @@ jobs:
 
   system-backwards-compatibility:
     needs: [changes, system-api-specs]
-    if: ${{ needs.changes.outputs.anything-py == 'true' && github.event_name == 'push' && github.event.pull_request != null }}
+    if: ${{ needs.changes.outputs.anything-py == 'true' || github.event_name == 'push' }}
     timeout-minutes: 10
     name: "[sys] api-server backwards compatibility"
     runs-on: ubuntu-latest
     steps:
+      - name: Ensure job passes if not PR # ensure pass so upstream jobs which depend on this will run (dockerhub deployment)
+        if: ${{ github.event.pull_request == null }}
+        run: echo "::notice Passing job because not in PR"; exit 0
       - name: setup python environment
         uses: actions/setup-python@v5
         with:

packages/common-library/src/common_library/unset.py

@@ -0,0 +1,12 @@
+from typing import Any
+
+
+class UnSet:
+    VALUE: "UnSet"
+
+
+UnSet.VALUE = UnSet()
+
+
+def as_dict_exclude_unset(**params) -> dict[str, Any]:
+    return {k: v for k, v in params.items() if not isinstance(v, UnSet)}

packages/common-library/tests/test_unset.py

@@ -0,0 +1,15 @@
+from typing import Any
+
+from common_library.unset import UnSet, as_dict_exclude_unset
+
+
+def test_as_dict_exclude_unset():
+    def f(
+        par1: str | UnSet = UnSet.VALUE, par2: int | UnSet = UnSet.VALUE
+    ) -> dict[str, Any]:
+        return as_dict_exclude_unset(par1=par1, par2=par2)
+
+    assert f() == {}
+    assert f(par1="hi") == {"par1": "hi"}
+    assert f(par2=4) == {"par2": 4}
+    assert f(par1="hi", par2=4) == {"par1": "hi", "par2": 4}

packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_service.py

@@ -92,32 +92,33 @@ class RunningDynamicServiceDetails(ServiceDetails):
         ignored_types=(cached_property,),
         json_schema_extra={
             "examples": [
+                # legacy
                 {
-                    "boot_type": "V0",
-                    "key": "simcore/services/dynamic/3dviewer",
-                    "version": "2.4.5",
-                    "user_id": 234,
-                    "project_id": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe",
-                    "uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa",
-                    "basepath": "/x/75c7f3f4-18f9-4678-8610-54a2ade78eaa",
-                    "host": "3dviewer_75c7f3f4-18f9-4678-8610-54a2ade78eaa",
-                    "internal_port": 8888,
-                    "state": "running",
-                    "message": "",
-                    "node_uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa",
+                    "service_key": "simcore/services/dynamic/raw-graphs",
+                    "service_version": "2.10.6",
+                    "user_id": 1,
+                    "project_id": "32fb4eb6-ab30-11ef-9ee4-0242ac140008",
+                    "service_uuid": "0cd049ba-cd6b-4a12-b416-a50c9bc8e7bb",
+                    "service_basepath": "/x/0cd049ba-cd6b-4a12-b416-a50c9bc8e7bb",
+                    "service_host": "raw-graphs_0cd049ba-cd6b-4a12-b416-a50c9bc8e7bb",
+                    "service_port": 4000,
+                    "published_port": None,
+                    "entry_point": "",
+                    "service_state": "running",
+                    "service_message": "",
                 },
+                # new style
                 {
+                    "service_key": "simcore/services/dynamic/jupyter-math",
+                    "service_version": "3.0.3",
+                    "user_id": 1,
+                    "project_id": "32fb4eb6-ab30-11ef-9ee4-0242ac140008",
+                    "service_uuid": "6e3cad3a-eb64-43de-b476-9ac3c413fd9c",
                     "boot_type": "V2",
-                    "key": "simcore/services/dynamic/dy-static-file-viewer-dynamic-sidecar",
-                    "version": "1.0.0",
-                    "user_id": 234,
-                    "project_id": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe",
-                    "uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa",
-                    "host": "dy-sidecar_75c7f3f4-18f9-4678-8610-54a2ade78eaa",
-                    "internal_port": 80,
-                    "state": "running",
-                    "message": "",
-                    "node_uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa",
+                    "service_host": "dy-sidecar_6e3cad3a-eb64-43de-b476-9ac3c413fd9c",
+                    "service_port": 8888,
+                    "service_state": "running",
+                    "service_message": "",
                 },
             ]
         },

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_scheduler/services.py

@@ -8,8 +8,10 @@
     DynamicServiceStop,
 )
 from models_library.api_schemas_webserver.projects_nodes import NodeGet, NodeGetIdle
+from models_library.projects import ProjectID
 from models_library.projects_nodes_io import NodeID
 from models_library.rabbitmq_basic_types import RPCMethodName
+from models_library.users import UserID
 from pydantic import NonNegativeInt, TypeAdapter
 from servicelib.logging_utils import log_decorator
 from servicelib.rabbitmq import RabbitMQRPCClient
@@ -29,6 +31,24 @@
 _RPC_METHOD_NAME_ADAPTER: TypeAdapter[RPCMethodName] = TypeAdapter(RPCMethodName)
 
 
+@log_decorator(_logger, level=logging.DEBUG)
+async def list_tracked_dynamic_services(
+    rabbitmq_rpc_client: RabbitMQRPCClient,
+    *,
+    user_id: UserID | None = None,
+    project_id: ProjectID | None = None,
+) -> list[DynamicServiceGet]:
+    result = await rabbitmq_rpc_client.request(
+        DYNAMIC_SCHEDULER_RPC_NAMESPACE,
+        _RPC_METHOD_NAME_ADAPTER.validate_python("list_tracked_dynamic_services"),
+        user_id=user_id,
+        project_id=project_id,
+        timeout_s=_RPC_DEFAULT_TIMEOUT_S,
+    )
+    assert isinstance(result, list)  # nosec
+    return result
+
+
 @log_decorator(_logger, level=logging.DEBUG)
 async def get_service_status(
     rabbitmq_rpc_client: RabbitMQRPCClient, *, node_id: NodeID

requirements/constraints.txt

@@ -43,32 +43,15 @@ pytest-asyncio<0.24
 #
 # Bugs
 #
-
+httpx!=0.28.0  # Waiting for fix in respx: https://github.com/lundberg/respx/pull/278
 
 
 
 #
 # Compatibility/coordination -----------------------------------------------------------------------------------------
 #
-
-
 pydantic>=2.10.0    # Avoids inter-version compatibility serialization errors as: _pickle.UnpicklingError: NEWOBJ class argument must be a type, not _AnnotatedAlias
 
-# constraint since https://github.com/MagicStack/uvloop/releases/tag/v0.15.0: drops support for 3.5/3.6  Feb.2021
-uvloop<0.15.0 ; python_version < '3.7'
-
-# All backports libraries add environ markers
-# NOTE: If >second dependency, this will annotate a marker in the compiled requirements file
-#
-async-exit-stack ; python_version < '3.7'
-async-generator ; python_version < '3.7'
-contextvars ; python_version < '3.7'
-dataclasses ; python_version < '3.7'
-importlib-metadata ; python_version < '3.8'
-importlib-resources ; python_version < '3.9'
-typing-extensions ; python_version < '3.7'
-zipp ; python_version < '3.7'
-
 
 
 #