Merge remote-tracking branch 'upstream/master' into add-notifications-service

Author: giancarloromeo

.codecov.yml

@@ -25,9 +25,6 @@ component_management:
         branches:
           - "!master"
   individual_components:
-    - component_id: api
-      paths:
-        - api/**
     - component_id: pkg_aws_library
       paths:
         - packages/aws-library/**
@@ -133,6 +130,7 @@ comment:
 
 
 ignore:
+  - "api/tests"
   - "test_*.py"
   - "**/generated_models/*.py"
   - "**/generated_code/*.py"

.env-devel

@@ -17,12 +17,12 @@ AGENT_VOLUMES_CLEANUP_S3_ENDPOINT=http://172.17.0.1:9001
 AGENT_VOLUMES_CLEANUP_S3_PROVIDER=MINIO
 AGENT_VOLUMES_CLEANUP_S3_REGION=us-east-1
 AGENT_VOLUMES_CLEANUP_S3_SECRET_KEY=12345678
-AGENT_TRACING=null
+AGENT_TRACING={}
 
 API_SERVER_DEV_FEATURES_ENABLED=0
 API_SERVER_LOGLEVEL=INFO
 API_SERVER_PROFILING=1
-API_SERVER_TRACING=null
+API_SERVER_TRACING={}
 TRAEFIK_API_SERVER_INFLIGHTREQ_AMOUNT=25
 
 AUTOSCALING_DASK=null
@@ -35,7 +35,7 @@ AUTOSCALING_LOGLEVEL=INFO
 AUTOSCALING_NODES_MONITORING=null
 AUTOSCALING_POLL_INTERVAL="00:00:10"
 AUTOSCALING_SSM_ACCESS=null
-AUTOSCALING_TRACING=null
+AUTOSCALING_TRACING={}
 
 AWS_S3_CLI_S3=null
 
@@ -47,7 +47,7 @@ CATALOG_PORT=8000
 CATALOG_PROFILING=1
 CATALOG_SERVICES_DEFAULT_RESOURCES='{"CPU": {"limit": 0.1, "reservation": 0.1}, "RAM": {"limit": 2147483648, "reservation": 2147483648}}'
 CATALOG_SERVICES_DEFAULT_SPECIFICATIONS='{}'
-CATALOG_TRACING=null
+CATALOG_TRACING={}
 
 CELERY_RESULT_EXPIRES=P7D
 
@@ -63,7 +63,7 @@ CLUSTERS_KEEPER_MAX_MISSED_HEARTBEATS_BEFORE_CLUSTER_TERMINATION=5
 CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES=null
 CLUSTERS_KEEPER_TASK_INTERVAL=00:00:30
 CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES=null
-CLUSTERS_KEEPER_TRACING=null
+CLUSTERS_KEEPER_TRACING={}
 
 DASK_SCHEDULER_HOST=dask-scheduler
 DASK_SCHEDULER_PORT=8786
@@ -83,7 +83,7 @@ DIRECTOR_PUBLISHED_HOST_NAME="127.0.0.1:9081"
 DIRECTOR_REGISTRY_CACHING_TTL=00:15:00
 DIRECTOR_REGISTRY_CACHING=True
 DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS=null
-DIRECTOR_TRACING=null
+DIRECTOR_TRACING={}
 
 DOCKER_API_PROXY_HOST=docker-api-proxy
 DOCKER_API_PROXY_PASSWORD=admin
@@ -98,11 +98,11 @@ EFS_GROUP_NAME=efs-group
 EFS_DNS_NAME=fs-xxx.efs.us-east-1.amazonaws.com
 EFS_MOUNTED_PATH=/tmp/efs
 EFS_PROJECT_SPECIFIC_DATA_DIRECTORY=project-specific-data
-EFS_GUARDIAN_TRACING=null
+EFS_GUARDIAN_TRACING={}
 EFS_DEFAULT_USER_SERVICE_SIZE_BYTES=10000
 
 # DATCORE_ADAPTER
-DATCORE_ADAPTER_TRACING=null
+DATCORE_ADAPTER_TRACING={}
 
 # DIRECTOR_V2 ----
 COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}'
@@ -128,14 +128,14 @@ DYNAMIC_SIDECAR_LOG_LEVEL=DEBUG
 DYNAMIC_SIDECAR_PROMETHEUS_MONITORING_NETWORKS=[]
 DYNAMIC_SIDECAR_PROMETHEUS_SERVICE_LABELS={}
 DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT=01:00:00
-DIRECTOR_V2_TRACING=null
+DIRECTOR_V2_TRACING={}
 
 # DYNAMIC_SCHEDULER ----
 DYNAMIC_SCHEDULER_LOGLEVEL=INFO
 DYNAMIC_SCHEDULER_PROFILING=1
 DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER=0
 DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=01:00:00
-DYNAMIC_SCHEDULER_TRACING=null
+DYNAMIC_SCHEDULER_TRACING={}
 DYNAMIC_SCHEDULER_UI_STORAGE_SECRET=adminadmin
 
 FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "[email protected]", "affiliation": "unknown"}}'
@@ -158,13 +158,13 @@ INVITATIONS_PORT=8000
 INVITATIONS_SECRET_KEY='REPLACE_ME_with_result__Fernet_generate_key='
 INVITATIONS_SWAGGER_API_DOC_ENABLED=1
 INVITATIONS_USERNAME=admin
-INVITATIONS_TRACING=null
+INVITATIONS_TRACING={}
 
 LOG_FORMAT_LOCAL_DEV_ENABLED=1
-LOG_FILTER_MAPPING='{}'
+LOG_FILTER_MAPPING='{"gunicorn.access":[" /v0/ ", " /v0/health "], "uvicorn.access":[" / "]}'
 
 NOTIFICATIONS_LOGLEVEL=INFO
-NOTIFICATIONS_TRACING=null
+NOTIFICATIONS_TRACING={}
 
 PAYMENTS_ACCESS_TOKEN_EXPIRE_MINUTES=30
 PAYMENTS_ACCESS_TOKEN_SECRET_KEY=2c0411810565e063309be1457009fb39ce023946f6a354e6935107b57676
@@ -186,7 +186,7 @@ PAYMENTS_STRIPE_API_SECRET='REPLACE_ME_with_api_secret'
 PAYMENTS_STRIPE_URL=https://api.stripe.com
 PAYMENTS_SWAGGER_API_DOC_ENABLED=1
 PAYMENTS_USERNAME=admin
-PAYMENTS_TRACING=null
+PAYMENTS_TRACING={}
 
 POSTGRES_DB=simcoredb
 POSTGRES_ENDPOINT=postgres:5432
@@ -227,7 +227,7 @@ RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_CHECK_ENABLED=1
 RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_COUNTER_FAIL=6
 RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC=300
 RESOURCE_USAGE_TRACKER_S3=null
-RESOURCE_USAGE_TRACKER_TRACING=null
+RESOURCE_USAGE_TRACKER_TRACING={}
 
 # NOTE: 172.17.0.1 is the docker0 interface, which redirect from inside a container onto the host network interface.
 R_CLONE_OPTION_BUFFER_SIZE=16M
@@ -259,7 +259,7 @@ STORAGE_HOST=storage
 STORAGE_LOGLEVEL=INFO
 STORAGE_PORT=8080
 STORAGE_PROFILING=1
-STORAGE_TRACING=null
+STORAGE_TRACING={}
 # STORAGE ----
 
 SWARM_STACK_NAME=master-simcore
@@ -300,7 +300,7 @@ WB_GC_SOCKETIO=1
 WB_GC_STATICWEB=null
 WB_GC_STUDIES_DISPATCHER=null
 WB_GC_TAGS=0
-WB_GC_TRACING=null
+WB_GC_TRACING={}
 WB_GC_USERS={}
 WB_GC_WALLETS=0
 
@@ -330,7 +330,7 @@ WB_DB_EL_STATICWEB=null
 WB_DB_EL_STORAGE=null
 WB_DB_EL_STUDIES_DISPATCHER=null
 WB_DB_EL_TAGS=0
-WB_DB_EL_TRACING=null
+WB_DB_EL_TRACING={}
 WB_DB_EL_USERS={}
 WB_DB_EL_WALLETS=0
 
@@ -401,5 +401,5 @@ WEBSERVER_SOCKETIO=1
 WEBSERVER_STATICWEB={}
 WEBSERVER_STUDIES_DISPATCHER={}
 WEBSERVER_TAGS=1
-WEBSERVER_TRACING=null
+WEBSERVER_TRACING={}
 WEBSERVER_USERS={}

api/specs/web-server/_auth.py

@@ -15,7 +15,7 @@
 from models_library.rest_error import EnvelopedError, Log
 from pydantic import BaseModel, Field, confloat
 from simcore_service_webserver._meta import API_VTAG
-from simcore_service_webserver.login._controller.rest.auth import (
+from simcore_service_webserver.login._controller.rest.auth_schemas import (
     LoginBody,
     LoginNextPage,
     LoginTwoFactorAuthBody,
@@ -30,7 +30,7 @@
     PhoneConfirmationBody,
     ResetPasswordConfirmation,
 )
-from simcore_service_webserver.login._controller.rest.registration import (
+from simcore_service_webserver.login._controller.rest.registration_schemas import (
     InvitationCheck,
     InvitationInfo,
     RegisterBody,

packages/aws-library/requirements/_base.txt

@@ -140,6 +140,7 @@ opentelemetry-api==1.34.1
     #   opentelemetry-exporter-otlp-proto-http
     #   opentelemetry-instrumentation
     #   opentelemetry-instrumentation-aio-pika
+    #   opentelemetry-instrumentation-asyncpg
     #   opentelemetry-instrumentation-botocore
     #   opentelemetry-instrumentation-logging
     #   opentelemetry-instrumentation-redis
@@ -160,12 +161,15 @@ opentelemetry-exporter-otlp-proto-http==1.34.1
 opentelemetry-instrumentation==0.55b1
     # via
     #   opentelemetry-instrumentation-aio-pika
+    #   opentelemetry-instrumentation-asyncpg
     #   opentelemetry-instrumentation-botocore
     #   opentelemetry-instrumentation-logging
     #   opentelemetry-instrumentation-redis
     #   opentelemetry-instrumentation-requests
 opentelemetry-instrumentation-aio-pika==0.55b1
     # via -r requirements/../../../packages/service-library/requirements/_base.in
+opentelemetry-instrumentation-asyncpg==0.55b1
+    # via -r requirements/../../../packages/service-library/requirements/_base.in
 opentelemetry-instrumentation-botocore==0.55b1
     # via -r requirements/_base.in
 opentelemetry-instrumentation-logging==0.55b1
@@ -189,6 +193,7 @@ opentelemetry-sdk==1.34.1
 opentelemetry-semantic-conventions==0.55b1
     # via
     #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-asyncpg
     #   opentelemetry-instrumentation-botocore
     #   opentelemetry-instrumentation-redis
     #   opentelemetry-instrumentation-requests

packages/celery-library/requirements/_base.txt

@@ -132,6 +132,7 @@ opentelemetry-api==1.34.1
     #   opentelemetry-exporter-otlp-proto-http
     #   opentelemetry-instrumentation
     #   opentelemetry-instrumentation-aio-pika
+    #   opentelemetry-instrumentation-asyncpg
     #   opentelemetry-instrumentation-logging
     #   opentelemetry-instrumentation-redis
     #   opentelemetry-instrumentation-requests
@@ -150,11 +151,14 @@ opentelemetry-exporter-otlp-proto-http==1.34.1
 opentelemetry-instrumentation==0.55b1
     # via
     #   opentelemetry-instrumentation-aio-pika
+    #   opentelemetry-instrumentation-asyncpg
     #   opentelemetry-instrumentation-logging
     #   opentelemetry-instrumentation-redis
     #   opentelemetry-instrumentation-requests
 opentelemetry-instrumentation-aio-pika==0.55b1
     # via -r requirements/../../../packages/service-library/requirements/_base.in
+opentelemetry-instrumentation-asyncpg==0.55b1
+    # via -r requirements/../../../packages/service-library/requirements/_base.in
 opentelemetry-instrumentation-logging==0.55b1
     # via -r requirements/../../../packages/service-library/requirements/_base.in
 opentelemetry-instrumentation-redis==0.55b1
@@ -174,6 +178,7 @@ opentelemetry-sdk==1.34.1
 opentelemetry-semantic-conventions==0.55b1
     # via
     #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-asyncpg
     #   opentelemetry-instrumentation-redis
     #   opentelemetry-instrumentation-requests
     #   opentelemetry-sdk

packages/celery-library/src/celery_library/backends/_redis.py

@@ -7,7 +7,7 @@
 from pydantic import ValidationError
 from servicelib.celery.models import (
     Task,
-    TaskContext,
+    TaskFilter,
     TaskID,
     TaskMetadata,
     TaskUUID,
@@ -82,10 +82,10 @@ async def get_task_progress(self, task_id: TaskID) -> ProgressReport | None:
             )
             return None
 
-    async def list_tasks(self, context: TaskContext) -> list[Task]:
+    async def list_tasks(self, task_filter: TaskFilter) -> list[Task]:
         search_key = (
             _CELERY_TASK_INFO_PREFIX
-            + build_task_id_prefix(context)
+            + build_task_id_prefix(task_filter)
             + _CELERY_TASK_ID_KEY_SEPARATOR
         )
         search_key_len = len(search_key)

packages/celery-library/src/celery_library/rpc/_async_jobs.py

@@ -4,9 +4,9 @@
 
 from celery.exceptions import CeleryError  # type: ignore[import-untyped]
 from models_library.api_schemas_rpc_async_jobs.async_jobs import (
+    AsyncJobFilter,
     AsyncJobGet,
     AsyncJobId,
-    AsyncJobNameData,
     AsyncJobResult,
     AsyncJobStatus,
 )
@@ -16,7 +16,7 @@
     JobNotDoneError,
     JobSchedulerError,
 )
-from servicelib.celery.models import TaskState
+from servicelib.celery.models import TaskFilter, TaskState
 from servicelib.celery.task_manager import TaskManager
 from servicelib.logging_utils import log_catch
 from servicelib.rabbitmq import RPCRouter
@@ -32,13 +32,14 @@
 
 @router.expose(reraise_if_error_type=(JobSchedulerError,))
 async def cancel(
-    task_manager: TaskManager, job_id: AsyncJobId, job_id_data: AsyncJobNameData
+    task_manager: TaskManager, job_id: AsyncJobId, job_filter: AsyncJobFilter
 ):
     assert task_manager  # nosec
-    assert job_id_data  # nosec
+    assert job_filter  # nosec
+    task_filter = TaskFilter.model_validate(job_filter.model_dump())
     try:
         await task_manager.cancel_task(
-            context=job_id_data.model_dump(),
+            task_filter=task_filter,
             task_uuid=job_id,
         )
     except CeleryError as exc:
@@ -47,14 +48,15 @@ async def cancel(
 
 @router.expose(reraise_if_error_type=(JobSchedulerError,))
 async def status(
-    task_manager: TaskManager, job_id: AsyncJobId, job_id_data: AsyncJobNameData
+    task_manager: TaskManager, job_id: AsyncJobId, job_filter: AsyncJobFilter
 ) -> AsyncJobStatus:
     assert task_manager  # nosec
-    assert job_id_data  # nosec
+    assert job_filter  # nosec
 
+    task_filter = TaskFilter.model_validate(job_filter.model_dump())
     try:
         task_status = await task_manager.get_task_status(
-            context=job_id_data.model_dump(),
+            task_filter=task_filter,
             task_uuid=job_id,
         )
     except CeleryError as exc:
@@ -76,21 +78,23 @@ async def status(
     )
 )
 async def result(
-    task_manager: TaskManager, job_id: AsyncJobId, job_id_data: AsyncJobNameData
+    task_manager: TaskManager, job_id: AsyncJobId, job_filter: AsyncJobFilter
 ) -> AsyncJobResult:
     assert task_manager  # nosec
     assert job_id  # nosec
-    assert job_id_data  # nosec
+    assert job_filter  # nosec
+
+    task_filter = TaskFilter.model_validate(job_filter.model_dump())
 
     try:
         _status = await task_manager.get_task_status(
-            context=job_id_data.model_dump(),
+            task_filter=task_filter,
             task_uuid=job_id,
         )
         if not _status.is_done:
             raise JobNotDoneError(job_id=job_id)
         _result = await task_manager.get_task_result(
-            context=job_id_data.model_dump(),
+            task_filter=task_filter,
             task_uuid=job_id,
         )
     except CeleryError as exc:
@@ -123,13 +127,14 @@ async def result(
 
 @router.expose(reraise_if_error_type=(JobSchedulerError,))
 async def list_jobs(
-    task_manager: TaskManager, filter_: str, job_id_data: AsyncJobNameData
+    task_manager: TaskManager, filter_: str, job_filter: AsyncJobFilter
 ) -> list[AsyncJobGet]:
     _ = filter_
     assert task_manager  # nosec
+    task_filter = TaskFilter.model_validate(job_filter.model_dump())
     try:
         tasks = await task_manager.list_tasks(
-            context=job_id_data.model_dump(),
+            task_filter=task_filter,
         )
     except CeleryError as exc:
         raise JobSchedulerError(exc=f"{exc}") from exc