merge master into bugfix-healthcheck-rabbitmq-cancelled-connection

Author: bisgaard-itis

.codecov.yml

@@ -18,6 +18,7 @@ flag_management:
 
 component_management:
   default_rules:
+    carryforward: true
     statuses:
       - type: project
         target: auto

.env-devel

@@ -21,6 +21,7 @@ AGENT_VOLUMES_CLEANUP_S3_SECRET_KEY=12345678
 API_SERVER_DEV_FEATURES_ENABLED=0
 API_SERVER_LOGLEVEL=INFO
 API_SERVER_PROFILING=1
+API_SERVER_TRACING={}
 TRAEFIK_API_SERVER_INFLIGHTREQ_AMOUNT=25
 
 AUTOSCALING_DASK=null
@@ -33,6 +34,7 @@ AUTOSCALING_LOGLEVEL=INFO
 AUTOSCALING_NODES_MONITORING=null
 AUTOSCALING_POLL_INTERVAL=10
 AUTOSCALING_SSM_ACCESS=null
+AUTOSCALING_TRACING={}
 
 AWS_S3_CLI_S3=null
 
@@ -44,6 +46,7 @@ CATALOG_PORT=8000
 CATALOG_PROFILING=1
 CATALOG_SERVICES_DEFAULT_RESOURCES='{"CPU": {"limit": 0.1, "reservation": 0.1}, "RAM": {"limit": 2147483648, "reservation": 2147483648}}'
 CATALOG_SERVICES_DEFAULT_SPECIFICATIONS='{}'
+CATALOG_TRACING={}
 
 CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}'
 CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG=master-github-latest
@@ -57,6 +60,7 @@ CLUSTERS_KEEPER_MAX_MISSED_HEARTBEATS_BEFORE_CLUSTER_TERMINATION=5
 CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES=null
 CLUSTERS_KEEPER_TASK_INTERVAL=30
 CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES=null
+CLUSTERS_KEEPER_TRACING={}
 
 DASK_SCHEDULER_HOST=dask-scheduler
 DASK_SCHEDULER_PORT=8786
@@ -70,6 +74,7 @@ DIRECTOR_HOST=director
 DIRECTOR_PORT=8080
 DIRECTOR_REGISTRY_CACHING_TTL=900
 DIRECTOR_REGISTRY_CACHING=True
+DIRECTOR_TRACING={}
 
 EFS_USER_ID=8006
 EFS_USER_NAME=efs
@@ -79,6 +84,10 @@ EFS_DNS_NAME=fs-xxx.efs.us-east-1.amazonaws.com
 EFS_MOUNTED_PATH=/tmp/efs
 EFS_PROJECT_SPECIFIC_DATA_DIRECTORY=project-specific-data
 EFS_ONLY_ENABLED_FOR_USERIDS=[]
+EFS_GUARDIAN_TRACING={}
+
+# DATCORE_ADAPTER
+DATCORE_ADAPTER_TRACING={}
 
 # DIRECTOR_V2 ----
 COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}'
@@ -88,7 +97,7 @@ COMPUTATIONAL_BACKEND_DEFAULT_FILE_LINK_TYPE=PRESIGNED
 COMPUTATIONAL_BACKEND_ON_DEMAND_CLUSTERS_FILE_LINK_TYPE=PRESIGNED
 DIRECTOR_V2_DEV_FEATURES_ENABLED=0
 DIRECTOR_V2_DYNAMIC_SCHEDULER_CLOSE_SERVICES_VIA_FRONTEND_WHEN_CREDITS_LIMIT_REACHED=1
-DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL=0
+DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL=PT0S
 DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}'
 DIRECTOR_V2_HOST=director-v2
 DIRECTOR_V2_LOGLEVEL=INFO
@@ -105,10 +114,13 @@ DYNAMIC_SIDECAR_LOG_LEVEL=DEBUG
 DYNAMIC_SIDECAR_PROMETHEUS_MONITORING_NETWORKS=[]
 DYNAMIC_SIDECAR_PROMETHEUS_SERVICE_LABELS={}
 DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT=3600
-# DIRECTOR_V2 ----
+DIRECTOR_V2_TRACING={}
+
+# DYNAMIC_SCHEDULER ----
 DYNAMIC_SCHEDULER_LOGLEVEL=DEBUG
 DYNAMIC_SCHEDULER_PROFILING=1
-DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=3600
+DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=PT1H
+DYNAMIC_SCHEDULER_TRACING={}
 
 FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "[email protected]", "affiliation": "unknown"}}'
 
@@ -122,6 +134,7 @@ INVITATIONS_PORT=8000
 INVITATIONS_SECRET_KEY='REPLACE_ME_with_result__Fernet_generate_key='
 INVITATIONS_SWAGGER_API_DOC_ENABLED=1
 INVITATIONS_USERNAME=admin
+INVITATIONS_TRACING={}
 
 LOG_FORMAT_LOCAL_DEV_ENABLED=1
 LOG_FILTER_MAPPING='{}'
@@ -146,6 +159,7 @@ PAYMENTS_STRIPE_API_SECRET='REPLACE_ME_with_api_secret'
 PAYMENTS_STRIPE_URL=https://api.stripe.com
 PAYMENTS_SWAGGER_API_DOC_ENABLED=1
 PAYMENTS_USERNAME=admin
+PAYMENTS_TRACING={}
 
 POSTGRES_DB=simcoredb
 POSTGRES_ENDPOINT=postgres:5432
@@ -185,6 +199,7 @@ RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_CHECK_ENABLED=1
 RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_COUNTER_FAIL=6
 RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC=300
 RESOURCE_USAGE_TRACKER_S3=null
+RESOURCE_USAGE_TRACKER_TRACING={}
 
 # NOTE: 172.17.0.1 is the docker0 interface, which redirect from inside a container onto the host network interface.
 R_CLONE_OPTION_BUFFER_SIZE=0M
@@ -218,6 +233,7 @@ STORAGE_HOST=storage
 STORAGE_LOGLEVEL=INFO
 STORAGE_PORT=8080
 STORAGE_PROFILING=1
+STORAGE_TRACING={}
 # STORAGE ----
 
 SWARM_STACK_NAME=master-simcore

.github/workflows/ci-testing-deploy.yml

@@ -450,6 +450,51 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
+  unit-test-webserver-04:
+      needs: changes
+      if: ${{ needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
+      timeout-minutes: 25 # if this timeout gets too small, then split the tests
+      name: "[unit] webserver 04"
+      runs-on: ${{ matrix.os }}
+      strategy:
+        matrix:
+          python: ["3.11"]
+          os: [ubuntu-22.04]
+        fail-fast: false
+      steps:
+        - uses: actions/checkout@v4
+        - name: setup docker buildx
+          id: buildx
+          uses: docker/setup-buildx-action@v3
+          with:
+            driver: docker-container
+        - name: setup python environment
+          uses: actions/setup-python@v5
+          with:
+            python-version: ${{ matrix.python }}
+        - name: install uv
+          uses: astral-sh/setup-uv@v3
+          with:
+            version: "0.4.x"
+            enable-cache: false
+            cache-dependency-glob: "**/web/server/requirements/ci.txt"
+        - name: show system version
+          run: ./ci/helpers/show_system_versions.bash
+        - name: install webserver
+          run: ./ci/github/unit-testing/webserver.bash install
+        - name: test
+          run: ./ci/github/unit-testing/webserver.bash test_with_db 04
+        - uses: codecov/[email protected]
+          env:
+            CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+          with:
+            flags: unittests #optional
+        - name: Upload test results to Codecov
+          if: ${{ !cancelled() }}
+          uses: codecov/test-results-action@v1
+          with:
+            token: ${{ secrets.CODECOV_TOKEN }}
+
   unit-test-storage:
     needs: changes
     if: ${{ needs.changes.outputs.storage == 'true' || github.event_name == 'push' }}
@@ -1875,6 +1920,7 @@ jobs:
         unit-test-webserver-01,
         unit-test-webserver-02,
         unit-test-webserver-03,
+        unit-test-webserver-04,
       ]
     runs-on: ubuntu-latest
     steps:

api/specs/web-server/_folders.py

@@ -21,7 +21,7 @@
 from models_library.workspaces import WorkspaceID
 from pydantic import Json
 from simcore_service_webserver._meta import API_VTAG
-from simcore_service_webserver.folders._folders_handlers import FoldersPathParams
+from simcore_service_webserver.folders._models import FolderFilters, FoldersPathParams
 
 router = APIRouter(
     prefix=f"/{API_VTAG}",
@@ -30,8 +30,6 @@
     ],
 )
 
-### Folders
-
 
 @router.post(
     "/folders",
@@ -57,6 +55,10 @@ async def list_folders(
             example='{"field": "name", "direction": "desc"}',
         ),
     ] = '{"field": "modified_at", "direction": "desc"}',
+    filters: Annotated[
+        Json | None,
+        Query(description=FolderFilters.schema_json(indent=1)),
+    ] = None,
 ):
     ...
 

api/specs/web-server/_groups.py

@@ -8,14 +8,21 @@
 
 from fastapi import APIRouter, Depends, status
 from models_library.api_schemas_webserver.groups import (
-    AllUsersGroups,
+    GroupCreate,
+    GroupGet,
+    GroupUpdate,
     GroupUserGet,
-    UsersGroup,
+    MyGroupsGet,
 )
 from models_library.generics import Envelope
-from models_library.users import GroupID, UserID
 from simcore_service_webserver._meta import API_VTAG
-from simcore_service_webserver.groups._handlers import _ClassifiersQuery
+from simcore_service_webserver.groups._handlers import (
+    GroupUserAdd,
+    GroupUserUpdate,
+    _ClassifiersQuery,
+    _GroupPathParams,
+    _GroupUserPathParams,
+)
 from simcore_service_webserver.scicrunch.models import ResearchResource, ResourceHit
 
 router = APIRouter(
@@ -28,106 +35,130 @@
 
 @router.get(
     "/groups",
-    response_model=Envelope[AllUsersGroups],
+    response_model=Envelope[MyGroupsGet],
 )
 async def list_groups():
-    ...
+    """
+    List all groups (organizations, primary, everyone and products) I belong to
+    """
 
 
 @router.post(
     "/groups",
-    response_model=Envelope[UsersGroup],
+    response_model=Envelope[GroupGet],
     status_code=status.HTTP_201_CREATED,
 )
-async def create_group():
-    ...
+async def create_group(_b: GroupCreate):
+    """
+    Creates an organization group
+    """
 
 
 @router.get(
     "/groups/{gid}",
-    response_model=Envelope[UsersGroup],
+    response_model=Envelope[GroupGet],
 )
-async def get_group(gid: GroupID):
-    ...
+async def get_group(_p: Annotated[_GroupPathParams, Depends()]):
+    """
+    Get an organization group
+    """
 
 
 @router.patch(
     "/groups/{gid}",
-    response_model=Envelope[UsersGroup],
+    response_model=Envelope[GroupGet],
 )
-async def update_group(gid: GroupID, _update: UsersGroup):
-    ...
+async def update_group(
+    _p: Annotated[_GroupPathParams, Depends()],
+    _b: GroupUpdate,
+):
+    """
+    Updates organization groups
+    """
 
 
 @router.delete(
     "/groups/{gid}",
     status_code=status.HTTP_204_NO_CONTENT,
 )
-async def delete_group(gid: GroupID):
-    ...
+async def delete_group(_p: Annotated[_GroupPathParams, Depends()]):
+    """
+    Deletes organization groups
+    """
 
 
 @router.get(
     "/groups/{gid}/users",
     response_model=Envelope[list[GroupUserGet]],
 )
-async def get_group_users(gid: GroupID):
-    ...
+async def get_all_group_users(_p: Annotated[_GroupPathParams, Depends()]):
+    """
+    Gets users in organization groups
+    """
 
 
 @router.post(
     "/groups/{gid}/users",
     status_code=status.HTTP_204_NO_CONTENT,
 )
 async def add_group_user(
-    gid: GroupID,
-    _new: GroupUserGet,
+    _p: Annotated[_GroupPathParams, Depends()],
+    _b: GroupUserAdd,
 ):
-    ...
+    """
+    Adds a user to an organization group
+    """
 
 
 @router.get(
     "/groups/{gid}/users/{uid}",
     response_model=Envelope[GroupUserGet],
 )
 async def get_group_user(
-    gid: GroupID,
-    uid: UserID,
+    _p: Annotated[_GroupUserPathParams, Depends()],
 ):
-    ...
+    """
+    Gets specific user in an organization group
+    """
 
 
 @router.patch(
     "/groups/{gid}/users/{uid}",
     response_model=Envelope[GroupUserGet],
 )
 async def update_group_user(
-    gid: GroupID,
-    uid: UserID,
-    _update: GroupUserGet,
+    _p: Annotated[_GroupUserPathParams, Depends()],
+    _b: GroupUserUpdate,
 ):
-    # FIXME: update type
-    ...
+    """
+    Updates user (access-rights) to an organization group
+    """
 
 
 @router.delete(
     "/groups/{gid}/users/{uid}",
     status_code=status.HTTP_204_NO_CONTENT,
 )
 async def delete_group_user(
-    gid: GroupID,
-    uid: UserID,
+    _p: Annotated[_GroupUserPathParams, Depends()],
 ):
-    ...
+    """
+    Removes a user from an organization group
+    """
+
+
+#
+# Classifiers
+#
 
 
 @router.get(
     "/groups/{gid}/classifiers",
     response_model=Envelope[dict[str, Any]],
 )
 async def get_group_classifiers(
-    gid: GroupID,
-    _query: Annotated[_ClassifiersQuery, Depends()],
+    _p: Annotated[_GroupPathParams, Depends()],
+    _q: Annotated[_ClassifiersQuery, Depends()],
 ):
     ...