Merge master in pydantic_v2_migration 11.10.2024

.env-devel

@@ -25,6 +25,8 @@ TRAEFIK_API_SERVER_INFLIGHTREQ_AMOUNT=25
 
 AUTOSCALING_DASK=null
 AUTOSCALING_DRAIN_NODES_WITH_LABELS=False
+AUTOSCALING_DOCKER_JOIN_DRAINED=True
+AUTOSCALING_WAIT_FOR_CLOUD_INIT_BEFORE_WARM_BUFFER_ACTIVATION=False
 AUTOSCALING_EC2_ACCESS=null
 AUTOSCALING_EC2_INSTANCES=null
 AUTOSCALING_LOGLEVEL=WARNING
@@ -48,6 +50,7 @@ CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG=master-github-latest
 CLUSTERS_KEEPER_DASK_NTHREADS=0
 CLUSTERS_KEEPER_DASK_WORKER_SATURATION=inf
 CLUSTERS_KEEPER_EC2_ACCESS=null
+CLUSTERS_KEEPER_SSM_ACCESS=null
 CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX=""
 CLUSTERS_KEEPER_LOGLEVEL=WARNING
 CLUSTERS_KEEPER_MAX_MISSED_HEARTBEATS_BEFORE_CLUSTER_TERMINATION=5
@@ -101,6 +104,7 @@ DYNAMIC_SIDECAR_IMAGE=${DOCKER_REGISTRY:-itisfoundation}/dynamic-sidecar:${DOCKE
 DYNAMIC_SIDECAR_LOG_LEVEL=DEBUG
 DYNAMIC_SIDECAR_PROMETHEUS_MONITORING_NETWORKS=[]
 DYNAMIC_SIDECAR_PROMETHEUS_SERVICE_LABELS={}
+DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT=3600
 # DIRECTOR_V2 ----
 
 DYNAMIC_SCHEDULER_PROFILING=1
@@ -149,6 +153,10 @@ POSTGRES_PASSWORD=adminadmin
 POSTGRES_PORT=5432
 POSTGRES_USER=scu
 
+POSTGRES_READONLY_PASSWORD=readonly
+POSTGRES_READONLY_USER=postgres_readonly
+
+
 RABBIT_HOST=rabbit
 RABBIT_PASSWORD=adminadmin
 RABBIT_PORT=5672
@@ -158,6 +166,8 @@ RABBIT_USER=admin
 REDIS_HOST=redis
 REDIS_PORT=6379
 REDIS_PASSWORD=adminadmin
+REDIS_SECURE=false
+REDIS_USER=null
 
 REGISTRY_AUTH=True
 REGISTRY_PW=adminadminadmin
@@ -211,6 +221,13 @@ STORAGE_PROFILING=1
 
 SWARM_STACK_NAME=master-simcore
 
+## VENDOR DEVELOPMENT SERVICES ---
+VENDOR_DEV_MANUAL_IMAGE=containous/whoami
+VENDOR_DEV_MANUAL_REPLICAS=1
+VENDOR_DEV_MANUAL_SUBDOMAIN=manual
+
+## VENDOR DEVELOPMENT SERVICES ---
+
 WB_API_WEBSERVER_HOST=wb-api-server
 WB_API_WEBSERVER_PORT=8080
 
@@ -282,6 +299,7 @@ WB_DB_EL_WALLETS=0
 
 # WEBSERVER ----
 AIODEBUG_SLOW_DURATION_SECS=0
+DIAGNOSTICS_HEALTHCHECK_ENABLED=False
 DIAGNOSTICS_MAX_AVG_LATENCY=10
 DIAGNOSTICS_MAX_TASK_DELAY=30
 DIAGNOSTICS_SLOW_DURATION_SECS=1
@@ -303,9 +321,11 @@ SESSION_COOKIE_SECURE=False
 SIMCORE_VCS_RELEASE_TAG=latest
 STUDIES_ACCESS_ANONYMOUS_ALLOWED=0
 STUDIES_DEFAULT_SERVICE_THUMBNAIL=https://via.placeholder.com/170x120.png
-TRACING_ENABLED=1
-TRACING_THRIFT_COMPACT_ENDPOINT=http://jaeger:5775
-TRACING_ZIPKIN_ENDPOINT=http://jaeger:9411
+TRACING_OPENTELEMETRY_COLLECTOR_BATCH_SIZE=2
+TRACING_OPENTELEMETRY_COLLECTOR_PORT=4318
+TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT=http://opentelemetry-collector
+TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE=100
+TRACING_OPENTELEMETRY_COLLECTOR_EXPORTER_ENDPOINT=http://jaeger:4318
 TRAEFIK_SIMCORE_ZONE=internal_simcore_stack
 TWILIO_ACCOUNT_SID=DUMMY
 TWILIO_AUTH_TOKEN=DUMMY

.github/dependabot.yml

@@ -12,9 +12,6 @@ updates:
     assignees:
       - pcrespov
     ignore:
-      - dependency-name: aiozipkin
-        versions:
-          - ">= 1.a, < 2"
       - dependency-name: docker-compose
         versions:
           - 1.28.2
@@ -41,9 +38,6 @@ updates:
     assignees:
       - pcrespov
     ignore:
-      - dependency-name: aiozipkin
-        versions:
-          - ">= 1.a, < 2"
       - dependency-name: openapi-core
         versions:
           - "> 0.12.0, < 1"

.github/workflows/ci-testing-deploy.yml

@@ -279,6 +279,18 @@ jobs:
       fail-fast: false
     name: "[build] docker images"
     steps:
+      - name: Remove unused software
+        run: |
+          echo "Available storage before:"
+          sudo df -h
+          echo
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          echo "Available storage after:"
+          sudo df -h
+          echo
       - uses: actions/checkout@v4
       - name: setup docker buildx
         id: buildx
@@ -593,7 +605,7 @@ jobs:
   unit-test-autoscaling:
     needs: changes
     if: ${{ needs.changes.outputs.autoscaling == 'true' || github.event_name == 'push' }}
-    timeout-minutes: 18 # if this timeout gets too small, then split the tests
+    timeout-minutes: 22 # temporary: mypy takes a huge amount of time to run here, maybe we should cache it
     name: "[unit] autoscaling"
     runs-on: ${{ matrix.os }}
     strategy:
@@ -1291,7 +1303,7 @@ jobs:
         uses: docker/setup-buildx-action@v3
         with:
           driver: docker-container
-      - uses: actions/[email protected].3
+      - uses: actions/[email protected].4
         with:
           node-version: ${{ matrix.node }}
           cache: "npm"
@@ -2404,7 +2416,7 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
-      - uses: actions/[email protected].3
+      - uses: actions/[email protected].4
         with:
           node-version: ${{ matrix.node }}
           cache: "npm"
@@ -2530,7 +2542,6 @@ jobs:
           name: ${{ github.job }}_tracing
           path: tests/e2e-playwright/test-results
 
-
   system-test-environment-setup:
     timeout-minutes: 30 # if this timeout gets too small, then split the tests
     name: "[sys] environment setup"

.pylintrc

@@ -466,6 +466,8 @@ max-statements=50
 # Minimum number of public methods for a class (see R0903).
 min-public-methods=2
 
+# Minimum number of public methods for a class (see R0903).
+max-positional-arguments=12
 
 [EXCEPTIONS]
 

.vscode/settings.template.json

@@ -9,11 +9,12 @@
   "files.associations": {
     ".*rc": "ini",
     ".env*": "ini",
+    "*.logs*": "log",
     "**/requirements/*.in": "pip-requirements",
     "**/requirements/*.txt": "pip-requirements",
     "*logs.txt": "log",
-    "*.logs*": "log",
     "*Makefile": "makefile",
+    "*sql.*": "sql",
     "docker-compose*.yml": "dockercompose",
     "Dockerfile*": "dockerfile"
   },

Makefile

@@ -269,6 +269,11 @@ CPU_COUNT = $(shell cat /proc/cpuinfo | grep processor | wc -l )
 		services/docker-compose.local.yml \
 		> $@
 
+.stack-vendor-services.yml: .env $(docker-compose-configs)
+	# Creating config for vendors stack to $@
+	@scripts/docker/docker-stack-config.bash -e $< \
+		services/docker-compose-dev-vendors.yml \
+		> $@
 
 .stack-ops.yml: .env $(docker-compose-configs)
 	# Creating config for ops stack to $@
@@ -288,7 +293,11 @@ endif
 
 
 
-.PHONY: up-devel up-prod up-prod-ci up-version up-latest .deploy-ops
+.PHONY: up-devel up-prod up-prod-ci up-version up-latest .deploy-ops .deploy-vendors
+
+.deploy-vendors: .stack-vendor-services.yml
+	# Deploy stack 'vendors'
+	docker stack deploy --detach=true --with-registry-auth -c $< vendors
 
 .deploy-ops: .stack-ops.yml
 	# Deploy stack 'ops'
@@ -310,18 +319,19 @@ TableWidth=140;\
 printf "%24s | %90s | %12s | %12s\n" Name Endpoint User Password;\
 printf "%.$${TableWidth}s\n" "$$separator";\
 printf "$$rows" "oSparc platform" "http://$(get_my_ip).nip.io:9081";\
-printf "$$rows" "oSparc web API doc" "http://$(get_my_ip).nip.io:9081/dev/doc";\
 printf "$$rows" "oSparc public API doc" "http://$(get_my_ip).nip.io:8006/dev/doc";\
-printf "$$rows" "Postgres DB" "http://$(get_my_ip).nip.io:18080/?pgsql=postgres&username="$${POSTGRES_USER}"&db="$${POSTGRES_DB}"&ns=public" $${POSTGRES_USER} $${POSTGRES_PASSWORD};\
-printf "$$rows" "Portainer" "http://$(get_my_ip).nip.io:9000" admin adminadmin;\
-printf "$$rows" "Redis" "http://$(get_my_ip).nip.io:18081";\
+printf "$$rows" "oSparc web API doc" "http://$(get_my_ip).nip.io:9081/dev/doc";\
 printf "$$rows" "Dask Dashboard" "http://$(get_my_ip).nip.io:8787";\
 printf "$$rows" "Docker Registry" "http://$${REGISTRY_URL}/v2/_catalog" $${REGISTRY_USER} $${REGISTRY_PW};\
 printf "$$rows" "Invitations" "http://$(get_my_ip).nip.io:8008/dev/doc" $${INVITATIONS_USERNAME} $${INVITATIONS_PASSWORD};\
+printf "$$rows" "Jaeger" "http://$(get_my_ip).nip.io:16686";\
 printf "$$rows" "Payments" "http://$(get_my_ip).nip.io:8011/dev/doc" $${PAYMENTS_USERNAME} $${PAYMENTS_PASSWORD};\
+printf "$$rows" "Portainer" "http://$(get_my_ip).nip.io:9000" admin adminadmin;\
+printf "$$rows" "Postgres DB" "http://$(get_my_ip).nip.io:18080/?pgsql=postgres&username="$${POSTGRES_USER}"&db="$${POSTGRES_DB}"&ns=public" $${POSTGRES_USER} $${POSTGRES_PASSWORD};\
 printf "$$rows" "Rabbit Dashboard" "http://$(get_my_ip).nip.io:15672" admin adminadmin;\
-printf "$$rows" "Traefik Dashboard" "http://$(get_my_ip).nip.io:8080/dashboard/";\
+printf "$$rows" "Redis" "http://$(get_my_ip).nip.io:18081";\
 printf "$$rows" "Storage S3 Minio" "http://$(get_my_ip).nip.io:9001" 12345678 12345678;\
+printf "$$rows" "Traefik Dashboard" "http://$(get_my_ip).nip.io:8080/dashboard/";\
 
 printf "\n%s\n" "⚠️ if a DNS is not used (as displayed above), the interactive services started via dynamic-sidecar";\
 echo "⚠️ will not be shown. The frontend accesses them via the uuid.services.YOUR_IP.nip.io:9081";
@@ -337,6 +347,7 @@ up-devel: .stack-simcore-development.yml .init-swarm $(CLIENT_WEB_OUTPUT) ## Dep
 	@$(MAKE_C) services/dask-sidecar certificates
 	# Deploy stack $(SWARM_STACK_NAME) [back-end]
 	@docker stack deploy --detach=true --with-registry-auth -c $< $(SWARM_STACK_NAME)
+	@$(MAKE) .deploy-vendors
 	@$(MAKE) .deploy-ops
 	@$(_show_endpoints)
 	@$(MAKE_C) services/static-webserver/client follow-dev-logs
@@ -347,6 +358,7 @@ up-devel-frontend: .stack-simcore-development-frontend.yml .init-swarm ## Every
 	@$(MAKE_C) services/dask-sidecar certificates
 	# Deploy stack $(SWARM_STACK_NAME)  [back-end]
 	@docker stack deploy --detach=true --with-registry-auth -c $< $(SWARM_STACK_NAME)
+	@$(MAKE) .deploy-vendors
 	@$(MAKE) .deploy-ops
 	@$(_show_endpoints)
 	@$(MAKE_C) services/static-webserver/client follow-dev-logs
@@ -357,6 +369,7 @@ ifeq ($(target),)
 	@$(MAKE_C) services/dask-sidecar certificates
 	# Deploy stack $(SWARM_STACK_NAME)
 	@docker stack deploy --detach=true --with-registry-auth -c $< $(SWARM_STACK_NAME)
+	@$(MAKE) .deploy-vendors
 	@$(MAKE) .deploy-ops
 else
 	# deploys ONLY $(target) service
@@ -368,6 +381,7 @@ up-version: .stack-simcore-version.yml .init-swarm ## Deploys versioned stack '$
 	@$(MAKE_C) services/dask-sidecar certificates
 	# Deploy stack $(SWARM_STACK_NAME)
 	@docker stack deploy --detach=true --with-registry-auth -c $< $(SWARM_STACK_NAME)
+	@$(MAKE) .deploy-vendors
 	@$(MAKE) .deploy-ops
 	@$(_show_endpoints)
 

api/specs/web-server/_auth.py

@@ -155,6 +155,21 @@ async def logout(_body: LogoutBody):
     """user logout"""
 
 
+@router.get(
+    "/auth:check",
+    operation_id="check_authentication",
+    status_code=status.HTTP_204_NO_CONTENT,
+    responses={
+        status.HTTP_401_UNAUTHORIZED: {
+            "model": Envelope[Error],
+            "description": "unauthorized reset due to invalid token code",
+        }
+    },
+)
+async def check_auth():
+    """checks if user is authenticated in the platform"""
+
+
 @router.post(
     "/auth/reset-password",
     response_model=Envelope[Log],

api/specs/web-server/_projects_crud.py

@@ -32,7 +32,10 @@
 from simcore_service_webserver._meta import API_VTAG
 from simcore_service_webserver.projects._common_models import ProjectPathParams
 from simcore_service_webserver.projects._crud_handlers import ProjectCreateParams
-from simcore_service_webserver.projects._crud_handlers_models import ProjectListParams
+from simcore_service_webserver.projects._crud_handlers_models import (
+    ProjectListFullSearchParams,
+    ProjectListParams,
+)
 
 router = APIRouter(
     prefix=f"/{API_VTAG}",
@@ -137,6 +140,23 @@ async def clone_project(
     ...
 
 
+@router.get(
+    "/projects:search",
+    response_model=Page[ProjectListFullSearchParams],
+)
+async def list_projects_full_search(
+    _params: Annotated[ProjectListFullSearchParams, Depends()],
+    order_by: Annotated[
+        Json,
+        Query(
+            description="Order by field (type|uuid|name|description|prj_owner|creation_date|last_change_date) and direction (asc|desc). The default sorting order is ascending.",
+            example='{"field": "last_change_date", "direction": "desc"}',
+        ),
+    ] = ('{"field": "last_change_date", "direction": "desc"}',),
+):
+    ...
+
+
 @router.get(
     "/projects/{project_id}/inactivity",
     response_model=Envelope[GetProjectInactivityResponse],

api/specs/web-server/_resource_usage.py

@@ -13,6 +13,9 @@
 
 from _common import assert_handler_signature_against_model
 from fastapi import APIRouter, Query, status
+from models_library.api_schemas_resource_usage_tracker.service_runs import (
+    OsparcCreditsAggregatedByServiceGet,
+)
 from models_library.api_schemas_webserver.resource_usage import (
     ConnectServiceToPricingPlanBodyParams,
     CreatePricingPlanBodyParams,
@@ -93,7 +96,7 @@ async def list_resource_usage_services(
 
 @router.get(
     "/services/-/aggregated-usages",
-    response_model=Envelope[list[ServiceRunGet]],
+    response_model=Envelope[list[OsparcCreditsAggregatedByServiceGet]],
     summary="Used credits based on aggregate by type, currently supported `services`. (user and product are taken from context, optionally wallet_id parameter might be provided).",
     tags=["usage"],
 )

api/tests/conftest.py

@@ -1,5 +1,6 @@
-# pylint: disable=unused-argument
 # pylint: disable=redefined-outer-name
+# pylint: disable=too-many-positional-arguments
+# pylint: disable=unused-argument
 # pylint: disable=unused-variable
 
 import logging