Merge remote-tracking branch 'upstream/master' into pr-osparc-s3-zip-stream-worker-code

Author: Andrei Neagu

.codecov.yml

@@ -18,7 +18,6 @@ flag_management:
 
 component_management:
   default_rules:
-    carryforward: true
     statuses:
       - type: project
         target: auto

.env-devel

@@ -49,6 +49,8 @@ CATALOG_SERVICES_DEFAULT_RESOURCES='{"CPU": {"limit": 0.1, "reservation": 0.1},
 CATALOG_SERVICES_DEFAULT_SPECIFICATIONS='{}'
 CATALOG_TRACING=null
 
+CELERY_RESULT_EXPIRES=P7D
+
 CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}'
 CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG=master-github-latest
 CLUSTERS_KEEPER_DASK_NTHREADS=0
@@ -250,8 +252,6 @@ SMTP_PROTOCOL=UNENCRYPTED
 SMTP_USERNAME=it_doesnt_matter
 
 # STORAGE ----
-BF_API_KEY=none
-BF_API_SECRET=none
 STORAGE_ENDPOINT=storage:8080
 STORAGE_HOST=storage
 STORAGE_LOGLEVEL=INFO

.github/CODEOWNERS

@@ -4,45 +4,46 @@
 
 
 # files and folders recursively
-.env-*                                    @mrnicegyu11 @YuryHrytsuk
-Makefile                                  @pcrespov @sanderegg
+.env-*                                             @mrnicegyu11 @YuryHrytsuk
+Makefile                                           @pcrespov @sanderegg
 
 
 # NOTE: '/' denotes the root of the repository
-/.github/                                 @sanderegg @pcrespov
-/api/                                     @sanderegg @pcrespov @matusdrobuliak66
-/ci/                                      @sanderegg @pcrespov
-/docs/                                    @pcrespov
-/packages/common-library/                 @giancarloromeo
-/packages/models-library/                 @sanderegg @pcrespov @matusdrobuliak66 @giancarloromeo
-/packages/postgres-database/              @matusdrobuliak66
-/packages/pytest-simcore/                 @pcrespov @sanderegg
-/packages/service-integration/            @pcrespov @sanderegg @GitHK
-/packages/service-library/                @pcrespov
-/packages/settings-library/               @pcrespov @sanderegg
-/requirements/                            @pcrespov @matusdrobuliak66
-/services/agent/                          @GitHK
-/services/api-server/                     @pcrespov
-/services/autoscaling/                    @sanderegg
-/services/catalog/                        @pcrespov @sanderegg
-/services/clusters-keeper/                @sanderegg
-/services/datcore-adapter/                @sanderegg
-/services/director*/                      @sanderegg @pcrespov @GitHK
-/services/docker-compose*.yml             @sanderegg @mrnicegyu11 @YuryHrytsuk
-/services/dynamic-sidecar/                @GitHK
-/services/efs-guardian/                   @matusdrobuliak66
-/services/invitations/                    @pcrespov
-/services/migration/                      @pcrespov
-/services/payments/                       @pcrespov @matusdrobuliak66
-/services/resource-usage-tracker/         @matusdrobuliak66
-/services/static-webserver/               @GitHK
-/services/static-webserver/client/        @odeimaiz
-/services/storage/                        @sanderegg
-/services/web/server/                     @pcrespov @sanderegg @GitHK @matusdrobuliak66
-/tests/e2e-frontend/                      @odeimaiz
-/tests/e2e-playwright/                    @matusdrobuliak66
-/tests/environment-setup/                 @pcrespov
-/tests/performance/                       @pcrespov @sanderegg
-/tests/public-api/                        @pcrespov
-requirements/*                            @pcrespov
-tools/*                                   @pcrespov
+/.github/                                          @sanderegg @pcrespov
+/api/                                              @sanderegg @pcrespov @matusdrobuliak66
+/ci/                                               @sanderegg @pcrespov
+/docs/                                             @pcrespov
+/packages/common-library/                          @giancarloromeo
+/packages/models-library/                          @sanderegg @pcrespov @matusdrobuliak66 @giancarloromeo
+/packages/postgres-database/                       @matusdrobuliak66
+/packages/pytest-simcore/                          @pcrespov @sanderegg
+/packages/service-integration/                     @pcrespov @sanderegg @GitHK
+/packages/service-library/                         @pcrespov
+/packages/settings-library/                        @pcrespov @sanderegg
+/requirements/                                     @pcrespov @matusdrobuliak66
+/services/agent/                                   @GitHK
+/services/api-server/                              @pcrespov
+/services/api-server/tests/unit/pact_broker/       @matusdrobuliak66
+/services/autoscaling/                             @sanderegg
+/services/catalog/                                 @pcrespov @sanderegg
+/services/clusters-keeper/                         @sanderegg
+/services/datcore-adapter/                         @sanderegg
+/services/director*/                               @sanderegg @pcrespov @GitHK
+/services/docker-compose*.yml                      @sanderegg @mrnicegyu11 @YuryHrytsuk
+/services/dynamic-sidecar/                         @GitHK
+/services/efs-guardian/                            @matusdrobuliak66
+/services/invitations/                             @pcrespov
+/services/migration/                               @pcrespov
+/services/payments/                                @pcrespov @matusdrobuliak66
+/services/resource-usage-tracker/                  @matusdrobuliak66
+/services/static-webserver/                        @GitHK
+/services/static-webserver/client/                 @odeimaiz
+/services/storage/                                 @sanderegg
+/services/web/server/                              @pcrespov @sanderegg @GitHK @matusdrobuliak66
+/tests/e2e-frontend/                               @odeimaiz
+/tests/e2e-playwright/                             @matusdrobuliak66
+/tests/environment-setup/                          @pcrespov
+/tests/performance/                                @pcrespov @sanderegg
+/tests/public-api/                                 @pcrespov
+requirements/*                                     @pcrespov
+tools/*                                            @pcrespov

.github/mergify.yml

@@ -12,13 +12,3 @@ queue_rules:
       - "#approved-reviews-by>=2" # Requires 2 approving reviews
       - "#changes-requested-reviews-by=0" # No changes requested
       - "#review-threads-unresolved=0" # All review threads resolved
-
-    merge_conditions: # conditions to be met before being able to merge
-      # list of CI checks that need to pass
-      - check-success=system-test-environment-setup
-      - check-success=changes
-      - check-success=build-test-images
-      - check-success=build-test-images-frontend
-      - check-success=unit-tests
-      - check-success=integration-tests
-      - check-success=system-tests

.github/workflows/ci-pact-master.yml

@@ -0,0 +1,41 @@
+# This workflow holds jobs which are required to pass before merging into master
+
+name: CI PACT Master
+on:
+  push:
+    branches:
+      - "master"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  pact-tests:
+    timeout-minutes: 10
+    name: "Run PACT tests"
+    runs-on: ubuntu-latest
+    env:
+      # secrets can be set in settings/secrets on github
+      PACT_BROKER_URL: ${{ secrets.PACT_BROKER_URL }}
+      PACT_BROKER_USERNAME: ${{ secrets.PACT_BROKER_USERNAME }}
+      PACT_BROKER_PASSWORD: ${{ secrets.PACT_BROKER_PASSWORD }}
+    steps:
+      - name: setup python environment
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.5.x"
+          enable-cache: false
+      - name: checkout source branch
+        uses: actions/checkout@v4
+      - name: Run pact tests
+        run: |
+          make devenv
+          source .venv/bin/activate
+          cd services/api-server
+          make install-ci
+          make test-pacts

.github/workflows/ci-testing-pull-request.yml

@@ -10,6 +10,20 @@ on:
     branches:
       - "master"
 
+  workflow_dispatch:
+    inputs:
+      target_repo:
+        description: full repository name (e.g. 'ITISFoundation/osparc-simcore')
+        required: true
+        default: "ITISFoundation/osparc-simcore"
+        type: environment
+      target_branch:
+        description: Check backwards compatibility against target_branch in target_repo
+        required: true
+        default: "master"
+        type: environment
+
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -39,7 +53,7 @@ jobs:
       - name: Check openapi specs are up to date
         run: |
           if ! ./ci/github/helpers/openapi-specs-diff.bash diff \
-            https://raw.githubusercontent.com/${{ github.event.pull_request.head.repo.full_name }}/${{ github.event.pull_request.head.sha }} \
+            "https://raw.githubusercontent.com/$GITHUB_REPOSITORY/$GITHUB_SHA" \
             .; then \
             echo "::error:: OAS are not up to date. Run 'make openapi-specs' to update them"; exit 1; \
           fi
@@ -56,10 +70,19 @@ jobs:
           python-version: "3.11"
       - name: checkout
         uses: actions/checkout@v4
+      - name: Set environment variables based on event type
+        run: |
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            echo "REPO=${{ inputs.target_repo }}" >> $GITHUB_ENV
+            echo "BRANCH=${{ inputs.target_branch }}" >> $GITHUB_ENV
+          else
+            echo "REPO=${{ github.event.pull_request.base.repo.full_name }}" >> $GITHUB_ENV
+            echo "BRANCH=${{ github.base_ref }}" >> $GITHUB_ENV
+          fi
       - name: check api-server backwards compatibility
         run: |
           ./scripts/openapi-diff.bash breaking --fail-on ERR\
-            https://raw.githubusercontent.com/${{ github.event.pull_request.base.repo.full_name }}/${{ github.event.after }}/services/api-server/openapi.json \
+            "https://raw.githubusercontent.com/$REPO/refs/heads/$BRANCH/services/api-server/openapi.json" \
             /specs/services/api-server/openapi.json
 
   all-oas-breaking:
@@ -75,8 +98,17 @@ jobs:
           python-version: "3.11"
       - name: checkout
         uses: actions/checkout@v4
+      - name: Set environment variables based on event type
+        run: |
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            echo "REPO=${{ inputs.target_repo }}" >> $GITHUB_ENV
+            echo "BRANCH=${{ inputs.target_branch }}" >> $GITHUB_ENV
+          else
+            echo "REPO=${{ github.event.pull_request.base.repo.full_name }}" >> $GITHUB_ENV
+            echo "BRANCH=${{ github.base_ref }}" >> $GITHUB_ENV
+          fi
       - name: Check openapi-specs backwards compatibility
         run: |
           ./ci/github/helpers/openapi-specs-diff.bash breaking \
-            https://raw.githubusercontent.com/${{ github.event.pull_request.base.repo.full_name }}/${{ github.event.after }} \
+            "https://raw.githubusercontent.com/$REPO/refs/heads/$BRANCH" \
             .

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ default_language_version:
   python: python3.11
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.2.0
+    rev: v5.0.0
     hooks:
       - id: check-added-large-files
         args: ["--maxkb=1024"]
@@ -22,7 +22,7 @@ repos:
       - id: no-commit-to-branch
   # NOTE: Keep order as pyupgrade (will update code) then pycln (remove unused imports), then isort (sort them) and black (final formatting)
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.34.0
+    rev: v3.19.1
     hooks:
       - id: pyupgrade
         args:
@@ -36,13 +36,13 @@ repos:
         args: [--all, --expand-stars]
         name: prune imports
   - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+    rev: 6.0.0
     hooks:
       - id: isort
         args: ["--profile", "black"]
         name: sort imports
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 25.1.0
     hooks:
       - id: black
         name: black format code

api/specs/web-server/Makefile

@@ -16,3 +16,6 @@ install-dev install: _check_venv_active
 .PHONY: all
 all: _check_venv_active install
 	python openapi.py
+
+.PHONY: openapi-specs
+openapi-specs: all

api/specs/web-server/_auth.py

@@ -47,8 +47,7 @@
     operation_id="request_product_account",
     status_code=status.HTTP_204_NO_CONTENT,
 )
-async def request_product_account(_body: AccountRequestInfo):
-    ...
+async def request_product_account(_body: AccountRequestInfo): ...
 
 
 @router.post(
@@ -75,8 +74,7 @@ async def register(_body: RegisterBody):
     status_code=status.HTTP_200_OK,
     responses={status.HTTP_409_CONFLICT: {"model": EnvelopedError}},
 )
-async def unregister_account(_body: UnregisterCheck):
-    ...
+async def unregister_account(_body: UnregisterCheck): ...
 
 
 @router.post(
@@ -171,26 +169,24 @@ async def check_auth():
 @router.post(
     "/auth/reset-password",
     response_model=Envelope[Log],
-    operation_id="auth_reset_password",
+    operation_id="initiate_reset_password",
     responses={status.HTTP_503_SERVICE_UNAVAILABLE: {"model": EnvelopedError}},
 )
-async def reset_password(_body: ResetPasswordBody):
-    """a non logged-in user requests a password reset"""
+async def initiate_reset_password(_body: ResetPasswordBody): ...
 
 
 @router.post(
     "/auth/reset-password/{code}",
     response_model=Envelope[Log],
-    operation_id="auth_reset_password_allowed",
+    operation_id="complete_reset_password",
     responses={
         status.HTTP_401_UNAUTHORIZED: {
             "model": EnvelopedError,
-            "description": "unauthorized reset due to invalid token code",
+            "description": "Invalid token code",
         }
     },
 )
-async def reset_password_allowed(code: str, _body: ResetPasswordConfirmation):
-    """changes password using a token code without being logged in"""
+async def complete_reset_password(code: str, _body: ResetPasswordConfirmation): ...
 
 
 @router.post(
@@ -268,5 +264,4 @@ async def email_confirmation(code: str):
     status_code=status.HTTP_200_OK,
     responses={status.HTTP_200_OK: {"content": {"image/png": {}}}},
 )
-async def request_captcha():
-    ...
+async def request_captcha(): ...

api/specs/web-server/_auth_api_keys.py

@@ -9,8 +9,10 @@
 from models_library.generics import Envelope
 from models_library.rest_error import EnvelopedError
 from simcore_service_webserver._meta import API_VTAG
-from simcore_service_webserver.api_keys._exceptions_handlers import _TO_HTTP_ERROR_MAP
-from simcore_service_webserver.api_keys._rest import ApiKeysPathParams
+from simcore_service_webserver.api_keys._controller_rest import ApiKeysPathParams
+from simcore_service_webserver.api_keys._controller_rest_exceptions import (
+    _TO_HTTP_ERROR_MAP,
+)
 
 router = APIRouter(
     prefix=f"/{API_VTAG}",