Merge remote-tracking branch 'upstream/master' into pr-osparc-resilient-long-running-jobs

Author: Andrei Neagu

.codecov.yml

@@ -18,7 +18,6 @@ flag_management:
 
 component_management:
   default_rules:
-    carryforward: true
     statuses:
       - type: project
         target: auto

.coveragerc

@@ -3,6 +3,7 @@ branch = True
 omit =
     */tests/*
     */generated_code/*
+    */_original_fastapi_encoders.py
 parallel = True
 
 [report]

.env-devel

@@ -49,6 +49,8 @@ CATALOG_SERVICES_DEFAULT_RESOURCES='{"CPU": {"limit": 0.1, "reservation": 0.1},
 CATALOG_SERVICES_DEFAULT_SPECIFICATIONS='{}'
 CATALOG_TRACING=null
 
+CELERY_RESULT_EXPIRES=P7D
+
 CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}'
 CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG=master-github-latest
 CLUSTERS_KEEPER_DASK_NTHREADS=0
@@ -84,10 +86,10 @@ DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS=null
 DIRECTOR_TRACING=null
 
 DOCKER_API_PROXY_HOST=docker-api-proxy
-DOCKER_API_PROXY_PASSWORD=null
+DOCKER_API_PROXY_PASSWORD=admin
 DOCKER_API_PROXY_PORT=8888
 DOCKER_API_PROXY_SECURE=False
-DOCKER_API_PROXY_USER=null
+DOCKER_API_PROXY_USER=admin
 
 EFS_USER_ID=8006
 EFS_USER_NAME=efs
@@ -130,7 +132,7 @@ DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT=01:00:00
 DIRECTOR_V2_TRACING=null
 
 # DYNAMIC_SCHEDULER ----
-DYNAMIC_SCHEDULER_LOGLEVEL=DEBUG
+DYNAMIC_SCHEDULER_LOGLEVEL=INFO
 DYNAMIC_SCHEDULER_PROFILING=1
 DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER=0
 DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=01:00:00
@@ -162,6 +164,9 @@ INVITATIONS_TRACING=null
 LOG_FORMAT_LOCAL_DEV_ENABLED=1
 LOG_FILTER_MAPPING='{}'
 
+NOTIFICATIONS_LOGLEVEL=INFO
+NOTIFICATIONS_TRACING=null
+
 PAYMENTS_ACCESS_TOKEN_EXPIRE_MINUTES=30
 PAYMENTS_ACCESS_TOKEN_SECRET_KEY=2c0411810565e063309be1457009fb39ce023946f6a354e6935107b57676
 PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT=10000
@@ -250,8 +255,6 @@ SMTP_PROTOCOL=UNENCRYPTED
 SMTP_USERNAME=it_doesnt_matter
 
 # STORAGE ----
-BF_API_KEY=none
-BF_API_SECRET=none
 STORAGE_ENDPOINT=storage:8080
 STORAGE_HOST=storage
 STORAGE_LOGLEVEL=INFO

.github/CODEOWNERS

@@ -29,16 +29,19 @@ Makefile                                           @pcrespov @sanderegg
 /services/clusters-keeper/                         @sanderegg
 /services/datcore-adapter/                         @sanderegg
 /services/director*/                               @sanderegg @pcrespov @GitHK
-/services/docker-compose*.yml                      @sanderegg @mrnicegyu11 @YuryHrytsuk
+/services/docker-compose.yml                       @sanderegg @mrnicegyu11 @YuryHrytsuk
+/services/docker-compose.*.yml                     @sanderegg
 /services/dynamic-sidecar/                         @GitHK
 /services/efs-guardian/                            @matusdrobuliak66
 /services/invitations/                             @pcrespov
 /services/migration/                               @pcrespov
+/services/notifications/                           @GitHK
 /services/payments/                                @pcrespov @matusdrobuliak66
 /services/resource-usage-tracker/                  @matusdrobuliak66
 /services/static-webserver/                        @GitHK
 /services/static-webserver/client/                 @odeimaiz
 /services/storage/                                 @sanderegg
+/services/storage/modules/celery                   @giancarloromeo
 /services/web/server/                              @pcrespov @sanderegg @GitHK @matusdrobuliak66
 /tests/e2e-frontend/                               @odeimaiz
 /tests/e2e-playwright/                             @matusdrobuliak66

.github/PULL_REQUEST_TEMPLATE.md

@@ -10,6 +10,7 @@
   ⬆️    Upgrade dependencies.
   📝    Add or update documentation.
   🔨    Add or update development scripts.
+  ✅    Add, update or pass tests.
   🔒️    Fix security issues.
   ⚠️    Changes in ops configuration etc. are required before deploying.
         [ Please add a link to the associated ops-issue or PR, such as in https://github.com/ITISFoundation/osparc-ops-environments or https://git.speag.com/oSparc/osparc-infra ]
@@ -28,32 +29,16 @@ or from https://gitmoji.dev/
 
 
 ## Related issue/s
-
-<!-- Link pull request to an issue
-  SEE https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue
-
-- resolves ITISFoundation/osparc-issues#428
-- fixes #26
--->
+<!-- LINK to other issues and add prefix `closes`, `fixes`, `resolves`-->
 
 
 ## How to test
 
 <!-- Give REVIEWERS some hits or code snippets on how could this be tested -->
 
-## Dev-ops checklist
-
-- [ ] No ENV changes or I properly updated ENV ([read the instruction](https://git.speag.com/oSparc/osparc-ops-deployment-configuration/-/blob/configs/README.md?ref_type=heads#how-to-update-env-variables))
-
-<!-- Some checks that might help your code run stable on production, and help devops assess criticality.
-Modified from https://oschvr.com/posts/what-id-like-as-sre/
+## Dev-ops
 
-- How can DevOps check the health of the service ?
-- How can DevOps safely and gracefully restart the service ?
-- How and why would this code fail ?
-- What kind of metrics are you exposing ?
-- Is there any documentation/design specification for the service ?
-- How (e.g. through which loglines) can DevOps detect unexpected situations that require escalation to human ?
-- What are the resource limitations (CPU, RAM) expected for this service ?
-- Are all relevant variables documented and adjustable via environment variables (i.e. no hardcoded magic numbers) ?
+<!--
+- No changes /updated ENV. SEE https://git.speag.com/oSparc/osparc-ops-deployment-configuration/-/blob/configs/README.md?ref_type=heads#how-to-update-env-variables)
+- SEE docs/devops-checklist.md
 -->

.github/copilot-instructions.md

@@ -0,0 +1,53 @@
+# GitHub Copilot Instructions
+
+This document provides guidelines and best practices for using GitHub Copilot in the `osparc-simcore` repository and other Python and Node.js projects.
+
+## General Guidelines
+
+1. **Use Python 3.11**: Ensure that all Python-related suggestions align with Python 3.11 features and syntax.
+2. **Node.js Compatibility**: For Node.js projects, ensure compatibility with the version specified in the project (e.g., Node.js 14 or later).
+3. **Follow Coding Conventions**: Adhere to the coding conventions outlined in the `docs/coding-conventions.md` file.
+4. **Test-Driven Development**: Write unit tests for all new functions and features. Use `pytest` for Python and appropriate testing frameworks for Node.js.
+5. **Environment Variables**: Use environment variables as specified in `docs/env-vars.md` for configuration. Avoid hardcoding sensitive information.
+6. **Documentation**: Prefer self-explanatory code; add documentation only if explicitly requested by the developer.
+
+## Python-Specific Instructions
+
+- Always use type hints and annotations to improve code clarity and compatibility with tools like `mypy`.
+  - An exception to that rule is in `test_*` functions return type hint must not be added
+- Follow the dependency management practices outlined in `requirements/`.
+- Use `ruff` for code formatting and for linting.
+- Use `black` for code formatting and `pylint` for linting.
+- ensure we use `sqlalchemy` >2 compatible code.
+- ensure we use `pydantic` >2 compatible code.
+- ensure we use `fastapi` >0.100 compatible code
+- use f-string formatting
+- Only add comments in function if strictly necessary
+
+
+### Json serialization
+
+- Generally use `json_dumps`/`json_loads` from `common_library.json_serialization` to built-in `json.dumps` / `json.loads`.
+- Prefer Pydantic model methods (e.g., `model.model_dump_json()`) for serialization.
+
+
+## Node.js-Specific Instructions
+
+- Use ES6+ syntax and features.
+- Follow the `package.json` configuration for dependencies and scripts.
+- Use `eslint` for linting and `prettier` for code formatting.
+- Write modular and reusable code, adhering to the project's structure.
+
+## Copilot Usage Tips
+
+1. **Be Specific**: Provide clear and detailed prompts to Copilot for better suggestions.
+2. **Iterate**: Review and refine Copilot's suggestions to ensure they meet project standards.
+3. **Split Tasks**: Break down complex tasks into smaller, manageable parts for better suggestions.
+4. **Test Suggestions**: Always test Copilot-generated code to ensure it works as expected.
+
+## Additional Resources
+
+- [Python Coding Conventions](../docs/coding-conventions.md)
+- [Environment Variables Guide](../docs/env-vars.md)
+- [Steps to Upgrade Python](../docs/steps-to-upgrade-python.md)
+- [Node.js Installation Script](../scripts/install_nodejs_14.bash)

.github/mergify.yml

@@ -12,13 +12,3 @@ queue_rules:
       - "#approved-reviews-by>=2" # Requires 2 approving reviews
       - "#changes-requested-reviews-by=0" # No changes requested
       - "#review-threads-unresolved=0" # All review threads resolved
-
-    merge_conditions: # conditions to be met before being able to merge
-      # list of CI checks that need to pass
-      - check-success=system-test-environment-setup
-      - check-success=changes
-      - check-success=build-test-images
-      - check-success=build-test-images-frontend
-      - check-success=unit-tests
-      - check-success=integration-tests
-      - check-success=system-tests

.github/workflows/_reusable-build-images.yml

@@ -0,0 +1,61 @@
+name: Reusable Build Images
+
+on:
+  workflow_call:
+    inputs:
+      build-backend:
+        required: true
+        type: boolean
+        description: "Whether backend images should be built"
+      build-frontend:
+        required: true
+        type: boolean
+        description: "Whether frontend images should be built"
+      python-version:
+        required: true
+        type: string
+        description: "Python version to use"
+      os:
+        required: true
+        type: string
+        description: "Runner OS to use"
+
+jobs:
+  build-test-images:
+    timeout-minutes: 30
+    runs-on: ${{ inputs.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: setup docker buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+      - name: expose github runtime for buildx
+        uses: crazy-max/ghaction-github-runtime@v3
+      - name: show system environs
+        run: ./ci/helpers/show_system_versions.bash
+      - name: build backend images
+        if: ${{ inputs.build-backend }}
+        run: |
+          export DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash)
+          mkdir --parents /${{ runner.temp }}/build
+          make build local-dest=/${{ runner.temp }}/build exclude=static-webserver
+      - name: build frontend images
+        if: ${{ inputs.build-frontend }}
+        run: |
+          export DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash)
+          mkdir --parents /${{ runner.temp }}/build
+          make build local-dest=/${{ runner.temp }}/build target=static-webserver
+      - name: upload backend artifacts
+        if: ${{ inputs.build-backend }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend
+          path: /${{ runner.temp }}/build
+      - name: upload frontend artifacts
+        if: ${{ inputs.build-frontend }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-frontend
+          path: /${{ runner.temp }}/build

.github/workflows/ci-arm-build.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        os: [ubuntu-22.04]
+        os: [ubuntu-24.04]
         python: ["3.11"]
     env:
       # secrets can be set in settings/secrets on github

.github/workflows/ci-multi-architecture-fusing.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        os: [ubuntu-22.04]
+        os: [ubuntu-24.04]
         python: ["3.11"]
     env:
       # secrets can be set in settings/secrets on github