⬆️ Pydantic v2 migration (#6578)

Co-authored-by: Giancarlo Romeo <[email protected]>
Co-authored-by: Andrei Neagu <[email protected]>
Co-authored-by: Andrei Neagu <[email protected]>
Co-authored-by: Pedro Crespo-Valero <[email protected]>
Co-authored-by: Mads Bisgaard <[email protected]>

Author: 6 people

.env-devel

@@ -33,7 +33,7 @@ AUTOSCALING_EC2_ACCESS=null
 AUTOSCALING_EC2_INSTANCES=null
 AUTOSCALING_LOGLEVEL=INFO
 AUTOSCALING_NODES_MONITORING=null
-AUTOSCALING_POLL_INTERVAL=10
+AUTOSCALING_POLL_INTERVAL="00:00:10"
 AUTOSCALING_SSM_ACCESS=null
 AUTOSCALING_TRACING={}
 
@@ -120,13 +120,13 @@ DYNAMIC_SIDECAR_IMAGE=${DOCKER_REGISTRY:-itisfoundation}/dynamic-sidecar:${DOCKE
 DYNAMIC_SIDECAR_LOG_LEVEL=DEBUG
 DYNAMIC_SIDECAR_PROMETHEUS_MONITORING_NETWORKS=[]
 DYNAMIC_SIDECAR_PROMETHEUS_SERVICE_LABELS={}
-DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT=3600
+DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT=01:00:00
 DIRECTOR_V2_TRACING={}
 
 # DYNAMIC_SCHEDULER ----
 DYNAMIC_SCHEDULER_LOGLEVEL=DEBUG
 DYNAMIC_SCHEDULER_PROFILING=1
-DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=PT1H
+DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=01:00:00
 DYNAMIC_SCHEDULER_TRACING={}
 
 FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "[email protected]", "affiliation": "unknown"}}'

.github/CODEOWNERS

@@ -13,7 +13,8 @@ Makefile                                  @pcrespov @sanderegg
 /api/                                     @sanderegg @pcrespov @matusdrobuliak66
 /ci/                                      @sanderegg @pcrespov
 /docs/                                    @pcrespov
-/packages/models-library/                 @sanderegg @pcrespov @matusdrobuliak66
+/packages/common-library/                 @giancarloromeo
+/packages/models-library/                 @sanderegg @pcrespov @matusdrobuliak66 @giancarloromeo
 /packages/postgres-database/              @matusdrobuliak66
 /packages/pytest-simcore/                 @pcrespov @sanderegg
 /packages/service-integration/            @pcrespov @sanderegg @GitHK

.github/workflows/ci-testing-deploy.yml

@@ -54,6 +54,7 @@ jobs:
       aws-library: ${{ steps.filter.outputs.aws-library }}
       dask-task-models-library: ${{ steps.filter.outputs.dask-task-models-library }}
       models-library: ${{ steps.filter.outputs.models-library }}
+      common-library: ${{ steps.filter.outputs.common-library }}
       notifications-library: ${{ steps.filter.outputs.notifications-library }}
       postgres-database: ${{ steps.filter.outputs.postgres-database }}
       service-integration: ${{ steps.filter.outputs.service-integration }}
@@ -110,6 +111,8 @@ jobs:
               - 'services/docker-compose*'
               - 'scripts/mypy/*'
               - 'mypy.ini'
+            common-library:
+              - 'packages/common-library/**'
             notifications-library:
               - 'packages/notifications-library/**'
               - 'packages/postgres-database/**'
@@ -1799,6 +1802,47 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
+  unit-test-common-library:
+    needs: changes
+    if: ${{ needs.changes.outputs.common-library == 'true' || github.event_name == 'push' }}
+    timeout-minutes: 18 # if this timeout gets too small, then split the tests
+    name: "[unit] common-library"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: ["3.11"]
+        os: [ubuntu-22.04]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+      - name: setup docker buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+      - name: setup python environment
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - name: install uv
+        uses: yezz123/setup-uv@v4
+      - uses: actions/cache@v4
+        id: cache-uv
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
+      - name: show system version
+        run: ./ci/helpers/show_system_versions.bash
+      - name: install
+        run: ./ci/github/unit-testing/common-library.bash install
+      - name: typecheck
+        run: ./ci/github/unit-testing/common-library.bash typecheck
+      - name: test
+        run: ./ci/github/unit-testing/common-library.bash test
+      - uses: codecov/[email protected]
+        with:
+          flags: unittests #optional
+
   unit-test-notifications-library:
     needs: changes
     if: ${{ needs.changes.outputs.notifications-library == 'true' || github.event_name == 'push' }}
@@ -1919,6 +1963,7 @@ jobs:
         unit-test-dynamic-sidecar,
         unit-test-efs-guardian,
         unit-test-models-library,
+        unit-test-common-library,
         unit-test-notifications-library,
         unit-test-osparc-gateway-server,
         unit-test-payments,

api/specs/web-server/_common.py

@@ -8,9 +8,10 @@
 from typing import Any, ClassVar, NamedTuple
 
 import yaml
+from common_library.json_serialization import json_dumps
+from common_library.pydantic_fields_extension import get_type
 from fastapi import FastAPI, Query
 from models_library.basic_types import LogLevel
-from models_library.utils.json_serialization import json_dumps
 from pydantic import BaseModel, Field, create_model
 from pydantic.fields import FieldInfo
 from servicelib.fastapi.openapi import override_fastapi_openapi_method
@@ -37,31 +38,28 @@ def __modify_schema__(cls, field_schema: dict[str, Any]) -> None:
 
 def as_query(model_class: type[BaseModel]) -> type[BaseModel]:
     fields = {}
-    for field_name, model_field in model_class.__fields__.items():
+    for field_name, field_info in model_class.model_fields.items():
 
-        field_type = model_field.type_
-        default_value = model_field.default
+        field_type = get_type(field_info)
+        default_value = field_info.default
 
         kwargs = {
-            "alias": model_field.field_info.alias,
-            "title": model_field.field_info.title,
-            "description": model_field.field_info.description,
-            "gt": model_field.field_info.gt,
-            "ge": model_field.field_info.ge,
-            "lt": model_field.field_info.lt,
-            "le": model_field.field_info.le,
-            "min_length": model_field.field_info.min_length,
-            "max_length": model_field.field_info.max_length,
-            "regex": model_field.field_info.regex,
-            **model_field.field_info.extra,
+            "alias": field_info.alias,
+            "title": field_info.title,
+            "description": field_info.description,
+            "metadata": field_info.metadata,
+            "json_schema_extra": field_info.json_schema_extra,
         }
 
         if issubclass(field_type, BaseModel):
             # Complex fields
+            assert "json_schema_extra" in kwargs  # nosec
+            assert kwargs["json_schema_extra"]  # nosec
             field_type = _create_json_type(
                 description=kwargs["description"],
-                example=kwargs.get("example_json"),
+                example=kwargs.get("json_schema_extra", {}).get("example_json"),
             )
+
             default_value = json_dumps(default_value) if default_value else None
 
         fields[field_name] = (field_type, Query(default=default_value, **kwargs))
@@ -147,7 +145,7 @@ def create_and_save_openapi_specs(
     )
     with file_path.open("wt") as fh:
         yaml.safe_dump(openapi, fh, indent=1, sort_keys=False)
-    print("Saved OAS to", file_path)
+    print("Saved OAS to", file_path)  # noqa: T201
 
 
 class ParamSpec(NamedTuple):
@@ -169,8 +167,8 @@ def assert_handler_signature_against_model(
 
     # query and path parameters
     implemented_params = [
-        ParamSpec(field.name, field.type_, field.field_info)
-        for field in model_cls.__fields__.values()
+        ParamSpec(name, get_type(info), info)
+        for name, info in model_cls.model_fields.items()
     ]
 
     implemented_names = {p.name for p in implemented_params}

ci/github/unit-testing/common-library.bash

@@ -0,0 +1,43 @@
+#!/bin/bash
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -o errexit  # abort on nonzero exitstatus
+set -o nounset  # abort on unbound variable
+set -o pipefail # don't hide errors within pipes
+IFS=$'\n\t'
+
+install() {
+  make devenv
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  pushd packages/common-library
+  make install-ci
+  popd
+  uv pip list
+}
+
+test() {
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  pushd packages/common-library
+  make tests-ci
+  popd
+}
+
+typecheck() {
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  uv pip install mypy
+  pushd packages/common-library
+  make mypy
+  popd
+}
+
+# Check if the function exists (bash specific)
+if declare -f "$1" >/dev/null; then
+  # call arguments verbatim
+  "$@"
+else
+  # Show a helpful error
+  echo "'$1' is not a known function name" >&2
+  exit 1
+fi

ci/helpers/requirements.txt

@@ -1,17 +1,19 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip compile requirements.in
+#    uv pip compile requirements.in -o requirements.txt
 aiohttp==3.9.5
+    # via
+    #   -c ../../requirements/constraints.txt
+    #   -r requirements.in
 aiosignal==1.3.1
     # via aiohttp
+annotated-types==0.7.0
+    # via pydantic
 anyio==4.3.0
     # via starlette
-async-timeout==4.0.3
-    # via aiohttp
 attrs==23.2.0
     # via aiohttp
-exceptiongroup==1.2.1
-    # via anyio
-fastapi==0.99.1
+fastapi==0.115.0
+    # via -r requirements.in
 frozenlist==1.4.1
     # via
     #   aiohttp
@@ -24,16 +26,22 @@ multidict==6.0.5
     # via
     #   aiohttp
     #   yarl
-pydantic==1.10.15
-    # via fastapi
+pydantic==2.9.2
+    # via
+    #   -c ../../requirements/constraints.txt
+    #   fastapi
+pydantic-core==2.23.4
+    # via pydantic
 sniffio==1.3.1
     # via anyio
-starlette==0.27.0
-    # via fastapi
+starlette==0.38.6
+    # via
+    #   -c ../../requirements/constraints.txt
+    #   fastapi
 typing-extensions==4.11.0
     # via
-    #   anyio
     #   fastapi
     #   pydantic
+    #   pydantic-core
 yarl==1.9.4
     # via aiohttp

packages/aws-library/requirements/_base.in

@@ -2,6 +2,7 @@
 # Specifies third-party dependencies for 'aws-library'
 #
 --constraint ../../../requirements/constraints.txt
+--requirement ../../../packages/common-library/requirements/_base.in
 --requirement ../../../packages/models-library/requirements/_base.in
 --requirement ../../../packages/service-library/requirements/_base.in
 --requirement ../../../packages/settings-library/requirements/_base.in