Merge branch 'master' into introduce-vip-models-pricing

Author: matusdrobuliak66

.coveragerc

@@ -7,25 +7,20 @@ parallel = True
 
 [report]
 # Regexes for lines to exclude from consideration
-exclude_lines =
-    # Have to re-enable the standard pragma
-    pragma: no cover
-
+exclude_also =
     # Don't complain about missing debug-only code:
     def __repr__
     if self\.debug
-
     # Don't complain if tests don't hit defensive assertion code:
     raise AssertionError
     raise NotImplementedError
-
     # Don't complain if non-runnable code isn't run:
     if 0:
     if __name__ == .__main__.:
     if __name__ == __main__.:
+    class .*\bProtocol\):
     # Don't complain about abstract methods, they aren't run:
     @(abc\.)?abstract(((class|static)?method)|property)
-
     # Don't complain about type checking
     if TYPE_CHECKING:
 

packages/models-library/src/models_library/projects_nodes_io.py

@@ -34,7 +34,7 @@
 
 UUIDStr: TypeAlias = Annotated[str, StringConstraints(pattern=UUID_RE)]
 
-NodeIDStr = UUIDStr
+NodeIDStr: TypeAlias = UUIDStr
 
 LocationID = int
 LocationName = str

packages/postgres-database/src/simcore_postgres_database/migration/versions/7ad64e963e0f_add_timezone_comp_tasks.py

@@ -0,0 +1,68 @@
+"""add_timezone_comp_tasks
+
+Revision ID: 7ad64e963e0f
+Revises: b7f23f6d8aa2
+Create Date: 2024-11-27 22:28:51.898433+00:00
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "7ad64e963e0f"
+down_revision = "b7f23f6d8aa2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "comp_tasks",
+        "submit",
+        existing_type=postgresql.TIMESTAMP(),
+        type_=sa.DateTime(timezone=True),
+        existing_nullable=True,
+    )
+    op.alter_column(
+        "comp_tasks",
+        "start",
+        existing_type=postgresql.TIMESTAMP(),
+        type_=sa.DateTime(timezone=True),
+        existing_nullable=True,
+    )
+    op.alter_column(
+        "comp_tasks",
+        "end",
+        existing_type=postgresql.TIMESTAMP(),
+        type_=sa.DateTime(timezone=True),
+        existing_nullable=True,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "comp_tasks",
+        "end",
+        existing_type=sa.DateTime(timezone=True),
+        type_=postgresql.TIMESTAMP(),
+        existing_nullable=True,
+    )
+    op.alter_column(
+        "comp_tasks",
+        "start",
+        existing_type=sa.DateTime(timezone=True),
+        type_=postgresql.TIMESTAMP(),
+        existing_nullable=True,
+    )
+    op.alter_column(
+        "comp_tasks",
+        "submit",
+        existing_type=sa.DateTime(timezone=True),
+        type_=postgresql.TIMESTAMP(),
+        existing_nullable=True,
+    )
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/migration/versions/b7f23f6d8aa2_added_distributed_comp_scheduler.py

@@ -0,0 +1,33 @@
+"""added_distributed_comp_scheduler
+
+Revision ID: b7f23f6d8aa2
+Revises: c9db8bf5091e
+Create Date: 2024-11-26 17:06:27.053774+00:00
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "b7f23f6d8aa2"
+down_revision = "c9db8bf5091e"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "comp_runs", sa.Column("scheduled", sa.DateTime(timezone=True), nullable=True)
+    )
+    op.add_column(
+        "comp_runs", sa.Column("processed", sa.DateTime(timezone=True), nullable=True)
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("comp_runs", "processed")
+    op.drop_column("comp_runs", "scheduled")
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/migration/versions/e05bdc5b3c7b_add_timezone_comp_runs.py

@@ -0,0 +1,87 @@
+"""add_timezone_comp_runs
+
+Revision ID: e05bdc5b3c7b
+Revises: 7ad64e963e0f
+Create Date: 2024-11-27 22:51:21.112336+00:00
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "e05bdc5b3c7b"
+down_revision = "7ad64e963e0f"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "comp_runs",
+        "created",
+        existing_type=postgresql.TIMESTAMP(),
+        type_=sa.DateTime(timezone=True),
+        existing_nullable=False,
+        existing_server_default="now()",
+    )
+    op.alter_column(
+        "comp_runs",
+        "modified",
+        existing_type=postgresql.TIMESTAMP(),
+        type_=sa.DateTime(timezone=True),
+        existing_nullable=False,
+        existing_server_default="now()",
+    )
+    op.alter_column(
+        "comp_runs",
+        "started",
+        existing_type=postgresql.TIMESTAMP(),
+        type_=sa.DateTime(timezone=True),
+        existing_nullable=True,
+    )
+    op.alter_column(
+        "comp_runs",
+        "ended",
+        existing_type=postgresql.TIMESTAMP(),
+        type_=sa.DateTime(timezone=True),
+        existing_nullable=True,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "comp_runs",
+        "ended",
+        existing_type=sa.DateTime(timezone=True),
+        type_=postgresql.TIMESTAMP(),
+        existing_nullable=True,
+    )
+    op.alter_column(
+        "comp_runs",
+        "started",
+        existing_type=sa.DateTime(timezone=True),
+        type_=postgresql.TIMESTAMP(),
+        existing_nullable=True,
+    )
+    op.alter_column(
+        "comp_runs",
+        "modified",
+        existing_type=sa.DateTime(timezone=True),
+        type_=postgresql.TIMESTAMP(),
+        existing_nullable=False,
+        existing_server_default="now()",
+    )
+    op.alter_column(
+        "comp_runs",
+        "created",
+        existing_type=sa.DateTime(timezone=True),
+        type_=postgresql.TIMESTAMP(),
+        existing_nullable=False,
+        existing_server_default="now()",
+    )
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/models/comp_runs.py

@@ -1,11 +1,11 @@
 """ Computational Runs Table
 
 """
+
 import sqlalchemy as sa
 from sqlalchemy.dialects.postgresql import JSONB
-from sqlalchemy.sql import func
 
-from ._common import RefActions
+from ._common import RefActions, column_created_datetime, column_modified_datetime
 from .base import metadata
 from .comp_pipeline import StateType
 
@@ -72,31 +72,18 @@
         doc="The result of the run entry",
     ),
     # dag node id and class
-    sa.Column(
-        "created",
-        sa.DateTime(),
-        nullable=False,
-        server_default=func.now(),
-        doc="When the run entry was created",
-    ),
-    sa.Column(
-        "modified",
-        sa.DateTime(),
-        nullable=False,
-        server_default=func.now(),
-        onupdate=func.now(),  # this will auto-update on modification
-        doc="When the run entry was last modified",
-    ),
+    column_created_datetime(timezone=True),
+    column_modified_datetime(timezone=True),
     # utc timestamps for submission/start/end
     sa.Column(
         "started",
-        sa.DateTime,
+        sa.DateTime(timezone=True),
         nullable=True,
         doc="When the run was started",
     ),
     sa.Column(
         "ended",
-        sa.DateTime,
+        sa.DateTime(timezone=True),
         nullable=True,
         doc="When the run was finished",
     ),
@@ -106,6 +93,18 @@
         nullable=True,
         doc="If filled, when cancellation was requested",
     ),
+    sa.Column(
+        "scheduled",
+        sa.DateTime(timezone=True),
+        nullable=True,
+        doc="last time the pipeline was scheduled to be processed",
+    ),
+    sa.Column(
+        "processed",
+        sa.DateTime(timezone=True),
+        nullable=True,
+        doc="last time the pipeline was actually processed",
+    ),
     sa.Column("metadata", JSONB, nullable=True, doc="the run optional metadata"),
     sa.Column(
         "use_on_demand_clusters",

packages/postgres-database/src/simcore_postgres_database/models/comp_tasks.py

@@ -1,6 +1,7 @@
 """ Computational Tasks Table
 
 """
+
 import enum
 
 import sqlalchemy as sa
@@ -77,9 +78,15 @@ class NodeClass(enum.Enum):
         doc="current progress of the task if available",
     ),
     # utc timestamps for submission/start/end
-    sa.Column("submit", sa.DateTime, doc="UTC timestamp for task submission"),
-    sa.Column("start", sa.DateTime, doc="UTC timestamp when task started"),
-    sa.Column("end", sa.DateTime, doc="UTC timestamp for task completion"),
+    sa.Column(
+        "submit", sa.DateTime(timezone=True), doc="UTC timestamp for task submission"
+    ),
+    sa.Column(
+        "start", sa.DateTime(timezone=True), doc="UTC timestamp when task started"
+    ),
+    sa.Column(
+        "end", sa.DateTime(timezone=True), doc="UTC timestamp for task completion"
+    ),
     sa.Column(
         "last_heartbeat",
         sa.DateTime(timezone=True),

packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py

@@ -93,7 +93,7 @@ async def get(connection: SAConnection, project_uuid: uuid.UUID) -> ProjectMetad
     row: RowProxy | None = await result.first()
     if row is None:
         raise DBProjectNotFoundError(project_uuid=project_uuid)
-    return ProjectMetadata.from_orm(row)
+    return ProjectMetadata.model_validate(row)
 
 
 def _check_valid_ancestors_combination(
@@ -202,7 +202,7 @@ async def set_project_ancestors(
         result: ResultProxy = await connection.execute(upsert_stmt)
         row: RowProxy | None = await result.first()
         assert row  # nosec
-        return ProjectMetadata.from_orm(row)
+        return ProjectMetadata.model_validate(row)
 
     except ForeignKeyViolation as err:
         assert err.pgerror is not None  # nosec  # noqa: PT017
@@ -234,7 +234,7 @@ async def set_project_custom_metadata(
         result: ResultProxy = await connection.execute(upsert_stmt)
         row: RowProxy | None = await result.first()
         assert row  # nosec
-        return ProjectMetadata.from_orm(row)
+        return ProjectMetadata.model_validate(row)
 
     except ForeignKeyViolation as err:
         raise DBProjectNotFoundError(project_uuid=project_uuid) from err

packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py

@@ -102,7 +102,7 @@ async def add(
             assert result  # nosec
             rows = await result.fetchall()
             assert rows is not None  # nosec
-            return [ProjectNode.from_orm(r) for r in rows]
+            return [ProjectNode.model_validate(r) for r in rows]
         except ForeignKeyViolation as exc:
             # this happens when the project does not exist, as we first check the node exists
             raise ProjectNodesProjectNotFoundError(
@@ -128,7 +128,7 @@ async def list(self, connection: SAConnection) -> list[ProjectNode]:
         assert result  # nosec
         rows = await result.fetchall()
         assert rows is not None  # nosec
-        return [ProjectNode.from_orm(row) for row in rows]
+        return [ProjectNode.model_validate(row) for row in rows]
 
     async def get(self, connection: SAConnection, *, node_id: uuid.UUID) -> ProjectNode:
         """get a node in the current project
@@ -154,7 +154,7 @@ async def get(self, connection: SAConnection, *, node_id: uuid.UUID) -> ProjectN
                 project_uuid=self.project_uuid, node_id=node_id
             )
         assert row  # nosec
-        return ProjectNode.from_orm(row)
+        return ProjectNode.model_validate(row)
 
     async def update(
         self, connection: SAConnection, *, node_id: uuid.UUID, **values
@@ -184,7 +184,7 @@ async def update(
                 project_uuid=self.project_uuid, node_id=node_id
             )
         assert row  # nosec
-        return ProjectNode.from_orm(row)
+        return ProjectNode.model_validate(row)
 
     async def delete(self, connection: SAConnection, *, node_id: uuid.UUID) -> None:
         """delete a node in the current project

packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py

@@ -93,7 +93,7 @@ async def creator(
                 .returning(sa.literal_column("*"))
             )
 
-            inserted_project = ProjectAtDB.from_orm(await result.first())
+            inserted_project = ProjectAtDB.model_validate(await result.first())
             project_nodes_repo = ProjectNodesRepo(project_uuid=project_uuid)
             # NOTE: currently no resources is passed until it becomes necessary
             default_node_config = {"required_resources": {}}