Automatically update shipment status, return dewar history

Author: gfrn

.github/workflows/_integration.yml

@@ -7,11 +7,30 @@ on:
         required: true
 
 jobs:
+  get-db-image:
+      runs-on: ubuntu-latest
+      env:
+        BRANCH_NAME: ${{ github.ref_type == 'tag' && 'master' || (github.head_ref || github.ref_name) }}
+      outputs:
+        branch: ${{ steps.clean.outputs.branch }}
+      steps:
+        - name: Checkout repository
+          uses: actions/checkout@v4
+        - id: clean
+          name: "Clean branch name"
+          continue-on-error: true
+          run: |
+            git fetch
+            if git diff --name-only --exit-code master HEAD database/; then
+              echo "branch=${BRANCH_NAME//\//-}" >> "$GITHUB_OUTPUT"
+            else
+              echo "branch=master" >> "$GITHUB_OUTPUT"
+            fi
   test:
     runs-on: ubuntu-latest
     services:
       integration-db:
-        image: ghcr.io/diamondlightsource/scaup-backend-db:master
+        image: ghcr.io/diamondlightsource/scaup-backend-db:${{ needs.get-db-image.outputs.branch }}
         ports:
           - 5432:5432
     strategy:

.pre-commit-config.yaml

@@ -10,15 +10,15 @@ repos:
     hooks:
       - id: format
         name: Format with ruff
-        stages: [commit]
+        stages: [pre-commit]
         language: system
         entry: ruff format --check --diff
         exclude: alembic
         types: [python]
 
       - id: ruff
         name: Run ruff
-        stages: [commit]
+        stages: [pre-commit]
         language: system
         entry: ruff check
         exclude: alembic

alembic/versions/dea9c772d734_add_laststatusupdate_column.py

@@ -0,0 +1,30 @@
+"""Add lastStatusUpdate column
+
+Revision ID: dea9c772d734
+Revises: 297144dfe234
+Create Date: 2025-06-04 09:21:34.341747
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'dea9c772d734'
+down_revision: Union[str, None] = '297144dfe234'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('Shipment', sa.Column('lastStatusUpdate', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('Shipment', 'lastStatusUpdate')
+    # ### end Alembic commands ###

database/data.sql

@@ -2,8 +2,8 @@
 -- PostgreSQL database dump
 --
 
--- Dumped from database version 17.2
--- Dumped by pg_dump version 17.2
+-- Dumped from database version 17.4
+-- Dumped by pg_dump version 17.4
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -255,7 +255,8 @@ CREATE TABLE public."Shipment" (
     comments character varying(255),
     "proposalCode" character varying(2) NOT NULL,
     "proposalNumber" integer NOT NULL,
-    "visitNumber" integer NOT NULL
+    "visitNumber" integer NOT NULL,
+    "lastStatusUpdate" timestamp with time zone DEFAULT now() NOT NULL
 );
 
 
@@ -461,17 +462,17 @@ COPY public."SampleParentChild" ("parentId", "childId", "creationDate") FROM std
 -- Data for Name: Shipment; Type: TABLE DATA; Schema: public; Owner: sample_handling
 --
 
-COPY public."Shipment" ("shipmentId", "creationDate", "shipmentRequest", status, name, "externalId", comments, "proposalCode", "proposalNumber", "visitNumber") FROM stdin;
-1	2024-05-02 13:12:36.528788+00	\N	\N	Shipment_01	\N	\N	cm	1	1
-2	2024-05-02 13:12:36.528788+00	\N	\N	Shipment_02	123	\N	cm	2	1
-89	2024-05-02 13:12:36.528788+00	\N	Booked	Shipment_03	256	\N	cm	2	1
-97	2024-06-26 12:55:39.211687+00	\N	\N	Shipment_04	\N	\N	cm	3	1
-106	2024-06-26 12:55:39.211687+00	\N	\N	Shipment_05	789	\N	cm	3	1
-117	2024-06-26 13:36:53.632782+00	1	Booked	1	63975	\N	bi	23047	100
-118	2024-06-26 13:40:32.191664+00	\N	\N	2	\N	\N	bi	23047	100
-126	2024-07-15 15:35:32.472987+00	\N	\N	1	\N	\N	bi	23047	99
-204	2024-07-15 15:35:32.472987+00	\N	Created	3	\N	\N	bi	23047	99
-229	2025-01-10 08:54:23.171217+00	\N	Created	100	\N	\N	bi	23047	102
+COPY public."Shipment" ("shipmentId", "creationDate", "shipmentRequest", status, name, "externalId", comments, "proposalCode", "proposalNumber", "visitNumber", "lastStatusUpdate") FROM stdin;
+1	2024-05-02 13:12:36.528788+00	\N	\N	Shipment_01	\N	\N	cm	1	1	2025-06-09 08:29:07.995804+00
+2	2024-05-02 13:12:36.528788+00	\N	\N	Shipment_02	123	\N	cm	2	1	2025-06-09 08:29:07.995804+00
+89	2024-05-02 13:12:36.528788+00	\N	Booked	Shipment_03	256	\N	cm	2	1	2025-06-09 08:29:07.995804+00
+97	2024-06-26 12:55:39.211687+00	\N	\N	Shipment_04	\N	\N	cm	3	1	2025-06-09 08:29:07.995804+00
+106	2024-06-26 12:55:39.211687+00	\N	\N	Shipment_05	789	\N	cm	3	1	2025-06-09 08:29:07.995804+00
+118	2024-06-26 13:40:32.191664+00	\N	\N	2	\N	\N	bi	23047	100	2025-06-09 08:29:07.995804+00
+126	2024-07-15 15:35:32.472987+00	\N	\N	1	\N	\N	bi	23047	99	2025-06-09 08:29:07.995804+00
+204	2024-07-15 15:35:32.472987+00	\N	Created	3	\N	\N	bi	23047	99	2025-06-09 08:29:07.995804+00
+229	2025-01-10 08:54:23.171217+00	\N	Created	100	\N	\N	bi	23047	102	2025-06-09 08:29:07.995804+00
+117	2025-06-05 14:15:42.285+00	1	at facility	1	63975	\N	bi	23047	100	2025-06-05 14:15:42.285+00
 \.
 
 
@@ -497,43 +498,43 @@ COPY public."TopLevelContainer" ("topLevelContainerId", "shipmentId", details, c
 --
 
 COPY public.alembic_version (version_num) FROM stdin;
-297144dfe234
+dea9c772d734
 \.
 
 
 --
 -- Name: Container_containerId_seq; Type: SEQUENCE SET; Schema: public; Owner: sample_handling
 --
 
-SELECT pg_catalog.setval('public."Container_containerId_seq"', 2039, true);
+SELECT pg_catalog.setval('public."Container_containerId_seq"', 2238, true);
 
 
 --
 -- Name: PreSession_preSessionId_seq; Type: SEQUENCE SET; Schema: public; Owner: sample_handling
 --
 
-SELECT pg_catalog.setval('public."PreSession_preSessionId_seq"', 379, true);
+SELECT pg_catalog.setval('public."PreSession_preSessionId_seq"', 423, true);
 
 
 --
 -- Name: Sample_sampleId_seq; Type: SEQUENCE SET; Schema: public; Owner: sample_handling
 --
 
-SELECT pg_catalog.setval('public."Sample_sampleId_seq"', 2096, true);
+SELECT pg_catalog.setval('public."Sample_sampleId_seq"', 2558, true);
 
 
 --
 -- Name: Shipment_shipmentId_seq; Type: SEQUENCE SET; Schema: public; Owner: sample_handling
 --
 
-SELECT pg_catalog.setval('public."Shipment_shipmentId_seq"', 286, true);
+SELECT pg_catalog.setval('public."Shipment_shipmentId_seq"', 307, true);
 
 
 --
 -- Name: TopLevelContainer_topLevelContainerId_seq; Type: SEQUENCE SET; Schema: public; Owner: sample_handling
 --
 
-SELECT pg_catalog.setval('public."TopLevelContainer_topLevelContainerId_seq"', 825, true);
+SELECT pg_catalog.setval('public."TopLevelContainer_topLevelContainerId_seq"', 980, true);
 
 
 --

pyproject.toml

@@ -41,6 +41,7 @@ dev = [
     "pytest",
     "pytest-cov",
     "ruff",
+    "freezegun~=1.5.2",
     "sphinx-autobuild",
     "sphinx-copybutton",
     "sphinx-design",

src/scaup/crud/proposals.py

@@ -1,15 +1,16 @@
 from typing import List
 
 from fastapi import HTTPException, status
-from lims_utils.models import ProposalReference
-from sqlalchemy import case, insert, select
+from lims_utils.models import Paged, ProposalReference
+from sqlalchemy import insert, select
 from sqlalchemy.exc import MultipleResultsFound
 
 from ..models.inner_db.tables import Sample, Shipment
 from ..models.samples import SublocationAssignment
 from ..models.shipments import ShipmentIn
 from ..utils.crud import assign_dcg_to_sublocation
-from ..utils.database import inner_db, paginate, unravel
+from ..utils.database import inner_db
+from ..utils.external import update_shipment_statuses
 
 
 def create_shipment(proposal_reference: ProposalReference, params: ShipmentIn):
@@ -30,17 +31,17 @@ def create_shipment(proposal_reference: ProposalReference, params: ShipmentIn):
     return new_shipment
 
 
-def get_shipments(proposal_reference: ProposalReference, limit: int, page: int):
-    query = select(
-        *unravel(Shipment),
-        case((Shipment.externalId.is_not(None), "submitted"), else_="draft").label("creationStatus"),
-    ).filter(
+def get_shipments(token: str, proposal_reference: ProposalReference, limit: int, page: int):
+    query = select(Shipment).filter(
         Shipment.proposalCode == proposal_reference.code,
         Shipment.proposalNumber == proposal_reference.number,
         Shipment.visitNumber == proposal_reference.visit_number,
     )
 
-    return paginate(query, limit, page, slow_count=False)
+    shipments: Paged[Shipment] = inner_db.paginate(query, limit, page, slow_count=False, scalar=False)
+    shipments.items = update_shipment_statuses(shipments.items, token)
+
+    return shipments
 
 
 def assign_dcg_to_sublocation_in_session(session: ProposalReference, parameters: List[SublocationAssignment]):

src/scaup/crud/shipments.py

@@ -24,7 +24,7 @@
 from ..utils.config import Config
 from ..utils.crud import assert_no_unassigned, assign_dcg_to_sublocation
 from ..utils.database import inner_db
-from ..utils.external import TYPE_TO_SHIPPING_SERVICE_TYPE, Expeye, ExternalRequest
+from ..utils.external import TYPE_TO_SHIPPING_SERVICE_TYPE, Expeye, ExternalRequest, update_shipment_statuses
 from ..utils.query import query_result_to_object
 
 
@@ -44,7 +44,19 @@ def _get_shipment_tree(shipmentId: int):
     return raw_shipment_data
 
 
-def get_shipment(shipmentId: int):
+def get_shipment(token: str, shipmentId: int, get_children: bool = False):
+    if not get_children:
+        shipment = inner_db.session.execute(select(Shipment).filter(Shipment.id == shipmentId)).scalar_one()
+
+        shipment = update_shipment_statuses(shipments=[shipment], token=token)[0]
+
+        return ShipmentChildren(
+            id=shipmentId,
+            name=shipment.name,
+            children=[],
+            data=ShipmentOut.model_validate(shipment, from_attributes=True).model_dump(mode="json"),
+        )
+
     raw_shipment_data = _get_shipment_tree(shipmentId)
 
     return ShipmentChildren(

src/scaup/crud/top_level_containers.py

@@ -1,12 +1,18 @@
 from fastapi import HTTPException, status
 from lims_utils.logging import app_logger
+from lims_utils.models import Paged
 from sqlalchemy import func, insert, select
 
 from ..models.inner_db.tables import Shipment, TopLevelContainer
-from ..models.top_level_containers import OptionalTopLevelContainer, TopLevelContainerIn
+from ..models.top_level_containers import (
+    OptionalTopLevelContainer,
+    TopLevelContainerHistory,
+    TopLevelContainerIn,
+    TopLevelContainerOut,
+)
 from ..utils.config import Config
 from ..utils.crud import assert_not_booked, edit_item
-from ..utils.database import inner_db, paginate
+from ..utils.database import inner_db
 from ..utils.external import ExternalRequest
 from ..utils.session import retry_if_exists
 
@@ -119,7 +125,31 @@ def edit_top_level_container(topLevelContainerId: int, params: OptionalTopLevelC
     return edit_item(TopLevelContainer, params, topLevelContainerId, token)
 
 
-def get_top_level_containers(shipmentId: int, limit: int, page: int):
+def get_top_level_containers(shipmentId: int, token: str, limit: int, page: int):
     query = select(TopLevelContainer).filter(TopLevelContainer.shipmentId == shipmentId).join(Shipment)
 
-    return paginate(query, limit, page, slow_count=False, scalar=False)
+    top_level_containers: Paged[TopLevelContainer | TopLevelContainerOut] = inner_db.paginate(
+        query, limit, page, slow_count=False, scalar=False
+    )
+
+    for i, tlc in enumerate(top_level_containers.items):
+        if tlc.externalId is not None:
+            response = ExternalRequest.request(token=token, url=f"/dewars/{tlc.externalId}/history")
+
+            if response.status_code == 200:
+                # More than 25 items could be returned, but it is statistically unlikely
+                # (less than 2.2% of dewars have 25 history items or more, and most of these
+                # are commissioning proposals), so we'll disregard that for now
+                new_tlc = TopLevelContainerOut.model_validate(tlc, from_attributes=True)
+                new_tlc.history = [TopLevelContainerHistory.model_validate(item) for item in response.json()["items"]]
+
+                top_level_containers.items[i] = new_tlc
+            else:
+                app_logger.warning(
+                    "Failed to get history from ISPyB for dewar %i (external ID: %i): %s",
+                    tlc.id,
+                    tlc.externalId,
+                    response.text,
+                )
+
+    return top_level_containers

src/scaup/models/inner_db/tables.py

@@ -42,6 +42,7 @@ class Shipment(Base, BaseColumns):
 
     shipmentRequest: Mapped[int | None] = mapped_column()
     status: Mapped[str | None] = mapped_column(String(25), server_default="Created")
+    lastStatusUpdate: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
 
 
 class TopLevelContainer(Base, BaseColumns):

src/scaup/models/shipments.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from typing import Any, Literal, Optional
+from typing import Any, Optional
 
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
@@ -34,13 +34,8 @@ class ShipmentOut(BaseModel):
     creationDate: Optional[datetime]
     status: Optional[str] = None
     shipmentRequest: Optional[int] = None
-
-
-class MixedShipment(ShipmentOut):
-    id: int = Field(validation_alias="shipmentId")
-    creationStatus: Literal["draft", "submitted"] = "draft"
-
-    model_config = ConfigDict(arbitrary_types_allowed=True)
+    lastStatusUpdate: datetime
+    externalId: int | None = None
 
 
 class GenericItemData(BaseModel):