Merge pull request #1 from DiamondLightSource/feature/lims-1669/parent-child

[LIMS-1669] Allow samples to have ancestors/descendants

Author: gfrn

Dockerfile

@@ -19,7 +19,7 @@ WORKDIR /project
 
 # make the wheel outside of the venv so 'build' does not dirty requirements.txt
 RUN pip install --upgrade pip build && \
-    #export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \
+    export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \
     python -m build && \
     touch requirements.txt
 

alembic/versions/297144dfe234_add_sample_parent_child_relationship.py

@@ -0,0 +1,41 @@
+"""Add sample parent/child relationship
+
+Revision ID: 297144dfe234
+Revises: 7325165750bc
+Create Date: 2025-03-13 11:02:07.757527
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '297144dfe234'
+down_revision: Union[str, None] = '7325165750bc'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('SampleParentChild',
+    sa.Column('parentId', sa.Integer(), nullable=False, comment='Sample(s) from which the child(ren) was derived from'),
+    sa.Column('childId', sa.Integer(), nullable=False, comment='Sample(s) derived from parent(s)'),
+    sa.Column('creationDate', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
+    sa.ForeignKeyConstraint(['childId'], ['Sample.sampleId'], ),
+    sa.ForeignKeyConstraint(['parentId'], ['Sample.sampleId'], ),
+    sa.PrimaryKeyConstraint('parentId', 'childId', name='parent_child_pk')
+    )
+    op.create_index(op.f('ix_SampleParentChild_childId'), 'SampleParentChild', ['childId'], unique=False)
+    op.create_index(op.f('ix_SampleParentChild_parentId'), 'SampleParentChild', ['parentId'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_SampleParentChild_parentId'), table_name='SampleParentChild')
+    op.drop_index(op.f('ix_SampleParentChild_childId'), table_name='SampleParentChild')
+    op.drop_table('SampleParentChild')
+    # ### end Alembic commands ###

database/data.sql

@@ -192,6 +192,33 @@ COMMENT ON COLUMN public."Sample"."externalId" IS 'Item ID in ISPyB';
 COMMENT ON COLUMN public."Sample"."subLocation" IS 'Additional location, such as cassette slot or multi-sample pin position';
 
 
+--
+-- Name: SampleParentChild; Type: TABLE; Schema: public; Owner: sample_handling
+--
+
+CREATE TABLE public."SampleParentChild" (
+    "parentId" integer NOT NULL,
+    "childId" integer NOT NULL,
+    "creationDate" timestamp with time zone DEFAULT now() NOT NULL
+);
+
+
+ALTER TABLE public."SampleParentChild" OWNER TO sample_handling;
+
+--
+-- Name: COLUMN "SampleParentChild"."parentId"; Type: COMMENT; Schema: public; Owner: sample_handling
+--
+
+COMMENT ON COLUMN public."SampleParentChild"."parentId" IS 'Sample(s) from which the child(ren) was derived from';
+
+
+--
+-- Name: COLUMN "SampleParentChild"."childId"; Type: COMMENT; Schema: public; Owner: sample_handling
+--
+
+COMMENT ON COLUMN public."SampleParentChild"."childId" IS 'Sample(s) derived from parent(s)';
+
+
 --
 -- Name: Sample_sampleId_seq; Type: SEQUENCE; Schema: public; Owner: sample_handling
 --
@@ -389,6 +416,8 @@ COPY public."Container" ("containerId", "shipmentId", "topLevelContainerId", "pa
 1307	204	\N	1336	gridBox	4	2	\N	f	\N	Grid_Box_01	\N	\N	f	f	\N	2025-01-10 08:54:42.073855+00
 1335	204	\N	1336	gridBox	4	3	\N	f	\N	Grid_Box_02	\N	\N	f	f	\N	2025-01-10 08:54:42.073855+00
 648	97	\N	646	gridBox	4	1	\N	f	\N	Grid_Box_02	\N	\N	f	f	\N	2025-01-10 08:54:42.073855+00
+1904	229	\N	1901	gridBox	4	1	\N	f	\N	Grid_Box_01	\N	\N	f	f	\N	2025-01-10 08:54:42.073855+00
+1901	229	720	\N	puck	4	\N	\N	f	\N	Puck_01	\N	\N	f	f	\N	2025-01-10 08:54:42.073855+00
 \.
 
 
@@ -415,6 +444,16 @@ COPY public."Sample" ("sampleId", "shipmentId", "proteinId", type, location, det
 612	117	338108	grid	\N	{"buffer": "", "concentration": "", "foil": "Quantifoil copper", "film": "Holey carbon", "mesh": "200", "hole": "R 0.6/1", "vitrification": "GP2", "vitrificationConditions": ""}	788	3P_1	\N	\N	2	2025-01-10 08:54:42.073855+00
 3	1	4407	sample	1	{"details": null, "shipmentId": 1, "foil": "Quantifoil copper", "film": "Holey carbon", "mesh": "200", "hole": "R 0.6/1", "vitrification": "GP2"}	4	Sample_02	6186947	\N	1	2025-01-10 08:54:42.073855+00
 561	117	338108	grid	1	{"buffer": "", "concentration": "", "foil": "Quantifoil copper", "film": "Holey carbon", "mesh": "200", "hole": "R 0.6/1", "vitrification": "GP2", "vitrificationConditions": ""}	776	3P_1	6212665	\N	1	2025-01-10 08:54:42.073855+00
+1877	229	338108	grid	1	{"buffer": "", "concentration": "", "foil": "Quantifoil copper", "film": "Holey carbon", "mesh": "200", "hole": "R 0.6/1", "vitrification": "GP2", "vitrificationConditions": ""}	1904	3P_1	\N	\N	1	2025-01-10 08:54:42.073855+00
+\.
+
+
+--
+-- Data for Name: SampleParentChild; Type: TABLE DATA; Schema: public; Owner: sample_handling
+--
+
+COPY public."SampleParentChild" ("parentId", "childId", "creationDate") FROM stdin;
+612	1877	2025-03-13 09:49:12.797986+00
 \.
 
 
@@ -449,6 +488,7 @@ COPY public."TopLevelContainer" ("topLevelContainerId", "shipmentId", details, c
 171	106	\N	DLS-4	dewar	Dewar_06	20	\N	f	1100af88-2e0b-46a7-93f9-2737a0b23d0c	2025-01-10 08:54:42.073855+00
 199	117	{}	DLS-BI-0020	dewar	DLS-BI-0020	72181		f	1100af88-2e0b-46a7-93f9-2737a0b23d0c	2025-01-10 08:54:42.073855+00
 221	\N	{}	DLS-BI-0020	dewar	DLS-BI-0020	\N		t	1100af88-2e0b-46a7-93f9-2737a0b23d0c	2025-01-10 08:54:42.073855+00
+720	229	{}	DLS-BI-0020	dewar	DLS-BI-0020	\N		f	1100af88-2e0b-46a7-93f9-2737a0b23d0c	2025-01-10 08:54:42.073855+00
 \.
 
 
@@ -457,43 +497,43 @@ COPY public."TopLevelContainer" ("topLevelContainerId", "shipmentId", details, c
 --
 
 COPY public.alembic_version (version_num) FROM stdin;
-7325165750bc
+297144dfe234
 \.
 
 
 --
 -- Name: Container_containerId_seq; Type: SEQUENCE SET; Schema: public; Owner: sample_handling
 --
 
-SELECT pg_catalog.setval('public."Container_containerId_seq"', 1898, true);
+SELECT pg_catalog.setval('public."Container_containerId_seq"', 2039, true);
 
 
 --
 -- Name: PreSession_preSessionId_seq; Type: SEQUENCE SET; Schema: public; Owner: sample_handling
 --
 
-SELECT pg_catalog.setval('public."PreSession_preSessionId_seq"', 349, true);
+SELECT pg_catalog.setval('public."PreSession_preSessionId_seq"', 379, true);
 
 
 --
 -- Name: Sample_sampleId_seq; Type: SEQUENCE SET; Schema: public; Owner: sample_handling
 --
 
-SELECT pg_catalog.setval('public."Sample_sampleId_seq"', 1876, true);
+SELECT pg_catalog.setval('public."Sample_sampleId_seq"', 2096, true);
 
 
 --
 -- Name: Shipment_shipmentId_seq; Type: SEQUENCE SET; Schema: public; Owner: sample_handling
 --
 
-SELECT pg_catalog.setval('public."Shipment_shipmentId_seq"', 271, true);
+SELECT pg_catalog.setval('public."Shipment_shipmentId_seq"', 286, true);
 
 
 --
 -- Name: TopLevelContainer_topLevelContainerId_seq; Type: SEQUENCE SET; Schema: public; Owner: sample_handling
 --
 
-SELECT pg_catalog.setval('public."TopLevelContainer_topLevelContainerId_seq"', 719, true);
+SELECT pg_catalog.setval('public."TopLevelContainer_topLevelContainerId_seq"', 825, true);
 
 
 --
@@ -616,6 +656,14 @@ ALTER TABLE ONLY public.alembic_version
     ADD CONSTRAINT alembic_version_pkc PRIMARY KEY (version_num);
 
 
+--
+-- Name: SampleParentChild parent_child_pk; Type: CONSTRAINT; Schema: public; Owner: sample_handling
+--
+
+ALTER TABLE ONLY public."SampleParentChild"
+    ADD CONSTRAINT parent_child_pk PRIMARY KEY ("parentId", "childId");
+
+
 --
 -- Name: ix_Container_containerId; Type: INDEX; Schema: public; Owner: sample_handling
 --
@@ -651,6 +699,20 @@ CREATE INDEX "ix_PreSession_preSessionId" ON public."PreSession" USING btree ("p
 CREATE UNIQUE INDEX "ix_PreSession_shipmentId" ON public."PreSession" USING btree ("shipmentId");
 
 
+--
+-- Name: ix_SampleParentChild_childId; Type: INDEX; Schema: public; Owner: sample_handling
+--
+
+CREATE INDEX "ix_SampleParentChild_childId" ON public."SampleParentChild" USING btree ("childId");
+
+
+--
+-- Name: ix_SampleParentChild_parentId; Type: INDEX; Schema: public; Owner: sample_handling
+--
+
+CREATE INDEX "ix_SampleParentChild_parentId" ON public."SampleParentChild" USING btree ("parentId");
+
+
 --
 -- Name: ix_Sample_containerId; Type: INDEX; Schema: public; Owner: sample_handling
 --
@@ -746,6 +808,22 @@ ALTER TABLE ONLY public."PreSession"
     ADD CONSTRAINT "PreSession_shipmentId_fkey" FOREIGN KEY ("shipmentId") REFERENCES public."Shipment"("shipmentId");
 
 
+--
+-- Name: SampleParentChild SampleParentChild_childId_fkey; Type: FK CONSTRAINT; Schema: public; Owner: sample_handling
+--
+
+ALTER TABLE ONLY public."SampleParentChild"
+    ADD CONSTRAINT "SampleParentChild_childId_fkey" FOREIGN KEY ("childId") REFERENCES public."Sample"("sampleId");
+
+
+--
+-- Name: SampleParentChild SampleParentChild_parentId_fkey; Type: FK CONSTRAINT; Schema: public; Owner: sample_handling
+--
+
+ALTER TABLE ONLY public."SampleParentChild"
+    ADD CONSTRAINT "SampleParentChild_parentId_fkey" FOREIGN KEY ("parentId") REFERENCES public."Sample"("sampleId");
+
+
 --
 -- Name: Sample Sample_containerId_fkey; Type: FK CONSTRAINT; Schema: public; Owner: sample_handling
 --

src/scaup/crud/containers.py

@@ -1,8 +1,9 @@
 from fastapi import HTTPException, status
 from lims_utils.models import ProposalReference
 from sqlalchemy import func, insert, select, update
+from sqlalchemy.orm import aliased
 
-from ..models.containers import ContainerIn, OptionalContainer
+from ..models.containers import ContainerIn, ContainerOut, OptionalContainer
 from ..models.inner_db.tables import Container, Sample, Shipment
 from ..utils.crud import assert_not_booked, edit_item
 from ..utils.database import inner_db, paginate
@@ -24,6 +25,39 @@ def create_container(params: ContainerIn, shipmentId: int | None = None):
     return container
 
 
+def get_container(container_id: int):
+    container = inner_db.session.scalar(select(Container).filter(Container.id == container_id))
+    validated_container = ContainerOut.model_validate(container)
+
+    # Internal containers are not in storage dewars/containers, but instead in transport containers.
+    # This means that their "parent" top level sample collection is a shipment, not a storage dewar.
+    if not validated_container.isInternal:
+        return validated_container
+
+    # Anchoring to parent member, to avoid one extra recursion layer
+    anchor_member = (
+        select(Container.parentId, Container.topLevelContainerId)
+        .filter(Container.id == validated_container.parentId)
+        .cte(name="anchor_member", recursive=True)
+    )
+
+    anchor_member_alias = anchor_member.alias("anchor_member_alias")
+    container_alias = aliased(Container)
+
+    anchor_member = anchor_member.union_all(
+        select(container_alias.parentId, container_alias.topLevelContainerId)
+        .filter(container_alias.id == anchor_member_alias.c.parentId)
+        .filter(anchor_member_alias.c.topLevelContainerId.is_(None))
+    )
+
+    tlc_id = inner_db.session.scalar(
+        select(anchor_member.c.topLevelContainerId).filter(anchor_member.c.topLevelContainerId.is_not(None))
+    )
+
+    validated_container.internalStorageContainer = tlc_id
+    return validated_container
+
+
 def get_containers(
     limit: int,
     page: int,

src/scaup/crud/samples.py

@@ -5,16 +5,19 @@
 from lims_utils.models import Paged, ProposalReference
 from sqlalchemy import and_, insert, select
 
-from ..models.inner_db.tables import Container, Sample, Shipment
+from ..models.inner_db.tables import Container, Sample, SampleParentChild, Shipment
 from ..models.samples import OptionalSample, SampleIn, SampleOut
+from ..utils.config import Config
 from ..utils.crud import assert_not_booked, edit_item
-from ..utils.database import inner_db, paginate, unravel
+from ..utils.database import inner_db, paginate
 from ..utils.external import ExternalRequest
 from ..utils.session import retry_if_exists
 
 
 def _get_protein(proteinId: int, token):
-    upstream_compound = ExternalRequest.request(token=token, url=f"/proteins/{proteinId}")
+    upstream_compound = ExternalRequest.request(
+        token=token, url=f"/proteins/{proteinId}"
+    )
 
     if upstream_compound.status_code != 200:
         app_logger.error(
@@ -79,6 +82,16 @@ def create_sample(shipmentId: int, params: SampleIn, token: str):
         ],
     ).all()
 
+    if params.parents:
+        inner_db.session.execute(
+            insert(SampleParentChild),
+            [
+                {"childId": child.id, "parentId": parent}
+                for child in samples
+                for parent in params.parents
+            ],
+        )
+
     inner_db.session.commit()
     return Paged(items=samples, total=params.copies, page=0, limit=params.copies)
 
@@ -103,13 +116,13 @@ def get_samples(
 ):
     query = (
         select(
-            *unravel(Sample),
-            Container.name.label("parent"),
+            Sample,
+            Container.name.label("containerName"),
             Shipment.name.label("parentShipmentName"),
         )
         .select_from(Shipment)
         .join(Sample, Sample.shipmentId == Shipment.id)
-        .join(Container, isouter=True)
+        .join(Container, Container.id == Sample.containerId, isouter=True)
     )
 
     if shipment_id:
@@ -133,34 +146,42 @@ def get_samples(
         query = query.filter(Container.isInternal.is_not(True))
 
     query = query.order_by(Container.name, Container.location, Sample.location)
-    samples = paginate(query, limit, page, slow_count=False)
+    samples = paginate(query, limit, page, slow_count=True, scalar=True)
 
     if ignore_external or token is None:
         return samples
 
-    ext_shipment_id = inner_db.session.scalar(select(Shipment.externalId).filter(Shipment.id == shipment_id))
+    ext_shipment_id = inner_db.session.scalar(
+        select(Shipment.externalId).filter(Shipment.id == shipment_id)
+    )
 
     if ext_shipment_id is None:
         return samples
 
-    ext_samples = ExternalRequest.request(token, method="GET", url=f"/shipments/{ext_shipment_id}/samples")
+    ext_samples = ExternalRequest.request(
+        Config.ispyb_api.jwt, method="GET", url=f"/shipments/{ext_shipment_id}/samples?limit=100"
+    )
 
     if ext_samples.status_code != 200:
-        app_logger.warning("Expeye returned %i: %s", ext_samples.status_code, ext_samples.text)
+        app_logger.warning(
+            "Expeye returned %i: %s", ext_samples.status_code, ext_samples.text
+        )
         return samples
 
+    validated_samples = Paged[SampleOut].model_validate(samples, from_attributes=True)
+
     for ext_sample in ext_samples.json()["items"]:
         if ext_sample["dataCollectionGroupId"]:
             try:
-                i, sample = next(
-                    (i, sample)
-                    for i, sample in enumerate(samples.items)
+                i = next(
+                    i
+                    for i, sample in enumerate(validated_samples.items)
                     if sample.externalId == ext_sample["blSampleId"]
                 )
-                new_sample = SampleOut.model_validate(sample, from_attributes=True)
-                new_sample.dataCollectionGroupId = ext_sample["dataCollectionGroupId"]
-                samples.items[i] = new_sample
+                validated_samples.items[i].dataCollectionGroupId = ext_sample[
+                    "dataCollectionGroupId"
+                ]
             except StopIteration:
                 pass
 
-    return samples
+    return validated_samples