Add tomogram search map endpoints

Author: gfrn

database/data.sql

@@ -2347,7 +2347,7 @@ CREATE TABLE `ContainerInspection` (
   CONSTRAINT `ContainerInspection_fk2` FOREIGN KEY (`inspectionTypeId`) REFERENCES `InspectionType` (`inspectionTypeId`) ON DELETE NO ACTION ON UPDATE NO ACTION,
   CONSTRAINT `ContainerInspection_fk3` FOREIGN KEY (`imagerId`) REFERENCES `Imager` (`imagerId`) ON DELETE NO ACTION ON UPDATE NO ACTION,
   CONSTRAINT `ContainerInspection_fk4` FOREIGN KEY (`scheduleComponentid`) REFERENCES `ScheduleComponent` (`scheduleComponentId`)
-) ENGINE=InnoDB AUTO_INCREMENT=5 DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci;
+) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci;
 /*!40101 SET character_set_client = @saved_cs_client */;
 
 --
@@ -2357,7 +2357,8 @@ CREATE TABLE `ContainerInspection` (
 LOCK TABLES `ContainerInspection` WRITE;
 /*!40000 ALTER TABLE `ContainerInspection` DISABLE KEYS */;
 INSERT INTO `ContainerInspection` VALUES
-(4,34874,1,NULL,NULL,'2018-08-07 15:20:00',NULL,'Completed',99,NULL,'2018-08-07 12:08:00','2018-08-07 15:36:00');
+(4,34874,1,NULL,NULL,'2018-08-07 15:20:00',NULL,'Completed',99,NULL,'2018-08-07 12:08:00','2018-08-07 15:36:00'),
+(5,34874,1,NULL,NULL,'2018-08-07 15:20:00',NULL,'Completed',99,NULL,'2018-08-07 12:08:00','2018-08-07 15:36:00');
 /*!40000 ALTER TABLE `ContainerInspection` ENABLE KEYS */;
 UNLOCK TABLES;
 
@@ -2950,7 +2951,7 @@ CREATE TABLE `DataCollection` (
   CONSTRAINT `DataCollection_ibfk_6` FOREIGN KEY (`startPositionId`) REFERENCES `MotorPosition` (`motorPositionId`),
   CONSTRAINT `DataCollection_ibfk_7` FOREIGN KEY (`endPositionId`) REFERENCES `MotorPosition` (`motorPositionId`),
   CONSTRAINT `DataCollection_ibfk_8` FOREIGN KEY (`blSubSampleId`) REFERENCES `BLSubSample` (`blSubSampleId`)
-) ENGINE=InnoDB AUTO_INCREMENT=6018040 DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci;
+) ENGINE=InnoDB AUTO_INCREMENT=6018043 DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci;
 /*!40101 SET character_set_client = @saved_cs_client */;
 
 --
@@ -3070,7 +3071,7 @@ CREATE TABLE `DataCollectionGroup` (
   CONSTRAINT `DataCollectionGroup_ibfk_1` FOREIGN KEY (`blSampleId`) REFERENCES `BLSample` (`blSampleId`) ON DELETE CASCADE ON UPDATE CASCADE,
   CONSTRAINT `DataCollectionGroup_ibfk_2` FOREIGN KEY (`sessionId`) REFERENCES `BLSession` (`sessionId`) ON DELETE CASCADE ON UPDATE CASCADE,
   CONSTRAINT `DataCollectionGroup_ibfk_4` FOREIGN KEY (`experimentTypeId`) REFERENCES `ExperimentType` (`experimentTypeId`)
-) ENGINE=InnoDB AUTO_INCREMENT=5441293 DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='a dataCollectionGroup is a group of dataCollection for a spe';
+) ENGINE=InnoDB AUTO_INCREMENT=5441296 DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='a dataCollectionGroup is a group of dataCollection for a spe';
 /*!40101 SET character_set_client = @saved_cs_client */;
 
 --
@@ -5429,7 +5430,7 @@ CREATE TABLE `ProcessingJob` (
   PRIMARY KEY (`processingJobId`),
   KEY `ProcessingJob_ibfk1` (`dataCollectionId`),
   CONSTRAINT `ProcessingJob_ibfk1` FOREIGN KEY (`dataCollectionId`) REFERENCES `DataCollection` (`dataCollectionId`)
-) ENGINE=InnoDB AUTO_INCREMENT=3990 DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='From this we get both job times and lag times';
+) ENGINE=InnoDB AUTO_INCREMENT=4017 DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='From this we get both job times and lag times';
 /*!40101 SET character_set_client = @saved_cs_client */;
 
 --
@@ -5524,7 +5525,7 @@ CREATE TABLE `ProcessingJobParameter` (
   KEY `ProcessingJobParameter_ibfk1` (`processingJobId`),
   KEY `ProcessingJobParameter_idx_paramKey_procJobId` (`parameterKey`,`processingJobId`),
   CONSTRAINT `ProcessingJobParameter_ibfk1` FOREIGN KEY (`processingJobId`) REFERENCES `ProcessingJob` (`processingJobId`)
-) ENGINE=InnoDB AUTO_INCREMENT=27355 DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci;
+) ENGINE=InnoDB AUTO_INCREMENT=27655 DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci;
 /*!40101 SET character_set_client = @saved_cs_client */;
 
 --
@@ -7825,8 +7826,8 @@ INSERT INTO `Tomogram` VALUES
 (3,6017409,56986678,'aligned_file_fri_aretomo.mrc',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'/dls','test.png','test.png','test.png','test.png','test.png','2023-01-19 14:15:44',NULL,NULL,NULL,NULL),
 (4,6017411,56986679,'aligned_file_fri_aretomo.mrc','/dls/m02/data/align_output/Position_1_9_stack.mrc',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'/dls','test.png','test.png','test.png','test.png','test.png','2023-01-19 14:15:44',NULL,NULL,NULL,NULL),
 (5,6017411,56986800,'aligned_file_fri_aretomo.mrc','/dls/m02/data/align_output/Position_1_9_stack.mrc',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'/dls','test.png','test.png','test.png','test.png','test.png','2023-01-19 14:15:44',NULL,NULL,NULL,NULL),
-(6,6017408,56986676,'aligned_file_fri_aretomo.mrc','/dls/m02/data/align_output/Position_1_9_stack.mrc',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'/dls','test.png','test.png','test.png','test.png','test.png','2023-01-19 14:15:44',NULL,NULL,NULL,NULL),
-(7,6017408,56986676,'aligned_file_fri_aretomo.mrc','/dls/m02/data/align_output/Position_1_9_stack.mrc',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'/dls','test.png','test.png','test.png','test.png','test.png','2023-01-19 14:15:44',NULL,NULL,NULL,NULL),
+(6,6017408,56986676,'aligned_file_fri_aretomo.mrc','/dls/m02/data/align_output/Position_1_9_stack.mrc',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'/dls','test.png','test.png','test.png','test.png','test.png','2023-01-19 14:15:44',NULL,1,NULL,NULL),
+(7,6017408,56986676,'aligned_file_fri_aretomo.mrc','/dls/m02/data/align_output/Position_1_9_stack.mrc',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'/dls','test.png','test.png','test.png','test.png','test.png','2023-01-19 14:15:44',NULL,1,NULL,NULL),
 (8,6017413,56986801,'aligned_file_fri_aretomo.mrc','/dls/m02/data/align_output/Position_1_9_stack.mrc',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'/dls','test.png','test.png','test.png','test.png','test.png','2023-01-19 14:15:44',1.2,NULL,NULL,NULL);
 /*!40000 ALTER TABLE `Tomogram` ENABLE KEYS */;
 UNLOCK TABLES;
@@ -8441,4 +8442,4 @@ UNLOCK TABLES;
 /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
 /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
 
--- Dump completed on 2025-07-09 10:50:48
+-- Dump completed on 2025-07-25 11:49:47

pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
     "pydantic[email]~=2.11.5",
     "fpdf2~=2.8.3",
     "types-requests",
-    "lims-utils~=0.4.6"
+    "lims-utils~=0.4.7"
 ]
 dynamic = ["version"]
 license.file = "LICENSE"

src/pato/crud/collections.py

@@ -43,7 +43,7 @@
     ProcessingJobResponse,
     TomogramFullResponse,
 )
-from ..utils.database import db
+from ..utils.database import db, unravel
 from ..utils.generic import MovieType, check_session_active, parse_count, validate_path
 from ..utils.pika import PikaPublisher
 
@@ -64,9 +64,7 @@
 
 def _generate_proc_job_params(proc_job_id: int | Column[int], params: dict):
     return [
-        ProcessingJobParameter(
-            processingJobId=proc_job_id, parameterKey=key, parameterValue=value
-        )
+        ProcessingJobParameter(processingJobId=proc_job_id, parameterKey=key, parameterValue=value)
         for (key, value) in params.items()
     ]
 
@@ -104,8 +102,7 @@ def get_data_collection_attachments(
 def get_data_collection_attachment(collection_id: int, attachment_id: int):
     attachment_path = db.session.scalar(
         select(DataCollectionFileAttachment.fileFullPath).filter(
-            DataCollectionFileAttachment.dataCollectionFileAttachmentId
-            == attachment_id,
+            DataCollectionFileAttachment.dataCollectionFileAttachmentId == attachment_id,
             DataCollectionFileAttachment.dataCollectionId == collection_id,
         )
     )
@@ -119,9 +116,7 @@ def get_data_collection_attachment(collection_id: int, attachment_id: int):
     return attachment_path
 
 
-def get_tomograms(
-    limit: int, page: int, collectionId: int
-) -> Paged[TomogramFullResponse]:
+def get_tomograms(limit: int, page: int, collectionId: int) -> Paged[TomogramFullResponse]:
     query = (
         select(
             Tomogram,
@@ -152,9 +147,7 @@ def get_motion_correction(limit: int, page: int, collectionId: int) -> Paged[Ful
     return db.paginate(query, limit, page, slow_count=True)
 
 
-def initiate_reprocessing_tomogram(
-    params: TomogramReprocessingParameters, collectionId: int
-):
+def initiate_reprocessing_tomogram(params: TomogramReprocessingParameters, collectionId: int):
     _validate_session_active(collectionId)
 
     motion_correction_records = db.session.execute(
@@ -170,9 +163,7 @@ def initiate_reprocessing_tomogram(
     for record in motion_correction_records:
         # Users can modify the file name, and we cannot control where exactly the tilt
         # angle ends up, so we have to manually extract them from each file
-        tilt_angle_regex = re.match(
-            r".*_([-]?[0-9]+\.[0-9]+)_.*", record.micrographFullPath
-        )
+        tilt_angle_regex = re.match(r".*_([-]?[0-9]+\.[0-9]+)_.*", record.micrographFullPath)
 
         if tilt_angle_regex is None:
             raise HTTPException(
@@ -313,9 +304,7 @@ def initiate_reprocessing_spa(params: SPAReprocessingParameters, collectionId: i
     db.session.add(new_job)
     db.session.flush()
 
-    db.session.bulk_save_objects(
-        _generate_proc_job_params(new_job.processingJobId, full_params)
-    )
+    db.session.bulk_save_objects(_generate_proc_job_params(new_job.processingJobId, full_params))
     db.session.commit()
 
     message = {"parameters": {"ispyb_process": new_job.processingJobId}}
@@ -356,8 +345,7 @@ def _with_ctf_joins(query: Select, collectionId: int):
         .join(CTF, CTF.motionCorrectionId == MotionCorrection.motionCorrectionId)
         .join(
             ParticlePicker,
-            ParticlePicker.firstMotionCorrectionId
-            == MotionCorrection.motionCorrectionId,
+            ParticlePicker.firstMotionCorrectionId == MotionCorrection.motionCorrectionId,
         )
     )
 
@@ -416,9 +404,7 @@ def get_particle_count_per_resolution(collectionId: int) -> ItemList[DataPoint]:
 @validate_path
 def get_central_slice(collection_id: int, movie_type: MovieType = None) -> str:
     tomogram_id = db.session.scalar(
-        select(Tomogram.tomogramId)
-        .filter(Tomogram.dataCollectionId == collection_id)
-        .limit(1)
+        select(Tomogram.tomogramId).filter(Tomogram.dataCollectionId == collection_id).limit(1)
     )
 
     if tomogram_id is None:
@@ -428,3 +414,29 @@ def get_central_slice(collection_id: int, movie_type: MovieType = None) -> str:
         )
 
     return get_slice_path(tomogram_id, movie_type)
+
+
+def get_data_collection(collection_id: int):
+    # Since we need the index to be able to paginate the tomogram page properly, we need to obtain the ordinality
+    # of the data collection in the set. This is quicker than finding the index in the front end, as 1/6th of data
+    # collection groups have over 100 data collections.
+    base_sub_query = (
+        select(
+            func.row_number().over(order_by=DataCollection.dataCollectionId).label("index"),
+            *unravel(DataCollection),
+        )
+        .select_from(DataCollectionGroup)
+        .join(DataCollection)
+        .filter(
+            DataCollectionGroup.dataCollectionGroupId
+            == select(DataCollection.dataCollectionGroupId)
+            .filter(DataCollection.dataCollectionId == collection_id)
+            .scalar_subquery()
+        )
+        .group_by(DataCollection.dataCollectionId)
+        .order_by(DataCollection.dataCollectionId)
+    ).subquery()
+
+    query = select(base_sub_query).filter(base_sub_query.c.dataCollectionId == collection_id)
+
+    return db.session.execute(query).one()

src/pato/crud/grid_squares.py

@@ -5,6 +5,7 @@
     MotionCorrection,
     Movie,
     ParticlePicker,
+    Tomogram,
 )
 from sqlalchemy import distinct, func, select
 from sqlalchemy.sql.functions import coalesce
@@ -23,7 +24,7 @@ def get_foil_holes(grid_square_id: int, page: int, limit: int):
             # Astigmatism can be negative if short/long axis get mixed up when calculating
             # astigmatism from defocus. It is also inserted in the database in angstroms,
             # when we want to return it in nm.
-            (func.abs(func.avg(CTF.astigmatism))/10).label("astigmatism"),
+            (func.abs(func.avg(CTF.astigmatism)) / 10).label("astigmatism"),
             func.avg(CTF.estimatedResolution).label("resolution"),
             func.avg(ParticlePicker.numberOfParticles).label("particleCount"),
             func.count(distinct(Movie.movieId)).label("movieCount"),
@@ -39,14 +40,13 @@ def get_foil_holes(grid_square_id: int, page: int, limit: int):
             CTF,
             CTF.motionCorrectionId == MotionCorrection.motionCorrectionId,
             isouter=True,
-        ).join(
+        )
+        .join(
             ParticlePicker,
-            ParticlePicker.firstMotionCorrectionId
-            == MotionCorrection.motionCorrectionId,
+            ParticlePicker.firstMotionCorrectionId == MotionCorrection.motionCorrectionId,
             isouter=True,
-        ).group_by(
-            FoilHole.foilHoleId
         )
+        .group_by(FoilHole.foilHoleId)
     )
 
     return db.paginate(query=query, limit=limit, page=page, slow_count=True)
@@ -55,7 +55,11 @@ def get_foil_holes(grid_square_id: int, page: int, limit: int):
 @validate_path
 def get_grid_square_image(grid_square_id: int):
     return db.session.scalar(
-        select(GridSquare.gridSquareImage).filter(
-            GridSquare.gridSquareId == grid_square_id
-        )
+        select(GridSquare.gridSquareImage).filter(GridSquare.gridSquareId == grid_square_id)
     )
+
+
+def get_tomograms(grid_square_id: int, limit: int, page: int):
+    query = select(Tomogram).filter(Tomogram.gridSquareId == grid_square_id)
+
+    return db.paginate(query, limit, page, slow_count=False, scalar=False)

src/pato/crud/tomograms.py

@@ -43,17 +43,13 @@ def _prepend_denoise(
     if len(split_file) != 3:
         raise HTTPException(status_code=500, detail="Unexpected filename")
 
-    denoised_prefix = (
-        ".denoised" if movie_type == "denoised" else f".denoised_{movie_type}"
-    )
+    denoised_prefix = ".denoised" if movie_type == "denoised" else f".denoised_{movie_type}"
 
     return split_file[0] + denoised_prefix + "".join(split_file[1:3])
 
 
 @validate_path
-def _get_movie(
-    tomogramId: int, movie_type: MovieType, image_type: Literal["thumbnail", "movie"]
-):
+def _get_movie(tomogramId: int, movie_type: MovieType, image_type: Literal["thumbnail", "movie"]):
     if movie_type:
         # If movie_Type is defined, this means that this is not the standard noisy tomogram we're looking for
         base_path = db.session.scalar(
@@ -71,9 +67,7 @@ def _get_movie(
         if movie_type == "picked":
             raise HTTPException(status_code=404, detail="No picked tomogram found")
 
-    column = (
-        Tomogram.tomogramMovie if image_type == "movie" else Tomogram.centralSliceImage
-    )
+    column = Tomogram.tomogramMovie if image_type == "movie" else Tomogram.centralSliceImage
     base_path = _get_generic_tomogram_file(tomogramId, column)
     return _prepend_denoise(base_path, image_type, movie_type)
 

src/pato/models/response.py

@@ -75,10 +75,9 @@ class SessionResponse(OrmBaseModel):
     collectionGroups: Optional[int] = None
     dataCollectionGroupId: Optional[int] = None
 
+
 class DataCollectionGroupSummaryResponse(OrmBaseModel):
-    dataCollectionGroupId: int = Field(
-        ..., lt=1e9, description="Data Collection Group ID"
-    )
+    dataCollectionGroupId: int = Field(..., lt=1e9, description="Data Collection Group ID")
     sessionId: int = Field(..., lt=1e9, description="Session ID")
     experimentType: Optional[str] = None
     atlasId: Optional[int] = None
@@ -147,23 +146,15 @@ class MotionCorrection(OrmBaseModel):
     motionCorrectionId: int
     dataCollectionId: Optional[int] = None
     autoProcProgramId: Optional[int] = None
-    imageNumber: Optional[int] = Field(
-        None, title="Movie number, sequential in time 1-n"
-    )
+    imageNumber: Optional[int] = Field(None, title="Movie number, sequential in time 1-n")
     firstFrame: Optional[int] = Field(None, title="First frame of movie used")
     lastFrame: Optional[int] = Field(None, title="Last frame of movie used")
     dosePerFrame: Optional[float] = Field(None, title="Dose per frame")
     doseWeight: Optional[float] = Field(None, title="Dose weight")
     totalMotion: Optional[float] = Field(None, title="Total motion")
-    averageMotionPerFrame: Optional[float] = Field(
-        None, title="Average motion per frame"
-    )
-    driftPlotFullPath: Optional[str] = Field(
-        None, max_length=255, title="Path to drift plot"
-    )
-    micrographFullPath: Optional[str] = Field(
-        None, max_length=255, title="Path to micrograph"
-    )
+    averageMotionPerFrame: Optional[float] = Field(None, title="Average motion per frame")
+    driftPlotFullPath: Optional[str] = Field(None, max_length=255, title="Path to drift plot")
+    micrographFullPath: Optional[str] = Field(None, max_length=255, title="Path to micrograph")
     micrographSnapshotFullPath: Optional[str] = Field(
         None, max_length=255, title="Path to micrograph"
     )
@@ -257,12 +248,15 @@ class ProcessingJobResponse(OrmBaseModel):
 
 
 class TomogramResponse(OrmBaseModel):
+    dataCollectionId: int
     tomogramId: int
     volumeFile: Optional[str] = Field(None, max_length=255)
     stackFile: Optional[str] = Field(None, max_length=255)
     sizeX: Optional[int] = None
     sizeY: Optional[int] = None
     sizeZ: Optional[int] = None
+    pixelLocationX: Optional[int] = None
+    pixelLocationY: Optional[int] = None
     pixelSpacing: Optional[float] = None
     residualErrorMean: Optional[float] = None
     residualErrorSD: Optional[float] = None

src/pato/routes/collections.py

@@ -6,7 +6,7 @@
 from ..crud import collection_report as report_crud
 from ..crud import collections as crud
 from ..crud import generic
-from ..models.collections import DataCollectionFileAttachmentOut
+from ..models.collections import BaseDataCollectionOut, DataCollectionFileAttachmentOut
 from ..models.reprocessing import (
     SPAReprocessingParameters,
     TomogramReprocessingParameters,
@@ -28,6 +28,12 @@
     prefix="/dataCollections",
 )
 
+@router.get("/{collectionId}", response_model=BaseDataCollectionOut)
+def get_data_collection(
+    collectionId: int = Depends(auth)
+):
+    """Get data collection"""
+    return crud.get_data_collection(collection_id=collectionId)
 
 @router.get("/{collectionId}/tomograms", response_model=Paged[TomogramFullResponse])
 def get_tomograms(

src/pato/routes/grid_squares.py

@@ -4,7 +4,7 @@
 
 from ..auth import Permissions
 from ..crud import grid_squares as crud
-from ..models.response import FoilHole
+from ..models.response import FoilHole, TomogramResponse
 
 router = APIRouter(
     tags=["Grid Squares"],
@@ -21,6 +21,14 @@ def get_foil_holes(
     return crud.get_foil_holes(grid_square_id=gridSquareId, **page)
 
 
+@router.get("/{gridSquareId}/tomograms", response_model=Paged[TomogramResponse])
+def get_tomograms(
+    gridSquareId: int = Depends(Permissions.grid_square),
+    page: dict[str, int] = Depends(pagination),
+):
+    """Get tomograms in a search map"""
+    return crud.get_tomograms(grid_square_id=gridSquareId, **page)
+
 @router.get("/{gridSquareId}/image", response_class=FileResponse)
 def get_grid_square_image(gridSquareId: int = Depends(Permissions.grid_square)):
     """Get image of grid square"""

tests/collections/test_get.py

@@ -0,0 +1,5 @@
+def test_get(mock_permissions, client):
+    """Get data collection"""
+    resp = client.get("/dataCollections/6017412")
+    assert resp.status_code == 200
+    assert resp.json()["index"] == 2