wip

Author: vredchenko

docs/how-to/use-core-http-api-client.md

@@ -68,7 +68,7 @@ acquisitions = await client.aget_acquisitions()
 ### Creating an Acquisition
 
 ```python
-from src.epu_data_intake.data_model import EpuSessionData
+from src.epu_data_intake.model.schemas import EpuSessionData
 from datetime import datetime
 
 # Create an acquisition from EPU session data
@@ -139,22 +139,26 @@ Here's an example of creating entities at each level:
 acquisition = client.create_acquisition(EpuSessionData(id="acq-1", name="Test Acquisition"))
 
 # Create a grid for the acquisition
-from src.epu_data_intake.data_model import Grid
-grid = Grid(data_dir="/path/to/grid")
+from src.epu_data_intake.model.schemas import GridData
+
+grid = GridData(data_dir="/path/to/grid")
 grid_response = client.create_acquisition_grid(acquisition.id, grid)
 
 # Create a grid square for the grid
-from src.epu_data_intake.data_model import GridSquareData
+from src.epu_data_intake.model.schemas import GridSquareData
+
 gridsquare = GridSquareData(id="gs-1", data_dir="/path/to/gridsquare")
 gridsquare_response = client.create_grid_gridsquare(grid_response.id, gridsquare)
 
 # Create a foil hole for the grid square
-from src.epu_data_intake.data_model import FoilHoleData
+from src.epu_data_intake.model.schemas import FoilHoleData
+
 foilhole = FoilHoleData(id="fh-1", gridsquare_id=gridsquare.id)
 foilhole_response = client.create_gridsquare_foilhole(gridsquare_response.id, foilhole)
 
 # Create a micrograph for the foil hole
-from src.epu_data_intake.data_model import MicrographData, MicrographManifest
+from src.epu_data_intake.model.schemas import MicrographData, MicrographManifest
+
 manifest = MicrographManifest(
     unique_id="mic-1",
     acquisition_datetime=datetime.now(),

docs/software-architecture.mermaid

@@ -1,4 +1,4 @@
-graph LR
+ graph TD
     subgraph k8s["Kubernetes Cluster (Scientific Compute)"]
         subgraph core["Core Services"]
             api["SmartEM Core Service & API"]
@@ -44,4 +44,4 @@ graph LR
     class infrastructure infra
     class ext ext
     %% Link styling
-    linkStyle 0,1,2,3,4,5,6,7,8 stroke:#666
+    linkStyle 0,1,2,3,4,5,6,7,8 stroke:#666

src/epu_data_intake/__main__.py

@@ -9,7 +9,7 @@
 from watchdog.observers import Observer
 
 from src.epu_data_intake.core_http_api_client import SmartEMAPIClient as APIClient
-from src.epu_data_intake.data_model import EpuAcquisitionSessionStore
+from src.epu_data_intake.model.store import InMemoryDataStore, PersistentDataStore
 from src.epu_data_intake.fs_parser import EpuParser
 from src.epu_data_intake.fs_watcher import (
     DEFAULT_PATTERNS,
@@ -72,15 +72,22 @@ def parse_callback(verbose: int = shared_verbosity_option):
     """Parse group callback to enable verbose flag on parse command"""
     pass
 
+
 @parse_cli.command("dir")
 def parse_epu_output_dir(
-    epu_output_dir: str,
-    verbose: int = shared_verbosity_option,
+        epu_output_dir: str,
+        verbose: int = shared_verbosity_option,
 ):
     """Parse an entire EPU output directory structure. May contain multiple grids"""
-    datastore = EpuAcquisitionSessionStore(epu_output_dir)
+    # Rationale here is that parsers don't persist data to API by design - only watch command does that
+    datastore = InMemoryDataStore(epu_output_dir)
+
+    # Initialize the acquisition_rels dict with a set for the acquisition
+    datastore.acquisition_rels[datastore.acquisition.uuid] = set()
+
+    # The EpuParser.parse_epu_output_dir would need to be updated to work with the new store
     datastore = EpuParser.parse_epu_output_dir(datastore)
-    logging.info(datastore)
+    False and logging.debug(datastore)
 
 
 @parse_cli.command("grid")
@@ -91,12 +98,14 @@ def parse_grid(
     is_valid, errors = EpuParser.validate_project_dir(Path(grid_data_dir))
 
     if not is_valid:
-        logging.info("Grid data dir dir is structurally invalid. Found the following issues:\n")
+        logging.warning("Grid data dir dir is structurally invalid. Found the following issues:\n")
         for error in errors:
-            logging.info(f"- {error}")
+            logging.warning(f"- {error}")
     else:
-        gridstore = EpuParser.parse_grid_dir(grid_data_dir)
-        logging.info(gridstore)
+        # Rationale here is that parsers don't persist data to API by design - only watch command does that
+        datastore = InMemoryDataStore(grid_data_dir) # TODO confirm this is the dir expected here - top-level watch dir or grid root dir?
+        grid_uuid = EpuParser.parse_grid_dir(grid_data_dir, datastore)
+        False and logging.debug(datastore)
 
 
 @parse_cli.command("session")

src/epu_data_intake/core_http_api_client.py

@@ -35,9 +35,9 @@
     FoilHoleResponse,
     MicrographResponse,
 )
-from src.epu_data_intake.data_model import (
-    EpuSessionData,
-    Grid,
+from src.epu_data_intake.model.schemas import (
+    AcquisitionData,
+    GridData,
     GridSquareData,
     FoilHoleData,
     MicrographData,
@@ -66,10 +66,11 @@ class EntityConverter:
     """
 
     @staticmethod
-    def epu_session_to_request(entity: EpuSessionData) -> AcquisitionCreateRequest:
+    def acquisition_to_request(entity: AcquisitionData) -> AcquisitionCreateRequest:
         """Convert EPU session data to acquisition request model"""
         return AcquisitionCreateRequest(
-            id=entity.id,
+            uuid=entity.uuid,
+            # TODO check if natural `id` should also be included
             name=entity.name,
             start_time=entity.start_time,
             storage_path=entity.storage_path,
@@ -80,11 +81,11 @@ def epu_session_to_request(entity: EpuSessionData) -> AcquisitionCreateRequest:
         )
 
     @staticmethod
-    def grid_to_request(entity: Grid, acquisition_id: str) -> GridCreateRequest:
+    def grid_to_request(entity: GridData, acquisition_id: str) -> GridCreateRequest:
         """Convert Grid data to grid request model"""
         return GridCreateRequest(
-            id=entity.id,
-            name=entity.session_data.name if entity.session_data else "Unknown",
+            uuid=entity.uuid,
+            name=entity.acquisition_data.name if entity.acquisition_data else "Unknown",  # Fixed to acquisition_data
             acquisition_id=acquisition_id,
             data_dir=str(entity.data_dir) if entity.data_dir else None,
             atlas_dir=str(entity.atlas_dir) if entity.atlas_dir else None,
@@ -407,11 +408,11 @@ async def aget_acquisitions(self) -> List[AcquisitionResponse]:
     get_acquisitions = _sync_wrap(aget_acquisitions)
 
     async def acreate_acquisition(self,
-                                  acquisition: Union[AcquisitionCreateRequest, EpuSessionData]) -> AcquisitionResponse:
+                                  acquisition: Union[AcquisitionCreateRequest, AcquisitionData]) -> AcquisitionResponse:
         """Create a new acquisition (async)"""
-        # Convert EpuSessionData to AcquisitionCreateRequest if needed
-        if isinstance(acquisition, EpuSessionData):
-            acquisition = EntityConverter.epu_session_to_request(acquisition)
+        # Convert AcquisitionData to AcquisitionCreateRequest if needed
+        if isinstance(acquisition, AcquisitionData):
+            acquisition = EntityConverter.acquisition_to_request(acquisition)
 
         response = await self._request("post", "acquisitions", acquisition, AcquisitionResponse)
         # Store ID mapping
@@ -473,11 +474,11 @@ async def aget_acquisition_grids(self, acquisition_id: str) -> List[GridResponse
     async def acreate_acquisition_grid(
             self,
             acquisition_id: str,
-            grid: Union[GridCreateRequest, Grid]
+            grid: Union[GridCreateRequest, GridData]
     ) -> GridResponse:
         """Create a new grid for a specific acquisition (async)"""
         # Convert Grid to GridCreateRequest if needed
-        if isinstance(grid, Grid):
+        if isinstance(grid, GridData):
             grid_request = EntityConverter.grid_to_request(grid, acquisition_id)
         else:
             grid_request = grid
@@ -787,13 +788,13 @@ def create(self, entity_type: str, entity_id: str, entity: Any, parent: Optional
             self._logger.info(f"Creating {entity_type}/{entity_id} via API" +
                               (f" with parent {parent[0]}/{parent[1]}" if parent else ""))
 
-            if entity_type == "acquisition" and isinstance(entity, EpuSessionData):
+            if entity_type == "acquisition" and isinstance(entity, AcquisitionData):
                 response = self.create_acquisition(entity)
                 self._store_entity_id_mapping("acquisition", entity_id, response.id)
                 self._logger.info(f"Successfully created acquisition {entity_id} (DB ID: {response.id})")
                 return True
 
-            elif entity_type == "grid" and isinstance(entity, Grid):
+            elif entity_type == "grid" and isinstance(entity, GridData):
                 if parent and parent[0] == "acquisition":
                     acquisition_db_id = self._get_db_id("acquisition", parent[1])
                     if not acquisition_db_id:
@@ -823,6 +824,27 @@ def create(self, entity_type: str, entity_id: str, entity: Any, parent: Optional
                     self._logger.error("Cannot create gridsquare: No valid grid parent")
                     return False
 
+            # TODO this looks like nonsense, check it can be thrown away:
+            # elif entity_type == "gridsquare" and isinstance(entity, GridSquareData):
+            #     if parent and parent[0] == "grid":
+            #         grid_uuid = parent[1]
+            #         # Ensure grid_uuid is set on the entity
+            #         if not hasattr(entity, 'grid_uuid') or not entity.grid_uuid:
+            #             entity.grid_uuid = grid_uuid
+            #
+            #         grid_db_id = self._get_db_id("grid", grid_uuid)
+            #         if not grid_db_id:
+            #             self._logger.error(f"Cannot create gridsquare: Grid {grid_uuid} not found in ID map")
+            #             return False
+            #
+            #         response = self.create_grid_gridsquare(grid_db_id, entity)
+            #         self._store_entity_id_mapping("gridsquare", entity_id, response.id)
+            #         self._logger.info(f"Successfully created gridsquare {entity_id} (DB ID: {response.id})")
+            #         return True
+            #     else:
+            #         self._logger.error("Cannot create gridsquare: No valid grid parent")
+            #         return False
+
             elif entity_type == "foilhole" and isinstance(entity, FoilHoleData):
                 if parent and parent[0] == "gridsquare":
                     gridsquare_db_id = self._get_db_id("gridsquare", parent[1])