Acquisition and Grid entities persist end-to-end

Author: vredchenko

docs/how-to/use-core-http-api-client.md

@@ -68,7 +68,7 @@ acquisitions = await client.aget_acquisitions()
 ### Creating an Acquisition
 
 ```python
-from src.epu_data_intake.data_model import EpuSessionData
+from src.epu_data_intake.model.schemas import EpuSessionData
 from datetime import datetime
 
 # Create an acquisition from EPU session data
@@ -139,22 +139,26 @@ Here's an example of creating entities at each level:
 acquisition = client.create_acquisition(EpuSessionData(id="acq-1", name="Test Acquisition"))
 
 # Create a grid for the acquisition
-from src.epu_data_intake.data_model import Grid
-grid = Grid(data_dir="/path/to/grid")
+from src.epu_data_intake.model.schemas import GridData
+
+grid = GridData(data_dir="/path/to/grid")
 grid_response = client.create_acquisition_grid(acquisition.id, grid)
 
 # Create a grid square for the grid
-from src.epu_data_intake.data_model import GridSquareData
+from src.epu_data_intake.model.schemas import GridSquareData
+
 gridsquare = GridSquareData(id="gs-1", data_dir="/path/to/gridsquare")
 gridsquare_response = client.create_grid_gridsquare(grid_response.id, gridsquare)
 
 # Create a foil hole for the grid square
-from src.epu_data_intake.data_model import FoilHoleData
+from src.epu_data_intake.model.schemas import FoilHoleData
+
 foilhole = FoilHoleData(id="fh-1", gridsquare_id=gridsquare.id)
 foilhole_response = client.create_gridsquare_foilhole(gridsquare_response.id, foilhole)
 
 # Create a micrograph for the foil hole
-from src.epu_data_intake.data_model import MicrographData, MicrographManifest
+from src.epu_data_intake.model.schemas import MicrographData, MicrographManifest
+
 manifest = MicrographManifest(
     unique_id="mic-1",
     acquisition_datetime=datetime.now(),

docs/software-architecture.mermaid

@@ -1,4 +1,4 @@
-graph LR
+ graph TD
     subgraph k8s["Kubernetes Cluster (Scientific Compute)"]
         subgraph core["Core Services"]
             api["SmartEM Core Service & API"]
@@ -44,4 +44,4 @@ graph LR
     class infrastructure infra
     class ext ext
     %% Link styling
-    linkStyle 0,1,2,3,4,5,6,7,8 stroke:#666
+    linkStyle 0,1,2,3,4,5,6,7,8 stroke:#666

src/epu_data_intake/__main__.py

@@ -9,7 +9,7 @@
 from watchdog.observers import Observer
 
 from src.epu_data_intake.core_http_api_client import SmartEMAPIClient as APIClient
-from src.epu_data_intake.data_model import EpuAcquisitionSessionStore
+from src.epu_data_intake.model.store import InMemoryDataStore, PersistentDataStore
 from src.epu_data_intake.fs_parser import EpuParser
 from src.epu_data_intake.fs_watcher import (
     DEFAULT_PATTERNS,
@@ -72,15 +72,22 @@ def parse_callback(verbose: int = shared_verbosity_option):
     """Parse group callback to enable verbose flag on parse command"""
     pass
 
+
 @parse_cli.command("dir")
 def parse_epu_output_dir(
-    epu_output_dir: str,
-    verbose: int = shared_verbosity_option,
+        epu_output_dir: str,
+        verbose: int = shared_verbosity_option,
 ):
     """Parse an entire EPU output directory structure. May contain multiple grids"""
-    datastore = EpuAcquisitionSessionStore(epu_output_dir)
+    # Rationale here is that parsers don't persist data to API by design - only watch command does that
+    datastore = InMemoryDataStore(epu_output_dir)
+
+    # Initialize the acquisition_rels dict with a set for the acquisition
+    datastore.acquisition_rels[datastore.acquisition.uuid] = set()
+
+    # The EpuParser.parse_epu_output_dir would need to be updated to work with the new store
     datastore = EpuParser.parse_epu_output_dir(datastore)
-    logging.info(datastore)
+    False and logging.debug(datastore)
 
 
 @parse_cli.command("grid")
@@ -91,12 +98,14 @@ def parse_grid(
     is_valid, errors = EpuParser.validate_project_dir(Path(grid_data_dir))
 
     if not is_valid:
-        logging.info("Grid data dir dir is structurally invalid. Found the following issues:\n")
+        logging.warning("Grid data dir dir is structurally invalid. Found the following issues:\n")
         for error in errors:
-            logging.info(f"- {error}")
+            logging.warning(f"- {error}")
     else:
-        gridstore = EpuParser.parse_grid_dir(grid_data_dir)
-        logging.info(gridstore)
+        # Rationale here is that parsers don't persist data to API by design - only watch command does that
+        datastore = InMemoryDataStore(grid_data_dir) # TODO confirm this is the dir expected here - top-level watch dir or grid root dir?
+        grid_uuid = EpuParser.parse_grid_dir(grid_data_dir, datastore)
+        False and logging.debug(datastore)
 
 
 @parse_cli.command("session")