rework message queue event model

README.md

@@ -92,7 +92,7 @@ pip install -e .[all] # .[dev,common,core]
 
 # Start services with verbosity controls:
 python -m smartem_decisions.run_api -v               # HTTP API with INFO logging
-python -m smartem_decisions.mq_consumer -v           # Message queue consumer with INFO logging  
+python -m smartem_decisions.consumer -v           # Message queue consumer with INFO logging  
 python -m epu_data_intake watch /path/to/data -v     # File watcher with INFO logging
 ```
 
@@ -161,17 +161,17 @@ Use the `-v` and `-vv` flags to control verbosity:
 
 ```bash
 # ERROR level only (default - minimal output)
-python -m smartem_decisions.mq_consumer
+python -m smartem_decisions.consumer
 python -m smartem_decisions.run_api
 python -m epu_data_intake watch /path/to/data
 
 # INFO level and above (-v flag)
-python -m smartem_decisions.mq_consumer -v
+python -m smartem_decisions.consumer -v
 python -m smartem_decisions.run_api -v
 python -m epu_data_intake watch /path/to/data -v
 
 # DEBUG level and above (-vv flag - most verbose)
-python -m smartem_decisions.mq_consumer -vv
+python -m smartem_decisions.consumer -vv
 python -m smartem_decisions.run_api -vv
 python -m epu_data_intake watch /path/to/data -vv
 ```
@@ -181,10 +181,10 @@ python -m epu_data_intake watch /path/to/data -vv
 For the HTTP API, you can also control logging via environment variables:
 
 ```bash
-# Set log level via environment variable
-SMARTEM_LOG_LEVEL=ERROR uvicorn src.smartem_decisions.http_api:app --host 0.0.0.0 --port 8000
-SMARTEM_LOG_LEVEL=INFO uvicorn src.smartem_decisions.http_api:app --host 0.0.0.0 --port 8000
-SMARTEM_LOG_LEVEL=DEBUG uvicorn src.smartem_decisions.http_api:app --host 0.0.0.0 --port 8000
+# Set log level via environment variable (equivalent to -v/-vv flags)
+SMARTEM_LOG_LEVEL=ERROR python -m smartem_decisions.run_api
+SMARTEM_LOG_LEVEL=INFO python -m smartem_decisions.run_api 
+SMARTEM_LOG_LEVEL=DEBUG python -m smartem_decisions.run_api
 ```
 
 ### Log Levels

docs/how-to/run-container.md

@@ -7,8 +7,54 @@ installed are available on [Github Container Registry](https://ghcr.io/DiamondLi
 
 To pull the container from github container registry and run:
 
-```
+```bash
+# Check version
 $ docker run ghcr.io/DiamondLightSource/smartem-decisions:latest --version
 ```
 
 To get a released version, use a numbered release instead of `latest`.
+
+## Running services
+
+The container can run in different modes controlled by the `ROLE` environment variable:
+
+### HTTP API Service
+
+```bash
+# Run API with default ERROR logging
+$ docker run -p 8000:8000 -e ROLE=api ghcr.io/DiamondLightSource/smartem-decisions:latest
+
+# Run API with INFO logging (equivalent to -v flag)
+$ docker run -p 8000:8000 -e ROLE=api -e SMARTEM_LOG_LEVEL=INFO ghcr.io/DiamondLightSource/smartem-decisions:latest
+
+# Run API with DEBUG logging (equivalent to -vv flag)
+$ docker run -p 8000:8000 -e ROLE=api -e SMARTEM_LOG_LEVEL=DEBUG ghcr.io/DiamondLightSource/smartem-decisions:latest
+
+# Custom port
+$ docker run -p 9000:9000 -e ROLE=api -e HTTP_API_PORT=9000 ghcr.io/DiamondLightSource/smartem-decisions:latest
+```
+
+### Message Queue Worker
+
+```bash
+# Run worker with default ERROR logging
+$ docker run -e ROLE=worker ghcr.io/DiamondLightSource/smartem-decisions:latest
+
+# Run worker with INFO logging
+$ docker run -e ROLE=worker -e SMARTEM_LOG_LEVEL=INFO ghcr.io/DiamondLightSource/smartem-decisions:latest
+
+# Run worker with DEBUG logging
+$ docker run -e ROLE=worker -e SMARTEM_LOG_LEVEL=DEBUG ghcr.io/DiamondLightSource/smartem-decisions:latest
+```
+
+## Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `ROLE` | `api` | Service role: `api` or `worker` |
+| `HTTP_API_PORT` | `8000` | Port for HTTP API service |
+| `SMARTEM_LOG_LEVEL` | `ERROR` | Log level: `ERROR`, `INFO`, or `DEBUG` |
+
+## Complete Development Stack
+
+For a complete development environment with database and message queue, see the [Kubernetes deployment guide](../../k8s/README.md) which provides a docker-compose-like experience.

entrypoint.sh

@@ -15,11 +15,11 @@ case "${ROLE:-api}" in
         echo "Starting HTTP API..."
         # TODO we don't want to do it indiscriminately on every container launch:
         python -m smartem_decisions.model.database
-        exec uvicorn smartem_decisions.http_api:app --host 0.0.0.0 --port $HTTP_API_PORT
+        exec python -m smartem_decisions.run_api --host 0.0.0.0 --port ${HTTP_API_PORT:-8000}
         ;;
     worker)
         echo "Starting RabbitMQ consumer..."
-        exec python -m smartem_decisions.mq_consumer
+        exec python -m smartem_decisions.consumer
         ;;
     *)
         echo "Unknown role: $ROLE"

src/epu_data_intake/fs_watcher.py

@@ -36,12 +36,19 @@
 DEFAULT_PATTERNS = [
     # TODO consider merging with props in EpuParser
     # TODO (techdebt) This should be treated as immutable - don't modify!
+    # Support both root-level files and files within acquisition subdirectories
     "EpuSession.dm",
+    "*/EpuSession.dm",
     "Metadata/GridSquare_*.dm",
+    "*/Metadata/GridSquare_*.dm",
     "Images-Disc*/GridSquare_*/GridSquare_*_*.xml",
+    "*/Images-Disc*/GridSquare_*/GridSquare_*_*.xml",
     "Images-Disc*/GridSquare_*/Data/FoilHole_*_Data_*_*_*_*.xml",
+    "*/Images-Disc*/GridSquare_*/Data/FoilHole_*_Data_*_*_*_*.xml",
     "Images-Disc*/GridSquare_*/FoilHoles/FoilHole_*_*_*.xml",
+    "*/Images-Disc*/GridSquare_*/FoilHoles/FoilHole_*_*_*.xml",
     "Sample*/Atlas/Atlas.dm",
+    "*/Sample*/Atlas/Atlas.dm",
 ]
 
 

src/smartem_decisions/README.md

@@ -5,9 +5,9 @@
 ./tools/dev-k8s.sh up
 
 # launch RabbitMQ worker (consumer)
-python -m smartem_decisions.mq_consumer              # ERROR level (default)
-python -m smartem_decisions.mq_consumer -v           # INFO level  
-python -m smartem_decisions.mq_consumer -vv          # DEBUG level
+python -m smartem_decisions.consumer              # ERROR level (default)
+python -m smartem_decisions.consumer -v           # INFO level  
+python -m smartem_decisions.consumer -vv          # DEBUG level
 
 # simulating an system event: 
 python -m smartem_decisions.simulate_msg --help # to see a list of options

src/smartem_decisions/cli/initialise_prediction_model_weights.py

@@ -1,21 +1,74 @@
 import typer
 from sqlmodel import Session, select
 
-from smartem_decisions.model.database import Grid, QualityPredictionModelWeight
-from smartem_decisions.utils import setup_postgres_connection
+from smartem_decisions.model.database import Grid, QualityPredictionModel, QualityPredictionModelWeight
+from smartem_decisions.utils import logger, setup_postgres_connection
 
 
-def initialise(name: str, weight: float, grid_uuid: str | None = None) -> None:
+def initialise_all_models_for_grid(grid_uuid: str) -> None:
+    """
+    Initialise prediction model weights for all available models for a specific grid.
+
+    Args:
+        grid_uuid: UUID of the grid to initialise weights for
+        default_weight: Default weight value to assign (default: DEFAULT_PREDICTION_MODEL_WEIGHT)
+    """
+    engine = setup_postgres_connection()
+    with Session(engine) as sess:
+        # Get all available prediction models
+        models = sess.exec(select(QualityPredictionModel)).all()
+
+        if not models:
+            logger.warning(f"No prediction models found to initialise for grid {grid_uuid}")
+            return
+
+        # Initialise weights for each model
+        default_weight = 1 / len(models)
+        for model in models:
+            # Check if weight already exists for this grid-model combination
+            existing_weight = sess.exec(
+                select(QualityPredictionModelWeight).where(
+                    QualityPredictionModelWeight.grid_uuid == grid_uuid,
+                    QualityPredictionModelWeight.prediction_model_name == model.name,
+                )
+            ).first()
+
+            if existing_weight is None:
+                weight_entry = QualityPredictionModelWeight(
+                    grid_uuid=grid_uuid, prediction_model_name=model.name, weight=default_weight
+                )
+                sess.add(weight_entry)
+                logger.info(f"Initialised weight {default_weight} for model '{model.name}' on grid {grid_uuid}")
+            else:
+                logger.debug(f"Weight already exists for model '{model.name}' on grid {grid_uuid}")
+
+        sess.commit()
+
+
+def initialise_prediction_model_for_grid(name: str, weight: float, grid_uuid: str | None = None) -> None:
+    """
+    Initialise a single prediction model weight for a grid (CLI interface).
+
+    Args:
+        name: Prediction model name
+        weight: Weight value to assign
+        grid_uuid: Grid UUID (if None, uses first available grid)
+    """
     engine = setup_postgres_connection()
     with Session(engine) as sess:
         if grid_uuid is None:
             grid = sess.exec(select(Grid)).first()
+            if grid is None:
+                logger.error("No grids found in database")
+                return
             grid_uuid = grid.uuid
-        sess.add(QualityPredictionModelWeight(grid_id=grid_uuid, prediction_model_name=name, weight=weight))
+
+        sess.add(QualityPredictionModelWeight(grid_uuid=grid_uuid, prediction_model_name=name, weight=weight))
         sess.commit()
+        logger.info(f"Initialised weight {weight} for model '{name}' on grid {grid_uuid}")
     return None
 
 
 def run() -> None:
-    typer.run(initialise)
+    typer.run(initialise_prediction_model_for_grid)
     return None

src/smartem_decisions/cli/random_model_predictions.py

@@ -4,8 +4,10 @@
 import typer
 from sqlmodel import Session, select
 
-from smartem_decisions.model.database import FoilHole, Grid, GridSquare, QualityPrediction
-from smartem_decisions.utils import setup_postgres_connection
+from smartem_decisions.model.database import FoilHole, Grid, GridSquare, QualityPrediction, QualityPredictionModel
+from smartem_decisions.utils import logger, setup_postgres_connection
+
+DEFAULT_PREDICTION_RANGE = (0.0, 1.0)
 
 
 def generate_random_predictions(
@@ -55,6 +57,121 @@ def generate_random_predictions(
     return None
 
 
+def generate_predictions_for_gridsquare(gridsquare_uuid: str, grid_uuid: str | None = None) -> None:
+    """
+    Generate random predictions for a single gridsquare using all available models.
+
+    Args:
+        gridsquare_uuid: UUID of the gridsquare to generate predictions for
+        grid_uuid: UUID of the parent grid (optional, will be looked up if not provided)
+    """
+    engine = setup_postgres_connection()
+    with Session(engine) as sess:
+        # Get all available prediction models
+        models = sess.exec(select(QualityPredictionModel)).all()
+
+        if not models:
+            logger.warning(f"No prediction models found to generate predictions for gridsquare {gridsquare_uuid}")
+            return
+
+        # If grid_uuid not provided, look it up
+        if grid_uuid is None:
+            gridsquare = sess.get(GridSquare, gridsquare_uuid)
+            if gridsquare is None:
+                logger.error(f"GridSquare {gridsquare_uuid} not found in database")
+                return
+            grid_uuid = gridsquare.grid_uuid
+
+        # Generate predictions for each model
+        predictions = []
+        for model in models:
+            # Check if prediction already exists for this gridsquare-model combination
+            existing_prediction = sess.exec(
+                select(QualityPrediction).where(
+                    QualityPrediction.gridsquare_uuid == gridsquare_uuid,
+                    QualityPrediction.prediction_model_name == model.name,
+                )
+            ).first()
+
+            if existing_prediction is None:
+                prediction = QualityPrediction(
+                    value=random.uniform(DEFAULT_PREDICTION_RANGE[0], DEFAULT_PREDICTION_RANGE[1]),
+                    prediction_model_name=model.name,
+                    gridsquare_uuid=gridsquare_uuid,
+                )
+                predictions.append(prediction)
+                logger.info(
+                    f"Generated prediction {prediction.value:.3f} for model '{model.name}' "
+                    f"on gridsquare {gridsquare_uuid}"
+                )
+            else:
+                logger.debug(f"Prediction already exists for model '{model.name}' on gridsquare {gridsquare_uuid}")
+
+        if predictions:
+            sess.add_all(predictions)
+            sess.commit()
+            logger.info(f"Generated {len(predictions)} predictions for gridsquare {gridsquare_uuid}")
+
+
+def generate_predictions_for_foilhole(foilhole_uuid: str, gridsquare_uuid: str | None = None) -> None:
+    """
+    Generate random predictions for a single foilhole using all available models.
+
+    Args:
+        foilhole_uuid: UUID of the foilhole to generate predictions for
+        gridsquare_uuid: UUID of the parent gridsquare (optional, for validation if provided)
+    """
+    engine = setup_postgres_connection()
+    with Session(engine) as sess:
+        # Get all available prediction models
+        models = sess.exec(select(QualityPredictionModel)).all()
+
+        if not models:
+            logger.warning(f"No prediction models found to generate predictions for foilhole {foilhole_uuid}")
+            return
+
+        # Optional validation: if gridsquare_uuid provided, verify the foilhole belongs to it
+        if gridsquare_uuid is not None:
+            foilhole = sess.get(FoilHole, foilhole_uuid)
+            if foilhole is None:
+                logger.error(f"FoilHole {foilhole_uuid} not found in database")
+                return
+            if foilhole.gridsquare_uuid != gridsquare_uuid:
+                logger.error(
+                    f"FoilHole {foilhole_uuid} belongs to gridsquare {foilhole.gridsquare_uuid}, not {gridsquare_uuid}"
+                )
+                return
+
+        # Generate predictions for each model
+        predictions = []
+        for model in models:
+            # Check if prediction already exists for this foilhole-model combination
+            existing_prediction = sess.exec(
+                select(QualityPrediction).where(
+                    QualityPrediction.foilhole_uuid == foilhole_uuid,
+                    QualityPrediction.prediction_model_name == model.name,
+                )
+            ).first()
+
+            if existing_prediction is None:
+                prediction = QualityPrediction(
+                    value=random.uniform(DEFAULT_PREDICTION_RANGE[0], DEFAULT_PREDICTION_RANGE[1]),
+                    prediction_model_name=model.name,
+                    foilhole_uuid=foilhole_uuid,
+                )
+                predictions.append(prediction)
+                logger.info(
+                    f"Generated prediction {prediction.value:.3f} for model '{model.name}' on foilhole {foilhole_uuid}"
+                )
+            else:
+                logger.debug(f"Prediction already exists for model '{model.name}' on foilhole {foilhole_uuid}")
+
+        if predictions:
+            sess.add_all(predictions)
+            sess.commit()
+            logger.info(f"Generated {len(predictions)} predictions for foilhole {foilhole_uuid}")
+
+
 def run() -> None:
     typer.run(generate_random_predictions)
     return None