Improve module structure

docs/how-to/config-server-guide.md

@@ -20,7 +20,7 @@ By default, this will return the file's raw string output - which includes thing
 (file-converters)=
 # File converters
 
-Converters can be used to turn a file into a standard format server-side, reducing the complexity of reading config files client-side. Converters can convert config files to a `dict` or `pydantic model`, and the same `pydantic model` can be reconstructed client-side by the `get_file_contents` method. Available converters exist [here](https://github.com/DiamondLightSource/daq-config-server/blob/main/converters._converters.py) - see if there's a suitable converter you can use before adding your own. [This dictionary](https://github.com/DiamondLightSource/daq-config-server/blob/main/converters._file_converter_map.py) maps files to converters. Add the path of your config file and a suitable converter to this dictionary and it will automatically be used by the config server when a request for that file is made.
+Converters can be used to turn a file into a standard format server-side, reducing the complexity of reading config files client-side. Converters can convert config files to a `dict` or pydantic model, and the same pydantic model can be reconstructed client-side by the `get_file_contents` method. Available converters and models exist [here](https://github.com/DiamondLightSource/daq-config-server/blob/main/src/daq_config_server/models/converters/), divided into modules based on the type of config they convert - see if there's a suitable converter you can use before adding your own. [This dictionary](https://github.com/DiamondLightSource/daq-config-server/blob/main/src/daq_config_server/models/converters/_file_converter_map.py) maps files to converters. Add the path of your config file and a suitable converter to this dictionary and it will automatically be used by the config server when a request for that file is made. Models should be added to this [`__init__.py`](https://github.com/DiamondLightSource/daq-config-server/blob/main/src/daq_config_server/models/__init__.py) so that they can be imported with `from daq_config_server.models import MyModel`.
 
 A request for `str` or `bytes` will fetch the raw file with no conversion.
 

src/daq_config_server/app.py

@@ -15,8 +15,8 @@
 from daq_config_server.constants import (
     ENDPOINTS,
 )
-from daq_config_server.converters import get_converted_file_contents
 from daq_config_server.log import set_up_logging
+from daq_config_server.models.converters.convert import get_converted_file_contents
 from daq_config_server.whitelist import get_whitelist
 
 # See https://github.com/DiamondLightSource/daq-config-server/issues/105

src/daq_config_server/client.py

@@ -10,7 +10,7 @@
 from requests.exceptions import HTTPError
 
 from daq_config_server.app import ValidAcceptHeaders
-from daq_config_server.converters.models import ConfigModel
+from daq_config_server.models import ConfigModel
 
 from .constants import ENDPOINTS
 

src/daq_config_server/models/__init__.py

@@ -0,0 +1,8 @@
+from .converters import ConfigModel
+from .converters.display_config import (
+    DisplayConfig,
+    DisplayConfigData,
+)
+from .converters.lookup_tables import GenericLookupTable
+
+__all__ = ["ConfigModel", "DisplayConfig", "DisplayConfigData", "GenericLookupTable"]

src/daq_config_server/models/converters/__init__.py

@@ -0,0 +1,8 @@
+from ._base_model import ConfigModel
+from ._converter_utils import parse_value, remove_comments
+
+__all__ = [
+    "ConfigModel",
+    "remove_comments",
+    "parse_value",
+]

src/daq_config_server/models/converters/_base_model.py

@@ -0,0 +1,4 @@
+from pydantic import BaseModel
+
+
+class ConfigModel(BaseModel): ...

src/daq_config_server/models/converters/_converter_utils.py

@@ -0,0 +1,25 @@
+import ast
+from typing import Any
+
+ALLOWED_BEAMLINE_PARAMETER_STRINGS = ["FB", "FULL", "deadtime"]
+
+
+class ConverterParseError(Exception): ...
+
+
+def remove_comments(lines: list[str]) -> list[str]:
+    return [
+        line.strip().split("#", 1)[0].strip()
+        for line in lines
+        if line.strip().split("#", 1)[0]
+    ]
+
+
+def parse_value(value: str, convert_to: type | None = None) -> Any:
+    """Convert a string value into an appropriate Python type. Optionally provide a type
+    to convert to. If not given, the type will be inferred.
+    """
+    value = ast.literal_eval(value.replace("Yes", "True").replace("No", "False"))
+    if convert_to:
+        value = convert_to(value)
+    return value

src/daq_config_server/converters/_file_converter_map.py → src/daq_config_server/models/converters/_file_converter_map.py

@@ -3,15 +3,20 @@
 
 import xmltodict
 
-from daq_config_server.converters._converters import (
+from daq_config_server.models.converters.beamline_parameters import (
     beamline_parameters_to_dict,
+)
+from daq_config_server.models.converters.display_config import (
+    display_config_to_model,
+)
+from daq_config_server.models.converters.lookup_tables import (
     beamline_pitch_lut,
     beamline_roll_lut,
     detector_xy_lut,
-    display_config_to_model,
     undulator_energy_gap_lut,
 )
-from daq_config_server.converters.models import ConfigModel
+
+from ._base_model import ConfigModel
 
 FILE_TO_CONVERTER_MAP: dict[str, Callable[[str], ConfigModel | dict[str, Any]]] = {  # type: ignore
     "/tests/test_data/test_good_lut.txt": undulator_energy_gap_lut,  # For system tests # noqa

src/daq_config_server/models/converters/beamline_parameters/__init__.py

@@ -0,0 +1,3 @@
+from ._converters import beamline_parameters_to_dict
+
+__all__ = ["beamline_parameters_to_dict"]

src/daq_config_server/models/converters/beamline_parameters/_converters.py

@@ -0,0 +1,28 @@
+from typing import Any
+
+from daq_config_server.models.converters import parse_value, remove_comments
+
+ALLOWED_BEAMLINE_PARAMETER_STRINGS = ["FB", "FULL", "deadtime"]
+
+
+def beamline_parameters_to_dict(contents: str) -> dict[str, Any]:
+    """Extracts the key value pairs from a beamline parameters file. If the value
+    is in ALLOWED_BEAMLINE_PARAMETER_STRINGS, it leaves it as a string. Otherwise, it is
+    converted to a number or bool."""
+    lines = contents.splitlines()
+    config_pairs: dict[str, Any] = {}
+
+    # Get dict of parameter keys and values
+    for line in remove_comments(lines):
+        splitline = line.split("=")
+        if len(splitline) >= 2:
+            param, value = line.split("=")
+            if param.strip() in config_pairs:
+                raise ValueError(f"Repeated key in parameters: {param}")
+            config_pairs[param.strip()] = value.strip()
+
+    # Parse each value
+    for param, value in config_pairs.items():
+        if value not in ALLOWED_BEAMLINE_PARAMETER_STRINGS:
+            config_pairs[param] = parse_value(value)
+    return dict(config_pairs)

src/daq_config_server/converters/convert.py → src/daq_config_server/models/converters/convert.py

@@ -2,9 +2,10 @@
 from pathlib import Path
 from typing import Any
 
-import daq_config_server.converters._file_converter_map as file_converter_map
-from daq_config_server.converters._converter_utils import ConverterParseError
-from daq_config_server.converters.models import ConfigModel
+import daq_config_server.models.converters._file_converter_map as file_converter_map
+
+from ._base_model import ConfigModel
+from ._converter_utils import ConverterParseError
 
 
 def get_converted_file_contents(file_path: Path) -> dict[str, Any]:

src/daq_config_server/models/converters/display_config/__init__.py

@@ -0,0 +1,4 @@
+from ._converters import display_config_to_model
+from ._models import DisplayConfig, DisplayConfigData
+
+__all__ = ["display_config_to_model", "DisplayConfig", "DisplayConfigData"]

src/daq_config_server/models/converters/display_config/_converters.py

@@ -0,0 +1,31 @@
+from daq_config_server.models.converters import parse_value, remove_comments
+
+from ._models import DisplayConfig, DisplayConfigData
+
+
+def display_config_to_model(contents: str) -> DisplayConfig:
+    lines = contents.splitlines()
+    config_dict: dict[float, dict[str, int | float]] = {}
+    zoom_level = None
+
+    for line in remove_comments(lines):
+        key, value = (item.strip() for item in line.split("=", 1))
+        if key == "zoomLevel":
+            zoom_level = float(value)
+            assert zoom_level not in config_dict.keys(), (
+                f"Multiple instances of zoomLevel {zoom_level}"
+            )
+            config_dict[zoom_level] = {}
+            continue
+
+        assert zoom_level is not None, "File must start with a zoom level"
+        assert key not in config_dict[zoom_level].keys(), (
+            "File can't have repeated keys for a given zoom level"
+        )
+        config_dict[zoom_level][key] = parse_value(value)
+    zoom_levels = {
+        key: DisplayConfigData.model_validate(value)
+        for key, value in config_dict.items()
+    }
+
+    return DisplayConfig(zoom_levels=zoom_levels)

src/daq_config_server/converters/models.py → src/daq_config_server/models/converters/display_config/_models.py

@@ -1,7 +1,6 @@
-from pydantic import BaseModel, model_validator
+from pydantic import model_validator
 
-
-class ConfigModel(BaseModel): ...
+from daq_config_server.models.converters import ConfigModel
 
 
 class DisplayConfigData(ConfigModel):
@@ -29,19 +28,3 @@ def check_zoom_levels_match_required(self):
                 f"do not match required zoom levels: {self.required_zoom_levels}"
             )
         return self
-
-
-class GenericLookupTable(ConfigModel):
-    column_names: list[str]
-    rows: list[list[int | float]]
-
-    @model_validator(mode="after")
-    def check_row_length_matches_n_columns(self):
-        n_columns = len(self.column_names)
-        for row in self.rows:
-            if len(row) != n_columns:
-                raise ValueError(
-                    f"Length of row {row} does not match number "
-                    f"of columns: {self.column_names}"
-                )
-        return self

src/daq_config_server/models/converters/lookup_tables/__init__.py

@@ -0,0 +1,15 @@
+from ._converters import (
+    beamline_pitch_lut,
+    beamline_roll_lut,
+    detector_xy_lut,
+    undulator_energy_gap_lut,
+)
+from ._models import GenericLookupTable
+
+__all__ = [
+    "GenericLookupTable",
+    "detector_xy_lut",
+    "beamline_pitch_lut",
+    "beamline_roll_lut",
+    "undulator_energy_gap_lut",
+]

src/daq_config_server/converters/_converter_utils.py → src/daq_config_server/models/converters/lookup_tables/_converters.py

@@ -1,30 +1,8 @@
-import ast
 from typing import Any
 
-from daq_config_server.converters.models import GenericLookupTable
+from daq_config_server.models.converters import parse_value, remove_comments
 
-ALLOWED_BEAMLINE_PARAMETER_STRINGS = ["FB", "FULL", "deadtime"]
-
-
-class ConverterParseError(Exception): ...
-
-
-def remove_comments(lines: list[str]) -> list[str]:
-    return [
-        line.strip().split("#", 1)[0].strip()
-        for line in lines
-        if line.strip().split("#", 1)[0]
-    ]
-
-
-def parse_value(value: str, convert_to: type | None = None) -> Any:
-    """Convert a string value into an appropriate Python type. Optionally provide a type
-    to convert to. If not given, the type will be inferred.
-    """
-    value = ast.literal_eval(value.replace("Yes", "True").replace("No", "False"))
-    if convert_to:
-        value = convert_to(value)
-    return value
+from ._models import GenericLookupTable
 
 
 def parse_lut(contents: str, *params: tuple[str, type | None]) -> GenericLookupTable:
@@ -46,3 +24,24 @@ def parse_lut(contents: str, *params: tuple[str, type | None]) -> GenericLookupT
             [parse_value(value, types[i]) for i, value in enumerate(line.split())]
         )
     return GenericLookupTable(column_names=column_names, rows=rows)
+
+
+def detector_xy_lut(contents: str) -> GenericLookupTable:
+    return parse_lut(
+        contents,
+        ("detector_distances_mm", float),
+        ("beam_centre_x_mm", float),
+        ("beam_centre_y_mm", float),
+    )
+
+
+def beamline_pitch_lut(contents: str) -> GenericLookupTable:
+    return parse_lut(contents, ("bragg_angle_deg", float), ("pitch_mrad", float))
+
+
+def beamline_roll_lut(contents: str) -> GenericLookupTable:
+    return parse_lut(contents, ("bragg_angle_deg", float), ("roll_mrad", float))
+
+
+def undulator_energy_gap_lut(contents: str) -> GenericLookupTable:
+    return parse_lut(contents, ("energy_eV", int), ("gap_mm", float))

src/daq_config_server/models/converters/lookup_tables/_models.py

@@ -0,0 +1,19 @@
+from pydantic import model_validator
+
+from daq_config_server.models.converters import ConfigModel
+
+
+class GenericLookupTable(ConfigModel):
+    column_names: list[str]
+    rows: list[list[int | float]]
+
+    @model_validator(mode="after")
+    def check_row_length_matches_n_columns(self):
+        n_columns = len(self.column_names)
+        for row in self.rows:
+            if len(row) != n_columns:
+                raise ValueError(
+                    f"Length of row {row} does not match number "
+                    f"of columns: {self.column_names}"
+                )
+        return self