134 improve module structure (#140)

* Split converters and models into modules

* Move tests

* Add models to public API

* Clean up APIs and private modules

Author: jacob720

docs/how-to/config-server-guide.md

@@ -20,7 +20,7 @@ By default, this will return the file's raw string output - which includes thing
 (file-converters)=
 # File converters
 
-Converters can be used to turn a file into a standard format server-side, reducing the complexity of reading config files client-side. Converters can convert config files to a `dict` or `pydantic model`, and the same `pydantic model` can be reconstructed client-side by the `get_file_contents` method. Available converters exist [here](https://github.com/DiamondLightSource/daq-config-server/blob/main/converters._converters.py) - see if there's a suitable converter you can use before adding your own. [This dictionary](https://github.com/DiamondLightSource/daq-config-server/blob/main/converters._file_converter_map.py) maps files to converters. Add the path of your config file and a suitable converter to this dictionary and it will automatically be used by the config server when a request for that file is made.
+Converters can be used to turn a file into a standard format server-side, reducing the complexity of reading config files client-side. Converters can convert config files to a `dict` or pydantic model, and the same pydantic model can be reconstructed client-side by the `get_file_contents` method. Available converters and models exist [here](https://github.com/DiamondLightSource/daq-config-server/blob/main/src/daq_config_server/models/converters/), divided into modules based on the type of config they convert - see if there's a suitable converter you can use before adding your own. [This dictionary](https://github.com/DiamondLightSource/daq-config-server/blob/main/src/daq_config_server/models/converters/_file_converter_map.py) maps files to converters. Add the path of your config file and a suitable converter to this dictionary and it will automatically be used by the config server when a request for that file is made. Models should be added to this [`__init__.py`](https://github.com/DiamondLightSource/daq-config-server/blob/main/src/daq_config_server/models/__init__.py) so that they can be imported with `from daq_config_server.models import MyModel`.
 
 A request for `str` or `bytes` will fetch the raw file with no conversion.
 

src/daq_config_server/app.py

@@ -15,8 +15,8 @@
 from daq_config_server.constants import (
     ENDPOINTS,
 )
-from daq_config_server.converters import get_converted_file_contents
 from daq_config_server.log import set_up_logging
+from daq_config_server.models.converters.convert import get_converted_file_contents
 from daq_config_server.whitelist import get_whitelist
 
 # See https://github.com/DiamondLightSource/daq-config-server/issues/105

src/daq_config_server/client.py

@@ -10,7 +10,7 @@
 from requests.exceptions import HTTPError
 
 from daq_config_server.app import ValidAcceptHeaders
-from daq_config_server.converters.models import ConfigModel
+from daq_config_server.models import ConfigModel
 
 from .constants import ENDPOINTS
 

src/daq_config_server/converters/__init__.py

@@ -0,0 +1,8 @@
+from .converters import ConfigModel
+from .converters.display_config import (
+    DisplayConfig,
+    DisplayConfigData,
+)
+from .converters.lookup_tables import GenericLookupTable
+
+__all__ = ["ConfigModel", "DisplayConfig", "DisplayConfigData", "GenericLookupTable"]

src/daq_config_server/converters/_converters.py

@@ -0,0 +1,8 @@
+from ._base_model import ConfigModel
+from ._converter_utils import parse_value, remove_comments
+
+__all__ = [
+    "ConfigModel",
+    "remove_comments",
+    "parse_value",
+]

src/daq_config_server/models/__init__.py

@@ -0,0 +1,4 @@
+from pydantic import BaseModel
+
+
+class ConfigModel(BaseModel): ...

src/daq_config_server/models/converters/__init__.py

@@ -0,0 +1,25 @@
+import ast
+from typing import Any
+
+ALLOWED_BEAMLINE_PARAMETER_STRINGS = ["FB", "FULL", "deadtime"]
+
+
+class ConverterParseError(Exception): ...
+
+
+def remove_comments(lines: list[str]) -> list[str]:
+    return [
+        line.strip().split("#", 1)[0].strip()
+        for line in lines
+        if line.strip().split("#", 1)[0]
+    ]
+
+
+def parse_value(value: str, convert_to: type | None = None) -> Any:
+    """Convert a string value into an appropriate Python type. Optionally provide a type
+    to convert to. If not given, the type will be inferred.
+    """
+    value = ast.literal_eval(value.replace("Yes", "True").replace("No", "False"))
+    if convert_to:
+        value = convert_to(value)
+    return value

src/daq_config_server/models/converters/_base_model.py

@@ -3,15 +3,20 @@
 
 import xmltodict
 
-from daq_config_server.converters._converters import (
+from daq_config_server.models.converters.beamline_parameters import (
     beamline_parameters_to_dict,
+)
+from daq_config_server.models.converters.display_config import (
+    display_config_to_model,
+)
+from daq_config_server.models.converters.lookup_tables import (
     beamline_pitch_lut,
     beamline_roll_lut,
     detector_xy_lut,
-    display_config_to_model,
     undulator_energy_gap_lut,
 )
-from daq_config_server.converters.models import ConfigModel
+
+from ._base_model import ConfigModel
 
 FILE_TO_CONVERTER_MAP: dict[str, Callable[[str], ConfigModel | dict[str, Any]]] = {  # type: ignore
     "/tests/test_data/test_good_lut.txt": undulator_energy_gap_lut,  # For system tests # noqa