Request format of file contents using headers (#80)

* feat: get_config endpoint uses accept header to format response

* feat: Add OpenAPI schema to endpoint

---------

Co-authored-by: Joseph Ware <[email protected]>

Author: olliesilvester

Dockerfile

@@ -13,13 +13,13 @@ RUN python -m venv /venv
 ENV PATH=/venv/bin:$PATH
 
 # The build stage installs the context into the venv
-FROM developer as build
+FROM developer AS build
 COPY . /context
 WORKDIR /context
 RUN pip install .[server]
 
 # The runtime stage copies the built venv into a slim runtime container
-FROM python:${PYTHON_VERSION}-slim as runtime
+FROM python:${PYTHON_VERSION}-slim AS runtime
 RUN apt-get update && apt-get install -y --no-install-recommends \
     curl \
     procps \

pyproject.toml

@@ -57,10 +57,6 @@ name = "Oliver Silvester"
 [tool.setuptools_scm]
 version_file = "src/daq_config_server/_version.py"
 
-[tool.pyright]
-typeCheckingMode = "strict"
-reportMissingImports = false # Ignore missing stubs in imported modules
-
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "function"

pyrightconfig.json

@@ -0,0 +1,13 @@
+{
+  "typeCheckingMode": "strict",
+  "reportMissingImports": false,
+  "reportPrivateUsage": true,
+  "executionEnvironments": [
+    {
+      "root": "./tests",
+      "reportPrivateUsage": false
+    }
+  ],
+  "reportMissingTypeStubs": false,
+  "extraPaths": ["."]
+}

src/daq_config_server/app.py

@@ -1,10 +1,15 @@
+import json
+import os
+from enum import StrEnum
 from pathlib import Path
 
 import uvicorn
-from fastapi import FastAPI
+from fastapi import FastAPI, HTTPException, Request
 from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse, Response
 
 from daq_config_server.constants import ENDPOINTS
+from daq_config_server.log import LOGGER
 
 app = FastAPI(
     title="DAQ config server",
@@ -22,17 +27,79 @@
 __all__ = ["main"]
 
 
-@app.get(ENDPOINTS.CONFIG + "/{file_path:path}")
-def get_configuration(file_path: Path):
-    """Read a file and return its contents completely unformatted as a string. After
-    https://github.com/DiamondLightSource/daq-config-server/issues/67, this endpoint
-    will convert commonly read files to a dictionary format
+class ValidAcceptHeaders(StrEnum):
+    JSON = "application/json"
+    PLAIN_TEXT = "text/plain"
+    RAW_BYTES = "application/octet-stream"
+
+
+@app.get(
+    ENDPOINTS.CONFIG + "/{file_path:path}",
+    responses={
+        200: {
+            "description": "Returns JSON, plain text, or binary file.",
+            "content": {
+                "application/json": {
+                    "schema": {
+                        "type": "object",
+                        "additionalProperties": True,
+                        "example": {
+                            "key": "value",
+                            "list": [1, 2, 3],
+                            "nested": {"a": 1},
+                        },
+                    }
+                },
+                "text/plain": {
+                    "schema": {
+                        "type": "string",
+                        "example": "This is a plain text response",
+                    }
+                },
+                "application/octet-stream": {
+                    "schema": {"type": "string", "format": "binary"},
+                },
+            },
+        },
+    },
+)
+def get_configuration(
+    file_path: Path,
+    request: Request,
+):
+    """
+    Read a file and return its contents in a format specified by the accept header.
     """
     if not file_path.is_file():
-        raise FileNotFoundError(f"File {file_path} cannot be found")
+        raise HTTPException(status_code=404, detail=f"File {file_path} cannot be found")
+
+    file_name = os.path.basename(file_path)
+    accept = request.headers.get("accept", ValidAcceptHeaders.PLAIN_TEXT)
+
+    try:
+        match accept:
+            case ValidAcceptHeaders.JSON:
+                with file_path.open("r", encoding="utf-8") as f:
+                    content = json.loads(f.read())
+                return JSONResponse(
+                    content=content,
+                )
+            case ValidAcceptHeaders.PLAIN_TEXT:
+                with file_path.open("r", encoding="utf-8") as f:
+                    content = f.read()
+                return Response(content=content, media_type=accept)
+            case _:
+                pass
+    except Exception as e:
+        LOGGER.warning(
+            f"Failed to convert {file_name} to {accept} and caught \
+            exception: {e} \nSending file as raw bytes instead"
+        )
+
+    with file_path.open("rb") as f:
+        content = f.read()
 
-    with file_path.open("r", encoding="utf-8") as f:
-        return f.read()
+    return Response(content=content, media_type=ValidAcceptHeaders.RAW_BYTES)
 
 
 def main():

src/daq_config_server/client.py

@@ -1,13 +1,24 @@
 import operator
+from enum import StrEnum
 from logging import Logger, getLogger
+from pathlib import Path
 from typing import Any
 
 import requests
 from cachetools import TTLCache, cachedmethod
+from requests import Response
+
+from daq_config_server.app import ValidAcceptHeaders
 
 from .constants import ENDPOINTS
 
 
+class RequestedResponseFormats(StrEnum):
+    DICT = ValidAcceptHeaders.JSON  # Convert to dict using Response.json()
+    DECODED_STRING = ValidAcceptHeaders.PLAIN_TEXT  # Use utf-8 decoding in response
+    RAW_BYTE_STRING = ValidAcceptHeaders.RAW_BYTES  # Use raw bytes in response
+
+
 class ConfigServer:
     def __init__(
         self,
@@ -27,77 +38,105 @@ def __init__(
         """
         self._url = url.rstrip("/")
         self._log = log if log else getLogger("daq_config_server.client")
-        self._cache: TTLCache[tuple[str, str | None], str] = TTLCache(
+        self._cache: TTLCache[tuple[str, str, Path], str] = TTLCache(
             maxsize=cache_size, ttl=cache_lifetime_s
         )
 
-    def _get(
-        self,
-        endpoint: str,
-        item: str | None = None,
-        reset_cached_result: bool = False,
-    ) -> Any:
-        """
-        Get data from the config server with cache management.
-        If a cached response doesn't already exist, makes a request to
-        the config server.
-        If reset_cached_result is true, remove the cache entry for that request and
-        make a new request
-
-        Args:
-            endpoint: API endpoint.
-            item: Optional item identifier.
-            reset_cached_result: Whether to reset cache.
-
-        Returns:
-            The response data.
-        """
-
-        if (endpoint, item) in self._cache and reset_cached_result:
-            del self._cache[(endpoint, item)]
-        return self._cached_get(endpoint, item)
-
     @cachedmethod(cache=operator.attrgetter("_cache"))
     def _cached_get(
         self,
         endpoint: str,
-        item: str | None = None,
-    ) -> Any:
+        accept_header: str,
+        file_path: Path,
+    ) -> Response:
         """
         Get data from the config server and cache it.
 
         Args:
             endpoint: API endpoint.
-            item: Optional item identifier.
+            file_path: absolute path to the file which will be read
 
         Returns:
             The response data.
         """
-        url = self._url + endpoint + (f"/{item}" if item else "")
 
         try:
-            r = requests.get(url)
+            request_url = self._url + endpoint + (f"/{file_path}")
+            r = requests.get(request_url, headers={"Accept": accept_header})
             r.raise_for_status()
-            data = r.json()
-            self._log.debug(f"Cache set for {endpoint}/{item}.")
-            return data
+            self._log.debug(f"Cache set for {request_url}.")
+            return r
         except requests.exceptions.HTTPError as e:
             self._log.error(f"HTTP error: {e}")
             raise
 
-    def read_unformatted_file(
-        self, file_path: str, reset_cached_result: bool = False
+    def _get(
+        self,
+        endpoint: str,
+        accept_header: str,
+        file_path: Path,
+        reset_cached_result: bool = False,
+    ):
+        """
+        Get data from the config server with cache management and use
+        the content-type response header to format the return value.
+        If data parsing fails, return the response contents in bytes
+        """
+        if (endpoint, accept_header, file_path) in self._cache and reset_cached_result:
+            del self._cache[(endpoint, accept_header, file_path)]
+        r = self._cached_get(endpoint, accept_header, file_path)
+
+        content_type = r.headers["content-type"].split(";")[0].strip()
+
+        if content_type != accept_header:
+            self._log.warning(
+                f"Server failed to parse the file as requested. Requested \
+                {accept_header} but response came as content-type {content_type}"
+            )
+
+        try:
+            match content_type:
+                case ValidAcceptHeaders.JSON:
+                    content = r.json()
+                case ValidAcceptHeaders.PLAIN_TEXT:
+                    content = r.text
+                case _:
+                    content = r.content
+        except Exception as e:
+            self._log.warning(
+                f"Failed trying to convert to content-type {content_type} due to\
+                exception {e} \nReturning as bytes instead"
+            )
+            content = r.content
+
+        return content
+
+    def get_file_contents(
+        self,
+        file_path: Path,
+        requested_response_format: RequestedResponseFormats = (
+            RequestedResponseFormats.DECODED_STRING
+        ),
+        reset_cached_result: bool = False,
     ) -> Any:
         """
-        Read an unformatted file from the config server.
+        Get contents of a file from the config server in the format specified.
+        If data parsing fails, contents will return as raw bytes. Optionally look
+        for cached result before making request.
 
         Args:
             file_path: Path to the file.
-            reset_cached_result: Whether to reset cache.
-
+            requested_response_format: Specify how to parse the response.
+            reset_cached_result: If true, make a request and store response in cache,
+                                otherwise look for cached response before making
+                                new request
         Returns:
-            The file content.
+            The file contents, in the format specified.
         """
+
         return self._get(
-            ENDPOINTS.CONFIG, file_path, reset_cached_result=reset_cached_result
+            ENDPOINTS.CONFIG,
+            requested_response_format,
+            file_path,
+            reset_cached_result=reset_cached_result,
         )

src/daq_config_server/log.py

@@ -0,0 +1,22 @@
+import logging
+
+
+def get_default_logger(name: str = __name__) -> logging.Logger:
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.INFO)
+
+    # Prevent adding handlers multiple times
+    if not logger.handlers:
+        console_handler = logging.StreamHandler()
+        formatter = logging.Formatter(
+            "%(asctime)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
+        )
+        console_handler.setFormatter(formatter)
+        logger.addHandler(console_handler)
+
+    return logger
+
+
+# For now use a basic console-writing logger. Integrate properly with kubernetes in the
+# future
+LOGGER = get_default_logger("daq-config-server")

tests/constants.py

@@ -1,3 +1,9 @@
 from pathlib import Path
 
-TEST_DATA_DIR = Path(f"{Path(__file__).parent}/test_data")
+TEST_DATA_DIR_PATH = Path(f"{Path(__file__).parent}/test_data")
+
+TEST_BEAMLINE_PARAMETERS_PATH = TEST_DATA_DIR_PATH.joinpath("beamline_parameters.txt")
+
+TEST_BAD_JSON_PATH = TEST_DATA_DIR_PATH.joinpath("test_bad_json")
+
+TEST_GOOD_JSON_PATH = TEST_DATA_DIR_PATH.joinpath("test_good_json.json")