Add caching for client get

.github/workflows/_test.yml

@@ -50,8 +50,8 @@ jobs:
           python-version: ${{ inputs.python-version }}
           pip-install: ".[dev,server]"
 
-      - name: Run tests
-        run: tox -e tests
+      - name: Run unit tests
+        run: tox -e unit_tests
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4

Dockerfile

@@ -27,8 +27,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY --from=build /venv/ /venv/
 COPY tests/test_data/beamline_parameters.txt tests/test_data/beamline_parameters.txt
 ENV PATH=/venv/bin:$PATH
-ARG RUN_APP_IN_DEV_MODE=0
-ENV DEV_MODE=${RUN_APP_IN_DEV_MODE}
+ARG BEAMLINE="dev"
+ENV BEAMLINE=${BEAMLINE}
 
 # change this entrypoint if it is not the same as the repo
 CMD daq-config-server

README.md

@@ -35,6 +35,7 @@ use_stub_offsets: bool = config_server.best_effort_get_feature_flag("use_stub_of
 
 ## Testing and deployment
 
+
 There is a convenient script in `./deployment/build_and_push.sh`, which takes
 a `--dev` option to push containers with `-dev` appended to their names and a `--no-push` option for local
 development. This ensures that environment variables for dev or prod builds are included in the built container. To push to

deployment/build_and_push.sh

@@ -44,10 +44,8 @@ MAIN_CONTAINER_TAG="${BASE_REPO_ADDR}${MAIN_CONTAINER_NAME}"
 # set env vars which will be used by build process:
 if [ $DEV -gt 0 ]; then
     export REACT_APP_BACKEND_ADDR="http://localhost:8555"
-    export MAIN_APP_DEV_MODE=1
 else
     export REACT_APP_BACKEND_ADDR="https://daq-config.diamond.ac.uk/api"
-    export MAIN_APP_DEV_MODE=0
 fi
 
 echo " "
@@ -57,7 +55,7 @@ echo "========================================="
 echo " "
 echo "Building ${MAIN_CONTAINER_NAME}"
 echo " "
-podman build --build-arg RUN_APP_IN_DEV_MODE=$MAIN_APP_DEV_MODE -t $MAIN_CONTAINER_NAME .
+podman build --build-arg -t $MAIN_CONTAINER_NAME .
 if [ $PUSH -gt 0 ]; then
     podman tag $MAIN_CONTAINER_NAME $MAIN_CONTAINER_TAG
     podman push $MAIN_CONTAINER_NAME $MAIN_CONTAINER_TAG

pyproject.toml

@@ -17,7 +17,7 @@ readme = "README.md"
 requires-python = ">=3.11"
 
 [project.optional-dependencies]
-server = ["fastapi", "uvicorn", "redis", "hiredis"]
+server = ["fastapi", "uvicorn", "redis", "hiredis", "cachetools"]
 dev = [
     "copier",
     "httpx",
@@ -31,6 +31,9 @@ dev = [
     "ruff",
     "tox-direct",
     "types-mock",
+    "cachetools",
+    "fastapi",
+    "uvicorn",
 ]
 
 [project.scripts]
@@ -51,13 +54,14 @@ write_to = "src/daq_config_server/_version.py"
 reportMissingImports = false # Ignore missing stubs in imported modules
 
 [tool.pytest.ini_options]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
 # Run pytest with all our checkers, and don't spam us with massive tracebacks on error
 addopts = """
     --tb=native -vv
-    --asyncio-mode=auto
     """
 markers = """
-    uses_live_server: mark a test which uses the live config server
+    requires_local_server: mark a test which requires locally hosting the config server
 """
 
 # https://iscinumpy.gitlab.io/post/bound-version-constraints/#watch-for-warnings
@@ -79,18 +83,22 @@ legacy_tox_ini = """
 [tox]
 skipsdist=True
 
-[testenv:{pre-commit,type-checking,tests}]
+[testenv]
 # Don't create a virtualenv for the command, requires tox-direct plugin
 direct = True
 passenv = *
 allowlist_externals =
     pytest
     pre-commit
     pyright
+    sphinx-build
+    sphinx-autobuild
+[testenv:{pre-commit,type-checking,unit_tests,system_tests}]
 commands =
     pre-commit: pre-commit run --all-files --show-diff-on-failure {posargs}
     type-checking: pyright src tests {posargs}
-    tests: pytest -m "not uses_live_server" --cov=daq_config_server --cov-report term --cov-report xml:cov.xml {posargs}
+    unit_tests: pytest --cov=daq_config_server --cov-report term --cov-report xml:cov.xml {posargs} tests/unit_tests
+    system_tests: pytest tests/system_tests
 """
 
 [tool.ruff]

src/daq_config_server/__main__.py

@@ -2,34 +2,37 @@
 
 from . import __version__
 
-try:
-    import redis  # noqa
-    import uvicorn  # noqa
-    from fastapi import FastAPI  # noqa
+__all__ = ["main"]
 
-    server_dependencies_exist = True
-except ImportError:
-    server_dependencies_exist = False
+INSUFFICIENT_DEPENDENCIES_MESSAGE = "To do anything other than print the version and be\
+    available for importing the client, you must install this package with [server]\
+    optional dependencies"
 
 
-__all__ = ["main"]
+def check_server_dependencies():
+    try:
+        import uvicorn  # noqa
+        from fastapi import FastAPI  # noqa
+
+        server_dependencies_exist = True
+
+    except ImportError:
+        server_dependencies_exist = False
+
+    return server_dependencies_exist
 
 
 def main():
     parser = ArgumentParser()
     parser.add_argument("-v", "--version", action="version", version=__version__)
-    parser.add_argument("-d", "--dev", action="store_true")
-    args = parser.parse_args()
-    if not server_dependencies_exist:
-        print(
-            "To do anything other than print the version and be available for "
-            "importing the client, you must install this package with [server] "
-            "optional dependencies"
-        )
+    parser.parse_args()
+
+    if not check_server_dependencies():
+        print(INSUFFICIENT_DEPENDENCIES_MESSAGE)
     else:
         from .app import main
 
-        main(args)
+        main()
 
 
 # test with: python -m daq_config_server

src/daq_config_server/app.py

@@ -1,31 +1,14 @@
-from os import environ
+from pathlib import Path
 
 import uvicorn
-from fastapi import FastAPI, Request, Response, status
+from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
-from pydantic import BaseModel
-from redis import Redis
 
-from .beamline_parameters import (
-    BEAMLINE_PARAMETER_PATHS,
-    GDABeamlineParameters,
-)
-from .constants import DATABASE_KEYS, ENDPOINTS
-
-DEV_MODE = bool(int(environ.get("DEV_MODE") or 0))
-
-ROOT_PATH = "/api"
-print(f"{DEV_MODE=}")
-print(f"{ROOT_PATH=}")
-if DEV_MODE:
-    print("Running in dev mode! not setting root path!")
-    ROOT_PATH = ""
+from daq_config_server.constants import ENDPOINTS
 
 app = FastAPI(
     title="DAQ config server",
-    description="""For storing and fetching beamline parameters, etc. which are needed
-    by more than one applicatioon or service""",
-    root_path=ROOT_PATH,
+    description="""For reading files stored on /dls_sw from another container""",
 )
 origins = ["*"]
 app.add_middleware(
@@ -36,124 +19,21 @@
     allow_headers=["*"],
 )
 
-valkey = Redis(host="localhost", port=6379, decode_responses=True)
-
 __all__ = ["main"]
 
-BEAMLINE_PARAM_PATH = ""
-BEAMLINE_PARAMS: GDABeamlineParameters | None = None
-
-
-@app.get(ENDPOINTS.BL_PARAM + "/{param}")
-def get_beamline_parameter(param: str):
-    """Get a single beamline parameter"""
-    assert BEAMLINE_PARAMS is not None
-    return {param: BEAMLINE_PARAMS.params.get(param)}
-
-
-class ParamList(BaseModel):
-    param_list: list[str]
-
-
-@app.get(ENDPOINTS.BL_PARAM)
-def get_all_beamline_parameters(param_list_data: ParamList | None = None):
-    """Get a dict of all the current beamline parameters."""
-    assert BEAMLINE_PARAMS is not None
-    if param_list_data is None:
-        return BEAMLINE_PARAMS.params
-    return {k: BEAMLINE_PARAMS.params.get(k) for k in param_list_data.param_list}
-
-
-@app.get(ENDPOINTS.FEATURE)
-def get_feature_flag_list(get_values: bool = False):
-    """Get a list of all the current feature flags, or a dict of all the current values
-    if get_values=true is passed"""
-    flags = valkey.smembers(DATABASE_KEYS.FEATURE_SET)
-    if not get_values:
-        return flags
-    else:
-        return {flag: bool(int(valkey.get(flag))) for flag in flags}  # type: ignore
-
-
-@app.get(ENDPOINTS.FEATURE + "/{flag_name}")
-def get_feature_flag(flag_name: str, response: Response):
-    """Get the value of a feature flag"""
-    if not valkey.sismember(DATABASE_KEYS.FEATURE_SET, flag_name):
-        response.status_code = status.HTTP_404_NOT_FOUND
-        return {"message": f"Feature flag {flag_name} does not exist!"}
-    else:
-        ret = int(valkey.get(flag_name))  # type: ignore # We checked if it exists above
-        return {flag_name: bool(ret)}
-
-
-@app.post(ENDPOINTS.FEATURE + "/{flag_name}", status_code=status.HTTP_201_CREATED)
-def create_feature_flag(flag_name: str, response: Response, value: bool = False):
-    """Sets a feature flag, creating it if it doesn't exist. Default to False."""
-    if valkey.sismember(DATABASE_KEYS.FEATURE_SET, flag_name):
-        response.status_code = status.HTTP_409_CONFLICT
-        return {"message": f"Feature flag {flag_name} already exists!"}
-    else:
-        valkey.sadd(DATABASE_KEYS.FEATURE_SET, flag_name)
-        return {"success": valkey.set(flag_name, int(value))}
-
-
-@app.put(ENDPOINTS.FEATURE + "/{flag_name}")
-def set_feature_flag(flag_name: str, value: bool, response: Response):
-    """Sets a feature flag, return an error if it doesn't exist."""
-    if not valkey.sismember(DATABASE_KEYS.FEATURE_SET, flag_name):
-        response.status_code = status.HTTP_404_NOT_FOUND
-        return {"message": f"Feature flag {flag_name} does not exist!"}
-    else:
-        return {"success": valkey.set(flag_name, int(value))}
-
-
-@app.delete(ENDPOINTS.FEATURE + "/{flag_name}")
-def delete_feature_flag(flag_name: str, response: Response):
-    """Delete a feature flag."""
-    if not valkey.sismember(DATABASE_KEYS.FEATURE_SET, flag_name):
-        response.status_code = status.HTTP_404_NOT_FOUND
-        return {"message": f"Feature flag {flag_name} does not exist!"}
-    else:
-        valkey.srem(DATABASE_KEYS.FEATURE_SET, flag_name)
-        return {"success": not valkey.sismember(DATABASE_KEYS.FEATURE_SET, flag_name)}
-
-
-@app.get(ENDPOINTS.INFO)
-def get_info(request: Request):
-    """Get some generic information about the request, mostly for debugging"""
-    return {
-        "message": "Welcome to daq-config API.",
-        "root_path": request.scope.get("root_path"),
-        "request_headers": request.headers,
-    }
-
-
-if DEV_MODE:
-
-    @app.api_route("/{full_path:path}")
-    async def catch_all(request: Request, full_path: str):
-        return {
-            "message": "resource not found, supplying info for debug",
-            "root_path": request.scope.get("root_path"),
-            "path": full_path,
-            "request_headers": repr(request.headers),
-        }
-
-
-def _load_beamline_params():
-    global BEAMLINE_PARAMS
-    BEAMLINE_PARAMS = GDABeamlineParameters.from_file(BEAMLINE_PARAM_PATH)
 
+@app.get(ENDPOINTS.CONFIG + "/{file_path:path}")
+def get_configuration(file_path: Path):
+    """Read a file and return its contents completely unformatted as a string. After
+    https://github.com/DiamondLightSource/daq-config-server/issues/67, this endpoint
+    will convert commonly read files to a dictionary format
+    """
+    if not file_path.is_file():
+        raise FileNotFoundError(f"File {file_path} cannot be found")
 
-def _set_beamline_param_path(dev: bool = True):
-    global BEAMLINE_PARAM_PATH
-    if dev:
-        BEAMLINE_PARAM_PATH = "tests/test_data/beamline_parameters.txt"
-    else:
-        BEAMLINE_PARAM_PATH = BEAMLINE_PARAMETER_PATHS["i03"]
+    with file_path.open("r", encoding="utf-8") as f:
+        return f.read()
 
 
-def main(args):
-    _set_beamline_param_path(args.dev)
-    _load_beamline_params()
+def main():
     uvicorn.run(app="daq_config_server.app:app", host="0.0.0.0", port=8555)