Add system test CI job, add tests which talk to central service (#138)

* Let some system tests talk to deployed server

* Add tox commands

* Run system tests in CI

* Adjust docs

Author: jacob720

.github/workflows/_test_system.yml

@@ -0,0 +1,46 @@
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        type: string
+        description: The version of python to install
+        required: true
+      runs-on:
+        type: string
+        description: The runner to run this job on
+        required: true
+    secrets:
+      CODECOV_TOKEN:
+        required: true
+
+env:
+  # https://github.com/pytest-dev/pytest/issues/2042
+  PY_IGNORE_IMPORTMISMATCH: "1"
+
+jobs:
+  run:
+    runs-on: ${{ inputs.runs-on }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # Need this to get version number from last tag
+          fetch-depth: 0
+
+      - if: inputs.python-version == 'dev'
+        name: Install dev versions of python packages
+        uses: ./.github/actions/install_requirements
+
+      - if: inputs.python-version != 'dev'
+        name: Install latest versions of python packages
+        uses: ./.github/actions/install_requirements
+        with:
+          python-version: ${{ inputs.python-version }}
+          pip-install: ".[dev,server]"
+
+      - name: Deploy local server
+        run: ./deployment/build_and_push.sh -r -b
+
+      - name: Run system tests
+        run: tox -e system_tests_local

.github/workflows/ci.yml

@@ -34,6 +34,25 @@ jobs:
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
+  test_system:
+    needs: check
+    if: needs.check.outputs.branch-pr == ''
+    strategy:
+      matrix:
+        runs-on: ["ubuntu-latest"] # can add windows-latest, macos-latest
+        python-version: ["3.11", "3.12", "3.13"]
+        include:
+          # Include one that runs in the dev environment
+          - runs-on: "ubuntu-latest"
+            python-version: "dev"
+      fail-fast: false
+    uses: ./.github/workflows/_test_system.yml
+    with:
+      runs-on: ${{ matrix.runs-on }}
+      python-version: ${{ matrix.python-version }}
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
   container:
     needs: check
     if: needs.check.outputs.branch-pr == ''

README.md

@@ -33,7 +33,7 @@ To run unit tests, type `tox -e unit_tests` from within the dev container
 
 There is a convenient script in `./deployment/build_and_push.sh` which can be used to easily build and run the container locally for testing, and optionally push the container to ghcr. In general we should rely on the CI to be pushing new containers - it should only be done manually for urgent debugging.
 
-To run system tests, start a local container by running `./deployment/build_and_push.sh -r -b`. Then, in the dev container, forward port 8555. There are instructions on port forwarding within vscode [here](https://code.visualstudio.com/docs/debugtest/port-forwarding). Next, in a terminal in the devcontainer, run `pytest .` from the `daq-config-server` directory.
+To run local system tests, start a local container by running `./deployment/build_and_push.sh -r -b`. Then, in the dev container, forward port 8555. There are instructions on port forwarding within vscode [here](https://code.visualstudio.com/docs/debugtest/port-forwarding). Next, in a terminal in the devcontainer, run `tox -e system_tests_local` from the `daq-config-server` directory.
 
 To test on argus, log in to argus in your namespace and run:
 

pyproject.toml

@@ -75,6 +75,7 @@ addopts = """
     """
 markers = """
     requires_local_server: mark a test which requires locally hosting the config server
+    requires_deployed_server: mark a test which requires talking to the deployed config server
     use_threading: mark a test which will kickoff the background thread in WhitelistFetcher
 """
 
@@ -101,7 +102,7 @@ legacy_tox_ini = """
 [tox]
 skipsdist=True
 
-[testenv:{pre-commit,type-checking,unit_tests,docs,system_tests}]
+[testenv:{pre-commit,type-checking,unit_tests,docs,system_tests, system_tests_local, system_tests_deployed}]
 # Don't create a virtualenv for the command, requires tox-direct plugin
 direct = True
 passenv = *
@@ -117,6 +118,8 @@ commands =
     unit_tests: pytest --cov=daq_config_server --cov-report term --cov-report xml:cov.xml {posargs} tests/unit_tests
     docs: sphinx-{posargs:build -EW --keep-going} -T docs build/html
     system_tests: pytest tests/system_tests
+    system_tests_local: pytest tests/system_tests -m requires_local_server
+    system_tests_deployed: pytest tests/system_tests -m requires_deployed_server
 """
 
 [tool.ruff]

tests/system_tests/test_client.py

@@ -20,6 +20,7 @@
 )
 
 SERVER_ADDRESS = "http://0.0.0.0:8555"
+DEPLOYED_SERVER_ADDRESS = "https://daq-config.diamond.ac.uk"
 
 # Docs for running these system tests will be added in https://github.com/DiamondLightSource/daq-config-server/issues/68
 
@@ -29,6 +30,11 @@ def server():
     return ConfigServer(SERVER_ADDRESS)
 
 
+@pytest.fixture
+def deployed_server():
+    return ConfigServer(DEPLOYED_SERVER_ADDRESS)
+
+
 @pytest.mark.requires_local_server
 def test_read_unformatted_file_as_plain_text(server: ConfigServer):
     with open(TestDataPaths.TEST_BEAMLINE_PARAMETERS_PATH) as f:
@@ -144,23 +150,29 @@ def test_request_for_file_with_converter_with_wrong_pydantic_model_errors(
         server.get_file_contents(ServerFilePaths.GOOD_LUT, DisplayConfig)
 
 
-@pytest.mark.requires_local_server
-def test_all_files_in_file_converter_map_can_be_converted_to_dict(server: ConfigServer):
+@pytest.mark.requires_deployed_server
+def test_all_files_in_file_converter_map_can_be_converted_to_dict(
+    deployed_server: ConfigServer,
+):
     for filename in file_converter_map.FILE_TO_CONVERTER_MAP.keys():
-        result = server.get_file_contents(filename, dict)
+        if filename.startswith("/tests/test_data/"):
+            continue
+        result = deployed_server.get_file_contents(filename, dict)
         assert isinstance(result, dict)
 
 
-@pytest.mark.requires_local_server
+@pytest.mark.requires_deployed_server
 def test_all_files_in_file_converter_map_can_be_converted_to_target_type(
-    server: ConfigServer,
+    deployed_server: ConfigServer,
 ):
     with patch(
         "daq_config_server.converters._file_converter_map.xmltodict.parse.__annotations__",
         {"return": dict},  # Force a return type for xmltodict.parse()
     ):
         for filename, converter in file_converter_map.FILE_TO_CONVERTER_MAP.items():
+            if filename.startswith("/tests/test_data/"):
+                continue
             return_type = get_type_hints(converter)["return"]
             assert return_type is dict or issubclass(return_type, ConfigModel)
-            result = server.get_file_contents(filename, return_type)
+            result = deployed_server.get_file_contents(filename, return_type)
             assert isinstance(result, return_type)