Run system tests in CI

Author: jacob720

.github/workflows/_test_system.yml

@@ -0,0 +1,57 @@
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        type: string
+        description: The version of python to install
+        required: true
+      runs-on:
+        type: string
+        description: The runner to run this job on
+        required: true
+    secrets:
+      CODECOV_TOKEN:
+        required: true
+
+env:
+  # https://github.com/pytest-dev/pytest/issues/2042
+  PY_IGNORE_IMPORTMISMATCH: "1"
+
+jobs:
+  run:
+    runs-on: ${{ inputs.runs-on }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # Need this to get version number from last tag
+          fetch-depth: 0
+
+      - if: inputs.python-version == 'dev'
+        name: Install dev versions of python packages
+        uses: ./.github/actions/install_requirements
+
+      - if: inputs.python-version == 'dev'
+        name: Write the requirements as an artifact
+        run: pip freeze --exclude-editable > /tmp/dev-requirements.txt
+
+      - if: inputs.python-version == 'dev'
+        name: Upload dev-requirements.txt
+        uses: actions/upload-artifact@v4
+        with:
+          name: dev-requirements
+          path: /tmp/dev-requirements.txt
+
+      - if: inputs.python-version != 'dev'
+        name: Install latest versions of python packages
+        uses: ./.github/actions/install_requirements
+        with:
+          python-version: ${{ inputs.python-version }}
+          pip-install: ".[dev,server]"
+
+      - name: Deploy local server
+        run: ./deployment/build_and_push.sh -r -b
+
+      - name: Run system tests
+        run: tox -e system_tests_local

.github/workflows/ci.yml

@@ -34,6 +34,25 @@ jobs:
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
+  test_system:
+    needs: check
+    if: needs.check.outputs.branch-pr == ''
+    strategy:
+      matrix:
+        runs-on: ["ubuntu-latest"] # can add windows-latest, macos-latest
+        python-version: ["3.11", "3.12", "3.13"]
+        include:
+          # Include one that runs in the dev environment
+          - runs-on: "ubuntu-latest"
+            python-version: "dev"
+      fail-fast: false
+    uses: ./.github/workflows/_test_system.yml
+    with:
+      runs-on: ${{ matrix.runs-on }}
+      python-version: ${{ matrix.python-version }}
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
   container:
     needs: check
     if: needs.check.outputs.branch-pr == ''