Skip to content
Test

WIP: CI Tests using docker image #239

Sign in to view logs

WIP: CI Tests using docker image

WIP: CI Tests using docker image #239

Workflow file for this run

.github/workflows/test.yml at 152aeb7
name: Test
on:
pull_request:
branches:
- "**"
paths:
# Run for changes to *this* workflow file, but not for other workflows
- ".github/workflows/test.yml"
# Trigger off all top level files by default
- "*"
# Trigger off source, test, and ci changes
# (API schema changes only matter when the generated data model code changes)
- "src/**"
- "tests/**"
- "ci/**"
# Python scripts under misc still need linting & typechecks
- "misc/**.py"
# Skip running the source code checks when only documentation has been updated
- "!**.md"
- "!**.rst"
- "!**.txt" # Any requirements file changes will also involve changing other files
push:
branches:
- main
# Require explicit job permissions
permissions: {}
defaults:
run:
# Use the Git for Windows bash shell, rather than supporting Powershell
# This also implies `set -eo pipefail` (rather than just `set -e`)
shell: bash
jobs:
tests-windows:
runs-on: windows-2022
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13"]
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Get pip cache dir
id: pip-cache
run: |
echo "dir=$(python -m pip cache dir)" >> $GITHUB_OUTPUT
- name: Cache bootstrapping dependencies
uses: actions/cache@v4
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: pip-windows-2022-${{ matrix.python-version }}-v1-${{ hashFiles('pdm.lock') }}
restore-keys: |
pip-windows-2022-${{ matrix.python-version }}-v1-
- name: Install PDM
run: |
python -m pip install --upgrade -r ci-bootstrap-requirements.txt
- name: Create development virtual environment
run: |
python -m pdm sync --no-self --dev
echo "VIRTUAL_ENV_BIN_DIR=$PWD/.venv/Scripts" >> "$GITHUB_ENV"
- name: CI-compatible tests (Windows)
run: |
source "$VIRTUAL_ENV_BIN_DIR/activate"
python -m tox -v -- -m 'not lmstudio'
- name: Upload coverage data
uses: actions/upload-artifact@v4
with:
name: coverage-data-windows-py${{ matrix.python-version }}
path: .coverage.*
include-hidden-files: true
if-no-files-found: ignore
tests-docker:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13"]
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Get pip cache dir
id: pip-cache
run: |
echo "dir=$(python -m pip cache dir)" >> $GITHUB_OUTPUT
- name: Cache bootstrapping dependencies
uses: actions/cache@v4
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: pip-ubuntu-22.04-${{ matrix.python-version }}-v1-${{ hashFiles('pdm.lock') }}
restore-keys: |
pip-ubuntu-22.04-${{ matrix.python-version }}-v1-
- name: Run the built image
id: run
uses: ./.github/actions/docker-daemon-run
with:
docker-image: lmstudio/llmster-preview:cpu
port: "41343:1234"
container-name: llmster
- name: Download models for tests
run: |
echo "Downloading required models..."
# Download text LLMs
docker exec llmster lms get https://huggingface.co/hugging-quants/Llama-3.2-1B-Instruct-Q4_K_M-GGUF -y --quiet
docker exec llmster lms get https://huggingface.co/Qwen/Qwen2.5-7B-Instruct-GGUF@q3_k_m -y --quiet
docker exec llmster lms get https://huggingface.co/ZiangWu/MobileVLM_V2-1.7B-GGUF -y --quiet
# Download additional model for speculative decoding examples
docker exec llmster lms get https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct-GGUF -y --quiet
echo "Model downloads complete"
- name: Load models into LM Studio
run: |
echo "Loading models..."
# Load embedding model
docker exec llmster lms load nomic-embed-text-v1.5 --identifier text-embedding-nomic-embed-text-v1.5 -y
# Load text LLMs
docker exec llmster lms load llama-3.2-1b-instruct --identifier llama-3.2-1b-instruct -y
docker exec llmster lms load qwen2.5-7b-instruct --identifier qwen2.5-7b-instruct-1m -y
# Load vision LLM
docker exec llmster lms load ZiangWu/MobileVLM_V2-1.7B-GGUF --identifier mobilevlm_v2-1.7b
echo "Model loading complete"
- name: Install PDM
run: |
python -m pip install --upgrade -r ci-bootstrap-requirements.txt
- name: Create development virtual environment
run: |
python -m pdm sync --no-self --dev
echo "VIRTUAL_ENV_BIN_DIR=$PWD/.venv/bin" >> "$GITHUB_ENV"
- name: All tests including LM Studio
run: |
source "$VIRTUAL_ENV_BIN_DIR/activate"
python -m tox -v
- name: Upload coverage data
uses: actions/upload-artifact@v4
with:
name: coverage-data-docker-py${{ matrix.python-version }}
path: .coverage.*
include-hidden-files: true
if-no-files-found: ignore
- name: Stop LM Studio Docker container
if: always()
run: |
docker stop llmster || true
docker rm llmster || true
# Coverage check for Docker tests (higher target since they include LM Studio tests)
coverage-docker:
name: Docker test coverage
if: always()
needs: tests-docker
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/setup-python@v5
with:
python-version: "3.13"
- uses: hynek/setup-cached-uv@757bedc3f972eb7227a1aa657651f15a8527c817
- uses: actions/download-artifact@v4
with:
pattern: coverage-data-docker-*
merge-multiple: true
- name: Combine coverage & fail if it goes down
run: |
uv tool install 'coverage[toml]'
coverage combine
coverage html --skip-covered --skip-empty
# Report and write to summary.
coverage report --format=markdown >> $GITHUB_STEP_SUMMARY
# Higher target for Docker tests since they include LM Studio functionality
coverage report --fail-under=75
- name: Upload HTML report if check failed
uses: actions/upload-artifact@v4
with:
name: html-report-docker
path: htmlcov
if: ${{ failure() }}
# Coverage check for Windows tests (lower target since they skip LM Studio tests)
coverage-windows:
name: Windows test coverage
if: always()
needs: tests-windows
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/setup-python@v5
with:
python-version: "3.13"
- uses: hynek/setup-cached-uv@757bedc3f972eb7227a1aa657651f15a8527c817
- uses: actions/download-artifact@v4
with:
pattern: coverage-data-windows-*
merge-multiple: true
- name: Combine coverage & fail if it goes down
run: |
uv tool install 'coverage[toml]'
coverage combine
coverage html --skip-covered --skip-empty
# Report and write to summary.
coverage report --format=markdown >> $GITHUB_STEP_SUMMARY
# Report again and fail if under 50%.
# Highest historical coverage: 65%
# Last noted local test coverage level: 94%
# Lower target for Windows tests since they skip LM Studio functionality
coverage report --fail-under=50
- name: Upload HTML report if check failed
uses: actions/upload-artifact@v4
with:
name: html-report-windows
path: htmlcov
if: ${{ failure() }}