Skip to content

Commit 3171fa0

Browse files
codelioncodelion
committed
add tests
1 parent d3a6bbe commit 3171fa0

15 files changed

+1555
-156
lines changed

‎.github/workflows/python-test.yml‎

Lines changed: 86 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,93 @@
1-
name: Python Unit Tests
1+
name: Tests
22

33
on: [push, pull_request]
44

55
jobs:
6-
test:
6+
unit-tests:
77
runs-on: ubuntu-latest
8+
steps:
9+
- name: Checkout code
10+
uses: actions/checkout@v3
11+
12+
- name: Set up Python
13+
uses: actions/setup-python@v4
14+
with:
15+
python-version: '3.9'
16+
17+
- name: Cache pip packages
18+
uses: actions/cache@v3
19+
with:
20+
path: ~/.cache/pip
21+
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
22+
restore-keys: |
23+
${{ runner.os }}-pip-
824
25+
- name: Install dependencies
26+
run: |
27+
python -m pip install --upgrade pip
28+
pip install -e .
29+
pip install pytest numpy
30+
31+
- name: Run unit tests
32+
env:
33+
OPENAI_API_KEY: test # Mock API key for unit tests
34+
run: |
35+
# Run unit tests (all tests except integration/)
36+
python -m unittest discover -s tests -p "test_*.py" -v
37+
38+
integration-tests:
39+
needs: unit-tests # Only run if unit tests pass
40+
runs-on: ubuntu-latest
941
steps:
10-
- name: Checkout code
11-
uses: actions/checkout@v3
12-
13-
- name: Set up Python
14-
uses: actions/setup-python@v4
15-
with:
16-
python-version: '3.9'
17-
18-
- name: Install dependencies
19-
run: |
20-
python -m pip install --upgrade pip
21-
pip install -e .
22-
# Install test dependencies
23-
pip install pytest numpy
24-
25-
- name: Run unit tests
26-
env:
27-
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
28-
run: |
29-
python -m unittest discover -s tests -p "test_*.py" -v
42+
- name: Checkout code
43+
uses: actions/checkout@v3
44+
45+
- name: Set up Python
46+
uses: actions/setup-python@v4
47+
with:
48+
python-version: '3.9'
49+
50+
- name: Cache pip packages
51+
uses: actions/cache@v3
52+
with:
53+
path: ~/.cache/pip
54+
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
55+
restore-keys: |
56+
${{ runner.os }}-pip-
57+
58+
- name: Install dependencies
59+
run: |
60+
python -m pip install --upgrade pip
61+
pip install -e .
62+
pip install pytest numpy optillm
63+
64+
- name: Start optillm server
65+
run: |
66+
echo "Starting optillm server for integration tests..."
67+
OPTILLM_API_KEY=optillm optillm --model google/gemma-3-270m-it --port 8000 &
68+
echo $! > server.pid
69+
70+
# Wait for server to be ready
71+
echo "Waiting for server to start..."
72+
sleep 15
73+
74+
# Test server health
75+
curl -s http://localhost:8000/health || echo "Server health check failed"
76+
env:
77+
OPTILLM_API_KEY: optillm
78+
79+
- name: Run integration tests
80+
env:
81+
OPENAI_API_KEY: optillm
82+
OPTILLM_API_KEY: optillm
83+
run: |
84+
pytest tests/integration -v --tb=short
85+
86+
- name: Stop optillm server
87+
if: always()
88+
run: |
89+
if [ -f server.pid ]; then
90+
kill $(cat server.pid) || true
91+
rm server.pid
92+
fi
93+
pkill -f "optillm.*8000" || true

‎Makefile‎

Lines changed: 52 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,18 @@ PIP := $(VENV_DIR)/bin/pip
99
.PHONY: help
1010
help:
1111
@echo "Available targets:"
12-
@echo " all - Install dependencies and run tests"
13-
@echo " venv - Create a virtual environment"
14-
@echo " install - Install Python dependencies"
15-
@echo " lint - Run Black code formatting"
16-
@echo " test - Run tests"
17-
@echo " docker-build - Build the Docker image"
18-
@echo " docker-run - Run the Docker container with the example"
19-
@echo " visualizer - Run the visualization script"
12+
@echo " all - Install dependencies and run unit tests"
13+
@echo " venv - Create a virtual environment"
14+
@echo " install - Install Python dependencies"
15+
@echo " install-dev - Install development dependencies including optillm"
16+
@echo " lint - Run Black code formatting"
17+
@echo " test - Run unit tests only"
18+
@echo " test-unit - Run unit tests only (same as test)"
19+
@echo " test-integration - Run integration tests with local LLM"
20+
@echo " test-all - Run both unit and integration tests"
21+
@echo " docker-build - Build the Docker image"
22+
@echo " docker-run - Run the Docker container with the example"
23+
@echo " visualizer - Run the visualization script"
2024

2125
.PHONY: all
2226
all: install test
@@ -31,16 +35,55 @@ venv:
3135
install: venv
3236
$(PIP) install -e .
3337

38+
# Install development dependencies including optillm for integration tests
39+
.PHONY: install-dev
40+
install-dev: venv
41+
$(PIP) install -e .
42+
$(PIP) install pytest optillm
43+
3444
# Run Black code formatting
3545
.PHONY: lint
3646
lint: venv
3747
$(PYTHON) -m black openevolve examples tests scripts
3848

39-
# Run tests using the virtual environment
49+
# Run unit tests only (fast, no LLM required)
4050
.PHONY: test
4151
test: venv
4252
$(PYTHON) -m unittest discover -s tests -p "test_*.py"
4353

54+
# Alias for test
55+
.PHONY: test-unit
56+
test-unit: test
57+
58+
# Run integration tests with local LLM (requires optillm)
59+
.PHONY: test-integration
60+
test-integration: install-dev
61+
@echo "Starting optillm server for integration tests..."
62+
@OPTILLM_API_KEY=optillm $(VENV_DIR)/bin/optillm --model google/gemma-3-270m-it --port 8000 &
63+
@OPTILLM_PID=$$! && \
64+
echo $$OPTILLM_PID > /tmp/optillm.pid && \
65+
echo "Waiting for optillm server to start..." && \
66+
sleep 10 && \
67+
echo "Running integration tests..." && \
68+
OPENAI_API_KEY=optillm $(PYTHON) -m pytest tests/integration -v --tb=short; \
69+
TEST_EXIT_CODE=$$?; \
70+
echo "Stopping optillm server..."; \
71+
kill $$OPTILLM_PID 2>/dev/null || true; \
72+
pkill -f "optillm.*8000" 2>/dev/null || true; \
73+
rm -f /tmp/optillm.pid; \
74+
exit $$TEST_EXIT_CODE
75+
76+
# Run integration tests with existing optillm server (for development)
77+
.PHONY: test-integration-dev
78+
test-integration-dev: venv
79+
@echo "Using existing optillm server at localhost:8000"
80+
@curl -s http://localhost:8000/health > /dev/null || (echo "Error: optillm server not running at localhost:8000" && exit 1)
81+
OPENAI_API_KEY=optillm $(PYTHON) -m pytest tests/integration -v
82+
83+
# Run all tests (unit first, then integration)
84+
.PHONY: test-all
85+
test-all: test test-integration
86+
4487
# Build the Docker image
4588
.PHONY: docker-build
4689
docker-build:

‎tests/integration/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# Integration tests directory

‎tests/integration/conftest.py‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
"""
2+
Pytest fixtures for integration tests with optillm server
3+
"""
4+
5+
import pytest
6+
import subprocess
7+
import time
8+
import os
9+
import tempfile
10+
import shutil
11+
from pathlib import Path
12+
13+
# Import our test utilities
14+
import sys
15+
sys.path.append(str(Path(__file__).parent.parent))
16+
from test_utils import (
17+
start_test_server,
18+
stop_test_server,
19+
is_server_running,
20+
get_integration_config,
21+
get_evolution_test_program,
22+
get_evolution_test_evaluator
23+
)
24+
25+
26+
@pytest.fixture(scope="session")
27+
def optillm_server():
28+
"""Start optillm server for the test session"""
29+
# Check if server is already running (for development)
30+
if is_server_running():
31+
print("Using existing optillm server at localhost:8000")
32+
yield None # Server already running, don't manage it
33+
return
34+
35+
print("Starting optillm server for integration tests...")
36+
proc = None
37+
try:
38+
proc = start_test_server()
39+
print("optillm server started successfully")
40+
yield proc
41+
except Exception as e:
42+
print(f"Failed to start optillm server: {e}")
43+
raise
44+
finally:
45+
if proc:
46+
print("Stopping optillm server...")
47+
stop_test_server(proc)
48+
print("optillm server stopped")
49+
50+
51+
@pytest.fixture
52+
def evolution_config(optillm_server):
53+
"""Get config for evolution tests"""
54+
return get_integration_config()
55+
56+
57+
@pytest.fixture
58+
def temp_workspace():
59+
"""Create a temporary workspace for test files"""
60+
temp_dir = tempfile.mkdtemp()
61+
yield Path(temp_dir)
62+
shutil.rmtree(temp_dir, ignore_errors=True)
63+
64+
65+
@pytest.fixture
66+
def test_program_file(temp_workspace):
67+
"""Create a test program file"""
68+
program_file = temp_workspace / "test_program.py"
69+
program_file.write_text(get_evolution_test_program())
70+
return program_file
71+
72+
73+
@pytest.fixture
74+
def test_evaluator_file(temp_workspace):
75+
"""Create a test evaluator file"""
76+
evaluator_file = temp_workspace / "evaluator.py"
77+
evaluator_file.write_text(get_evolution_test_evaluator())
78+
return evaluator_file
79+
80+
81+
@pytest.fixture
82+
def evolution_output_dir(temp_workspace):
83+
"""Create output directory for evolution tests"""
84+
output_dir = temp_workspace / "output"
85+
output_dir.mkdir()
86+
return output_dir

0 commit comments

Comments
 (0)