Skip to content

Commit 1ad2a1d

Browse files
tom-sapletta-comtom-sapletta-com
committed
examples
1 parent 58b5533 commit 1ad2a1d

File tree

3 files changed

+775
-0
lines changed

3 files changed

+775
-0
lines changed

‎.github/workflows/comprehensive-tests.yml‎

Lines changed: 365 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,365 @@
1+
#!/usr/bin/env python3
2+
"""GitHub Actions workflow for comprehensive testing."""
3+
4+
name: Comprehensive Tests
5+
6+
on:
7+
push:
8+
branches: [ main, develop ]
9+
pull_request:
10+
branches: [ main, develop ]
11+
schedule:
12+
# Run tests weekly on Sundays at 2 AM UTC
13+
- cron: '0 2 * * 0'
14+
15+
jobs:
16+
test-matrix:
17+
runs-on: ${{ matrix.os }}
18+
strategy:
19+
fail-fast: false
20+
matrix:
21+
os: [ubuntu-latest, windows-latest, macos-latest]
22+
python-version: ['3.10', '3.11', '3.12', '3.13']
23+
24+
steps:
25+
- uses: actions/checkout@v4
26+
27+
- name: Set up Python ${{ matrix.python-version }}
28+
uses: actions/setup-python@v4
29+
with:
30+
python-version: ${{ matrix.python-version }}
31+
32+
- name: Install dependencies
33+
run: |
34+
python -m pip install --upgrade pip
35+
pip install -e .[dev,all]
36+
37+
- name: Run unit tests
38+
run: |
39+
pytest tests/ -v --cov=vallm --cov-report=xml --cov-report=html
40+
41+
- name: Run E2E CLI tests
42+
run: |
43+
pytest tests/test_cli_e2e.py -v
44+
45+
- name: Run installation tests
46+
run: |
47+
pytest tests/test_installation.py -v
48+
49+
- name: Test semantic validation (if available)
50+
run: |
51+
pytest tests/test_semantic_validation.py -v
52+
continue-on-error: true # May fail without LLM setup
53+
54+
- name: Upload coverage to Codecov
55+
uses: codecov/codecov-action@v3
56+
with:
57+
file: ./coverage.xml
58+
flags: unittests
59+
name: codecov-umbrella
60+
61+
docker-tests:
62+
runs-on: ubuntu-latest
63+
needs: test-matrix
64+
65+
steps:
66+
- uses: actions/checkout@v4
67+
68+
- name: Set up Docker Buildx
69+
uses: docker/setup-buildx-action@v3
70+
71+
- name: Test Docker installation across systems
72+
run: |
73+
chmod +x scripts/test_docker_installation.sh
74+
./scripts/test_docker_installation.sh
75+
76+
- name: Build and test Docker images
77+
run: |
78+
# Test each Docker stage
79+
for stage in ubuntu-22 ubuntu-24 debian-12 alpine fedora-39 centos-9 python-slim python-alpine; do
80+
echo "Testing $stage..."
81+
docker build --target $stage -t vallm-test-$stage -f Dockerfile.test .
82+
83+
# Test basic functionality
84+
docker run --rm vallm-test-$stage vallm --help
85+
docker run --rm vallm-test-$stage vallm info
86+
87+
# Test validation
88+
docker run --rm vallm-test-$stage sh -c "
89+
echo 'def hello(): return \"world\"' > test.py
90+
vallm validate --file test.py
91+
"
92+
done
93+
94+
integration-tests:
95+
runs-on: ubuntu-latest
96+
needs: test-matrix
97+
98+
steps:
99+
- uses: actions/checkout@v4
100+
101+
- name: Set up Python
102+
uses: actions/setup-python@v4
103+
with:
104+
python-version: '3.11'
105+
106+
- name: Install dependencies
107+
run: |
108+
python -m pip install --upgrade pip
109+
pip install -e .[all]
110+
111+
- name: Set up Ollama
112+
run: |
113+
curl -fsSL https://ollama.ai/install.sh | sh
114+
ollama serve &
115+
sleep 10
116+
ollama pull qwen2.5-coder:7b
117+
118+
- name: Test semantic validation with LLM
119+
run: |
120+
# Create test files
121+
mkdir -p test_project
122+
echo 'def fibonacci(n: int) -> list[int]:
123+
if n <= 0:
124+
return []
125+
fib = [0, 1]
126+
for i in range(2, n):
127+
fib.append(fib[i-1] + fib[i-2])
128+
return fib' > test_project/good.py
129+
130+
echo 'def get_input():
131+
name = input("Enter name: ")
132+
print("Hello", name)
133+
return name' > test_project/bad.py
134+
135+
# Test semantic validation
136+
vallm validate --file test_project/good.py --semantic --model qwen2.5-coder:7b
137+
vallm batch test_project --recursive --semantic --model qwen2.5-coder:7b
138+
139+
- name: Test multi-language validation
140+
run: |
141+
mkdir -p multilang_test
142+
143+
# Python
144+
echo 'def hello(): return "world"' > multilang_test/test.py
145+
146+
# JavaScript
147+
echo 'function hello() { return "world"; }' > multilang_test/test.js
148+
149+
# Go
150+
echo 'package main
151+
func main() { println("Hello, World!") }' > multilang_test/test.go
152+
153+
# Rust
154+
echo 'fn main() { println!("Hello, World!"); }' > multilang_test/test.rs
155+
156+
# Test batch validation
157+
vallm batch multilang_test --recursive --include "*.py,*.js,*.go,*.rs"
158+
159+
performance-tests:
160+
runs-on: ubuntu-latest
161+
needs: test-matrix
162+
163+
steps:
164+
- uses: actions/checkout@v4
165+
166+
- name: Set up Python
167+
uses: actions/setup-python@v4
168+
with:
169+
python-version: '3.11'
170+
171+
- name: Install dependencies
172+
run: |
173+
python -m pip install --upgrade pip
174+
pip install -e .[all]
175+
pip install pytest-benchmark
176+
177+
- name: Run performance tests
178+
run: |
179+
# Create large test project
180+
mkdir -p large_project
181+
for i in {1..100}; do
182+
echo "def function_$i():
183+
return $i" > large_project/file_$i.py
184+
done
185+
186+
# Benchmark batch validation
187+
time vallm batch large_project --recursive
188+
time vallm batch large_project --recursive --format json
189+
time vallm batch large_project --recursive --format text
190+
191+
security-tests:
192+
runs-on: ubuntu-latest
193+
needs: test-matrix
194+
195+
steps:
196+
- uses: actions/checkout@v4
197+
198+
- name: Set up Python
199+
uses: actions/setup-python@v4
200+
with:
201+
python-version: '3.11'
202+
203+
- name: Install dependencies
204+
run: |
205+
python -m pip install --upgrade pip
206+
pip install -e .[security]
207+
pip install bandit safety
208+
209+
- name: Run security scans
210+
run: |
211+
# Bandit security scan
212+
bandit -r src/ -f json -o bandit-report.json
213+
214+
# Safety check for dependencies
215+
safety check --json --output safety-report.json
216+
217+
# Test vallm security validation
218+
mkdir -p security_test
219+
echo 'import os
220+
os.system("rm -rf /")' > security_test/dangerous.py
221+
222+
echo 'eval(user_input)' > security_test/eval.py
223+
224+
echo 'subprocess.Popen(["ls", "-la"], shell=True)' > security_test/shell.py
225+
226+
vallm batch security_test --recursive --security
227+
228+
- name: Upload security reports
229+
uses: actions/upload-artifact@v3
230+
with:
231+
name: security-reports
232+
path: |
233+
bandit-report.json
234+
safety-report.json
235+
236+
compatibility-tests:
237+
runs-on: ${{ matrix.os }}
238+
strategy:
239+
matrix:
240+
os: [ubuntu-latest, windows-latest, macos-latest]
241+
242+
steps:
243+
- uses: actions/checkout@v4
244+
245+
- name: Set up Python
246+
uses: actions/setup-python@v4
247+
with:
248+
python-version: '3.11'
249+
250+
- name: Test pip installation
251+
run: |
252+
python -m pip install --upgrade pip
253+
pip install -e .
254+
vallm --help
255+
vallm info
256+
257+
- name: Test pipx installation
258+
run: |
259+
pip install pipx
260+
pipx install --editable .
261+
vallm --help
262+
vallm info
263+
pipx uninstall vallm
264+
continue-on-error: true # pipx might not be available on all systems
265+
266+
- name: Test different Python versions compatibility
267+
run: |
268+
# Test with different Python versions if available
269+
for py in python3.10 python3.11 python3.12; do
270+
if command -v $py &> /dev/null; then
271+
echo "Testing with $py"
272+
$py -m venv test_env
273+
source test_env/bin/activate || test_env\\Scripts\\activate
274+
pip install -e .
275+
vallm --help
276+
deactivate || true
277+
rm -rf test_env
278+
fi
279+
done
280+
281+
documentation-tests:
282+
runs-on: ubuntu-latest
283+
284+
steps:
285+
- uses: actions/checkout@v4
286+
287+
- name: Set up Python
288+
uses: actions/setup-python@v4
289+
with:
290+
python-version: '3.11'
291+
292+
- name: Install dependencies
293+
run: |
294+
python -m pip install --upgrade pip
295+
pip install -e .[all]
296+
pip install mkdocs mkdocs-material
297+
298+
- name: Test documentation examples
299+
run: |
300+
# Test all examples from README
301+
mkdir -p examples_test
302+
303+
# Test Python API example
304+
echo 'from vallm import Proposal, validate
305+
code = "def hello(): return \"world\""
306+
proposal = Proposal(code=code, language="python")
307+
result = validate(proposal)
308+
print(f"Verdict: {result.verdict.value}")' > examples_test/api_test.py
309+
python examples_test/api_test.py
310+
311+
# Test CLI examples
312+
echo 'def fibonacci(n: int) -> list[int]:
313+
if n <= 0:
314+
return []
315+
fib = [0, 1]
316+
for i in range(2, n):
317+
fib.append(fib[i-1] + fib[i-2])
318+
return fib' > examples_test/fibonacci.py
319+
320+
vallm validate --file examples_test/fibonacci.py
321+
vallm check examples_test/fibonacci.py
322+
vallm batch examples_test --recursive
323+
324+
- name: Validate README examples
325+
run: |
326+
# Extract and test code blocks from README
327+
python -c "
328+
import re
329+
from pathlib import Path
330+
331+
readme = Path('README.md').read_text()
332+
code_blocks = re.findall(r'```bash\n(.*?)\n```', readme, re.DOTALL)
333+
334+
for i, block in enumerate(code_blocks):
335+
if 'vallm' in block and not block.startswith('#'):
336+
print(f'Testing command {i+1}: {block.strip()}')
337+
# Note: Some commands might require setup, so we just validate syntax
338+
if block.strip().startswith('vallm'):
339+
print(' ✓ Valid vallm command')
340+
"
341+
342+
final-summary:
343+
runs-on: ubuntu-latest
344+
needs: [test-matrix, docker-tests, integration-tests, performance-tests, security-tests, compatibility-tests, documentation-tests]
345+
if: always()
346+
347+
steps:
348+
- name: Test Summary
349+
run: |
350+
echo "## Test Summary" >> $GITHUB_STEP_SUMMARY
351+
echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY
352+
echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY
353+
echo "| Test Matrix | ${{ needs.test-matrix.result }} |" >> $GITHUB_STEP_SUMMARY
354+
echo "| Docker Tests | ${{ needs.docker-tests.result }} |" >> $GITHUB_STEP_SUMMARY
355+
echo "| Integration Tests | ${{ needs.integration-tests.result }} |" >> $GITHUB_STEP_SUMMARY
356+
echo "| Performance Tests | ${{ needs.performance-tests.result }} |" >> $GITHUB_STEP_SUMMARY
357+
echo "| Security Tests | ${{ needs.security-tests.result }} |" >> $GITHUB_STEP_SUMMARY
358+
echo "| Compatibility Tests | ${{ needs.compatibility-tests.result }} |" >> $GITHUB_STEP_SUMMARY
359+
echo "| Documentation Tests | ${{ needs.documentation-tests.result }} |" >> $GITHUB_STEP_SUMMARY
360+
361+
if [[ "${{ needs.test-matrix.result }}" == "success" && "${{ needs.docker-tests.result }}" == "success" ]]; then
362+
echo "✅ All critical tests passed!" >> $GITHUB_STEP_SUMMARY
363+
else
364+
echo "❌ Some tests failed. Please check the logs." >> $GITHUB_STEP_SUMMARY
365+
fi

0 commit comments

Comments
 (0)