Skip to content

Commit 588c460

Browse files
sicoylesicoyle
authored
test: automate testing our quickstarts (#253)
* test: add quickstart integration tests Signed-off-by: Samantha Coyle <[email protected]> * build: add ci to run integration tests on comment Signed-off-by: Samantha Coyle <[email protected]> * test: only run one test func for now Signed-off-by: Samantha Coyle <[email protected]> * fix: lint + add me notes Signed-off-by: Samantha Coyle <[email protected]> * style: clean up old references Signed-off-by: Samantha Coyle <[email protected]> * style: fixes for tox -e flake8 Signed-off-by: Samantha Coyle <[email protected]> * fix: try this to exclude the tests Signed-off-by: Samantha Coyle <[email protected]> * fix: allow me to test my ci changes Signed-off-by: Samantha Coyle <[email protected]> * fix: enable better ci insights Signed-off-by: Samantha Coyle <[email protected]> * fix: address feedback plus make tests more like user install with own venv and installation Signed-off-by: Samantha Coyle <[email protected]> * fix: sync all deps Signed-off-by: Samantha Coyle <[email protected]> * style: tox -e ruff Signed-off-by: Samantha Coyle <[email protected]> * fix: update for tox -e flake8 Signed-off-by: Samantha Coyle <[email protected]> * fix: rm personal todo comment Signed-off-by: Samantha Coyle <[email protected]> * fix: just use openai key for now Signed-off-by: Samantha Coyle <[email protected]> --------- Signed-off-by: Samantha Coyle <[email protected]>
1 parent efb9c93 commit 588c460

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+1754
-585
lines changed

‎.github/workflows/e2e-tests.yaml‎

Lines changed: 0 additions & 62 deletions
This file was deleted.

‎.github/workflows/integration-tests.yaml‎

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
name: Integration Tests
2+
3+
on:
4+
issue_comment:
5+
types: [created]
6+
workflow_dispatch:
7+
# Allow manual trigger for testing
8+
9+
10+
permissions:
11+
contents: read
12+
pull-requests: read
13+
issues: write
14+
15+
jobs:
16+
integration-tests:
17+
runs-on: ubuntu-latest
18+
strategy:
19+
fail-fast: false
20+
matrix:
21+
python_ver: ["3.10", "3.11", "3.12", "3.13"]
22+
# Only run if comment contains "/ok-to-test" and is on a PR
23+
# Also check that commenter is a maintainer/owner
24+
# Or if it's a manual workflow_dispatch
25+
if: |
26+
(github.event_name == 'issue_comment' &&
27+
github.event.issue.pull_request &&
28+
contains(github.event.comment.body, '/ok-to-test') &&
29+
(github.event.comment.author_association == 'OWNER' ||
30+
github.event.comment.author_association == 'MEMBER' ||
31+
github.event.comment.author_association == 'COLLABORATOR')) ||
32+
github.event_name == 'pull_request' ||
33+
github.event_name == 'workflow_dispatch'
34+
steps:
35+
- name: Get PR details
36+
if: github.event_name == 'issue_comment'
37+
id: pr
38+
uses: actions/github-script@v7
39+
with:
40+
script: |
41+
const pr = context.payload.issue.pull_request;
42+
if (!pr) {
43+
core.setFailed('Not a pull request');
44+
return;
45+
}
46+
const { data } = await github.rest.pulls.get({
47+
owner: context.repo.owner,
48+
repo: context.repo.repo,
49+
pull_number: context.payload.issue.number,
50+
});
51+
core.setOutput('head_sha', data.head.sha);
52+
core.setOutput('head_ref', data.head.ref);
53+
core.setOutput('head_repo', data.head.repo.full_name);
54+
55+
- name: Checkout PR
56+
if: github.event_name == 'issue_comment'
57+
uses: actions/checkout@v4
58+
with:
59+
ref: ${{ steps.pr.outputs.head_sha }}
60+
repository: ${{ steps.pr.outputs.head_repo }}
61+
62+
- name: Checkout
63+
if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'
64+
uses: actions/checkout@v4
65+
66+
- name: Set up Python ${{ matrix.python_ver }}
67+
uses: actions/setup-python@v5
68+
with:
69+
python-version: ${{ matrix.python_ver }}
70+
71+
- name: Set up Docker
72+
uses: docker/setup-docker-action@v3
73+
74+
- name: Install Dapr CLI
75+
run: |
76+
wget -q https://raw.githubusercontent.com/dapr/cli/master/install/install.sh -O - | /bin/bash
77+
78+
- name: Initialize Dapr
79+
run: |
80+
dapr init
81+
82+
- name: Install uv
83+
uses: astral-sh/setup-uv@v3
84+
with:
85+
version: "latest"
86+
87+
- name: Install dependencies
88+
run: |
89+
uv venv
90+
source .venv/bin/activate
91+
uv pip install -e .
92+
uv pip install -e ".[test]"
93+
94+
- name: Verify secrets are available
95+
run: |
96+
if [ -z "${{ secrets.OPENAI_API_KEY }}" ]; then
97+
echo "OPENAI_API_KEY secret is not available"
98+
echo "This is expected for PRs from forks (GitHub doesn't expose secrets to forks for security)"
99+
echo "For PRs from the same repository, check that secrets are configured in repository settings"
100+
else
101+
echo "OPENAI_API_KEY is available"
102+
fi
103+
104+
if [ -z "${{ secrets.ELEVENLABS_API_KEY }}" ]; then
105+
echo "ELEVENLABS_API_KEY secret is not available"
106+
else
107+
echo "ELEVENLABS_API_KEY is available"
108+
fi
109+
110+
if [ -z "${{ secrets.NVIDIA_API_KEY }}" ]; then
111+
echo "NVIDIA_API_KEY secret is not available"
112+
else
113+
echo "NVIDIA_API_KEY is available"
114+
fi
115+
116+
- name: Run Integration Tests
117+
env:
118+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
119+
run: |
120+
source .venv/bin/activate
121+
pytest -m integration -v tests/integration/quickstarts/test_01_hello_world.py::TestHelloWorldQuickstart::test_basic_llm
122+
# Run all integration tests in parallel (one file per quickstart directory)
123+
# -n auto uses all available CPUs, or specify -n 4 for fixed number
124+
# pytest -m integration -v -n auto tests/integration/quickstarts/
125+
126+
# Summary job that runs after all matrix jobs complete and comments on PR
127+
comment-summary:
128+
needs: integration-tests
129+
runs-on: ubuntu-latest
130+
if: always() && github.event_name == 'issue_comment'
131+
steps:
132+
- name: Comment PR with test results summary
133+
uses: actions/github-script@v7
134+
with:
135+
script: |
136+
const prNumber = context.payload.issue.number;
137+
const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({
138+
owner: context.repo.owner,
139+
repo: context.repo.repo,
140+
run_id: context.runId,
141+
});
142+
143+
// Get all integration test jobs (one per Python version from matrix)
144+
const testJobs = jobs.jobs.filter(job => job.name === 'integration-tests');
145+
146+
// Build summary table
147+
let summary = '## Integration Tests Summary\n\n';
148+
summary += '| Python Version | Status |\n';
149+
summary += '|----------------|--------|\n';
150+
151+
let allPassed = true;
152+
const pythonVersions = ['3.10', '3.11', '3.12', '3.13'];
153+
154+
// Sort jobs by Python version order
155+
for (const pythonVer of pythonVersions) {
156+
const job = testJobs.find(j =>
157+
j.steps.some(step => step.name && step.name.includes(pythonVer))
158+
) || testJobs[pythonVersions.indexOf(pythonVer)];
159+
160+
if (job) {
161+
const status = job.conclusion === 'success' ? '✅ PASSED' : '❌ FAILED';
162+
const jobUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}/job/${job.id}`;
163+
summary += `| ${pythonVer} | [${status}](${jobUrl}) |\n`;
164+
if (job.conclusion !== 'success') {
165+
allPassed = false;
166+
}
167+
}
168+
}
169+
170+
summary += `\n**Overall Status:** ${allPassed ? '✅ All tests passed' : '❌ Some tests failed'}\n`;
171+
summary += `**Workflow:** [View run](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})\n`;
172+
173+
// Find existing comment or create new one
174+
const { data: comments } = await github.rest.issues.listComments({
175+
owner: context.repo.owner,
176+
repo: context.repo.repo,
177+
issue_number: prNumber,
178+
});
179+
180+
const existingComment = comments.find(c =>
181+
c.user.type === 'Bot' &&
182+
c.body.includes('## Integration Tests Summary')
183+
);
184+
185+
if (existingComment) {
186+
await github.rest.issues.updateComment({
187+
owner: context.repo.owner,
188+
repo: context.repo.repo,
189+
comment_id: existingComment.id,
190+
body: summary,
191+
});
192+
} else {
193+
await github.rest.issues.createComment({
194+
owner: context.repo.owner,
195+
repo: context.repo.repo,
196+
issue_number: prNumber,
197+
body: summary,
198+
});
199+
}
200+

‎Makefile‎

Lines changed: 1 addition & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
# Get all directories within quickstarts
2-
QUICKSTART_DIRS := $(shell find quickstarts -mindepth 1 -maxdepth 1 -type d -exec basename {} \;)
3-
41
# Test targets
52
.PHONY: test
63
test:
@@ -19,59 +16,4 @@ test-install:
1916

2017
.PHONY: test-all
2118
test-all: test-install test-cov
22-
@echo "All tests completed!"
23-
24-
# Main target to validate all quickstarts
25-
.PHONY: validate-quickstarts
26-
validate-quickstarts:
27-
@echo "Validating all quickstart directories..."
28-
@for dir in $(QUICKSTART_DIRS); do \
29-
echo "\n=== Validating $$dir ==="; \
30-
( \
31-
cd quickstarts && \
32-
cd "$$dir" && \
33-
if [ -f requirements.txt ]; then \
34-
echo "Creating virtual environment for $$dir..." && \
35-
python3 -m venv .venv && \
36-
echo "Activating virtual environment and installing requirements..." && \
37-
. .venv/bin/activate && \
38-
pip install -r requirements.txt && \
39-
USING_VENV=true; \
40-
else \
41-
echo "No requirements.txt found in $$dir, skipping virtual environment setup"; \
42-
USING_VENV=false; \
43-
fi && \
44-
cd .. && \
45-
echo "Running validation script for $$dir..." && \
46-
./validate.sh "$$dir"; \
47-
RESULT=$$?; \
48-
if [ "$$USING_VENV" = "true" ]; then \
49-
cd "$$dir" && \
50-
echo "Deactivating and cleaning up virtual environment..." && \
51-
deactivate && \
52-
rm -rf .venv; \
53-
fi; \
54-
exit $$RESULT; \
55-
); \
56-
if [ $$? -ne 0 ]; then \
57-
echo "Validation failed for $$dir. Stopping all validations."; \
58-
exit 1; \
59-
fi; \
60-
sleep 1; \
61-
done
62-
@echo "\nAll validations completed successfully!"
63-
64-
# Useful for local development, with a single, controlled venv
65-
.PHONY: validate-quickstarts-local
66-
validate-quickstarts-local:
67-
@echo "Validating all quickstart directories..."
68-
@for dir in $(QUICKSTART_DIRS); do \
69-
echo "\n=== Validating $$dir ==="; \
70-
(cd quickstarts && ./validate.sh $$dir); \
71-
if [ $$? -ne 0 ]; then \
72-
echo "Validation failed for $$dir. Stopping all validations."; \
73-
exit 1; \
74-
fi; \
75-
sleep 2; \
76-
done
77-
@echo "\nAll validations completed successfully!"
19+
@echo "All tests completed!"

0 commit comments

Comments
 (0)