Skip to content

Commit 258045d

Browse files
RoxyFarhadRoxyFarhad
committed
simplifying to matrix
1 parent 550d043 commit 258045d

File tree

2 files changed

+42
-351
lines changed

2 files changed

+42
-351
lines changed

‎.github/workflows/agentex-tutorials-test.yml‎

Lines changed: 30 additions & 315 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,31 @@ on:
44
workflow_dispatch:
55

66
jobs:
7-
test-tutorials:
8-
timeout-minutes: 15
9-
name: test-tutorials
7+
find-tutorials:
108
runs-on: ubuntu-latest
9+
outputs:
10+
tutorials: ${{ steps.get-tutorials.outputs.tutorials }}
11+
steps:
12+
- name: Checkout agentex-python repo
13+
uses: actions/checkout@v4
14+
15+
- name: Find all tutorials
16+
id: get-tutorials
17+
run: |
18+
cd examples/tutorials
19+
tutorials=$(find . -name "manifest.yaml" -exec dirname {} \; | sort | sed 's|^\./||' | jq -R -s -c 'split("\n") | map(select(length > 0))')
20+
echo "tutorials=$tutorials" >> $GITHUB_OUTPUT
21+
echo "Found tutorials: $tutorials"
22+
23+
test-tutorial:
24+
needs: find-tutorials
25+
runs-on: ubuntu-latest
26+
timeout-minutes: 15
27+
strategy:
28+
matrix:
29+
tutorial: ${{ fromJson(needs.find-tutorials.outputs.tutorials) }}
30+
fail-fast: false
31+
name: test-${{ matrix.tutorial }}
1132

1233
steps:
1334
- name: Checkout agentex-python repo
@@ -34,15 +55,14 @@ jobs:
3455
sudo chmod +x /usr/local/bin/yq
3556
3657
# Add extra_hosts to agentex service to make host.docker.internal work
37-
# This allows the AgentEx container to reach the GitHub runner host
3858
yq eval '.services.agentex.extra_hosts = ["host.docker.internal:host-gateway"]' -i docker-compose.yml
3959
4060
echo "✅ Added extra_hosts configuration to agentex service"
4161
42-
- name: Navigate to scale-agentex repo
62+
- name: Start AgentEx Server
4363
run: |
4464
cd scale-agentex/agentex
45-
echo "🚀 Starting dependencies (Postgres, Redis, Temporal, MongoDB)..."
65+
echo "🚀 Starting AgentEx server and dependencies..."
4666
4767
# Start all services
4868
docker compose up -d
@@ -77,316 +97,11 @@ jobs:
7797
echo "✅ SDK built successfully"
7898
ls -la dist/
7999
80-
- name: Run Parallel Tutorial Tests
100+
- name: Test Tutorial
81101
working-directory: ./examples/tutorials
82102
env:
83103
OPENAI_API_KEY: ${{ secrets.TUTORIAL_OPENAI_API_KEY }}
84104
run: |
85-
# Verify uv is working
86-
echo "UV version: $(uv --version)"
87-
echo "UV path: $(which uv)"
88-
89-
# Start background job to poll AgentEx container logs and write to file
90-
(
91-
echo "AgentEx container log polling started at $(date)" > /tmp/agentex_container_logs.txt
92-
93-
# Function to get container logs
94-
get_logs() {
95-
echo "=== AgentEx Container Logs $(date) ===" >> /tmp/agentex_container_logs.txt
96-
docker logs agentex --tail=20 2>/dev/null >> /tmp/agentex_container_logs.txt || {
97-
echo "⚠️ Failed to get logs from 'agentex' container" >> /tmp/agentex_container_logs.txt
98-
echo "Available containers:" >> /tmp/agentex_container_logs.txt
99-
docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" >> /tmp/agentex_container_logs.txt
100-
} 2>/dev/null
101-
echo "=== End Logs ===" >> /tmp/agentex_container_logs.txt
102-
echo "" >> /tmp/agentex_container_logs.txt
103-
}
104-
105-
# Poll logs every 10 seconds during test execution
106-
while true; do
107-
get_logs
108-
sleep 10
109-
done
110-
) &
111-
112-
LOG_POLLER_PID=$!
113-
echo "📋 AgentEx log polling started in background (PID: $LOG_POLLER_PID)"
114-
115-
116-
# Find all tutorial directories
117-
tutorial_paths=()
118-
for dir in $(find . -name "manifest.yaml" -exec dirname {} \; | sort); do
119-
tutorial_paths+=("${dir#./}") # Remove leading ./
120-
done
121-
122-
echo "Found ${#tutorial_paths[@]} tutorials:"
123-
printf ' %s\n' "${tutorial_paths[@]}"
124-
125-
# Run tests in parallel with unique ports
126-
pids=()
127-
failed_tests=()
128-
passed_tests=()
129-
130-
# Run all tutorial tests
131-
for i in "${!tutorial_paths[@]}"; do
132-
tutorial="${tutorial_paths[$i]}"
133-
port=$((8000 + i))
134-
135-
echo ""
136-
echo "========================================="
137-
echo "Starting test $((i+1))/${#tutorial_paths[@]}: $tutorial (port $port)"
138-
echo "========================================="
139-
140-
# Modify manifest.yaml to use unique port
141-
manifest_path="$tutorial/manifest.yaml"
142-
if [ -f "$manifest_path" ]; then
143-
# Backup original manifest
144-
cp "$manifest_path" "$manifest_path.backup"
145-
146-
# Update port in manifest (modify the line containing 'port: 8000' or 'port: XXXX')
147-
sed -i "s/port: [0-9]*/port: $port/" "$manifest_path"
148-
149-
# Keep host_address as host.docker.internal for CI (allows Docker container to reach GitHub runner host)
150-
# Note: The AgentEx server runs in Docker and needs host.docker.internal to reach the tutorial agent on the host
151-
fi
152-
153-
154-
# Run test in background with unique port and show pytest output in real-time
155-
(
156-
test_output_file="/tmp/test_output_$i.log"
157-
echo "Running test: $tutorial" | tee "$test_output_file"
158-
echo "Port: $port" | tee -a "$test_output_file"
159-
echo "========================================" | tee -a "$test_output_file"
160-
161-
# Use stdbuf to ensure unbuffered output for real-time visibility
162-
AGENTEX_API_BASE_URL="http://localhost:5003" \
163-
stdbuf -oL -eL ./run_agent_test.sh --build-cli --quiet "$tutorial" 2>&1 | tee -a "$test_output_file"
164-
165-
exit_code=${PIPESTATUS[0]}
166-
if [ $exit_code -eq 0 ]; then
167-
echo "✅ PASSED: $tutorial (port $port)"
168-
echo "$tutorial" > "/tmp/passed_$i.txt"
169-
echo "$test_output_file" > "/tmp/passed_output_$i.txt"
170-
else
171-
echo "❌ FAILED: $tutorial (port $port)"
172-
echo "$tutorial" > "/tmp/failed_$i.txt"
173-
echo "$test_output_file" > "/tmp/failed_output_$i.txt"
174-
fi
175-
exit $exit_code
176-
) &
177-
178-
pids+=($!)
179-
done
180-
181-
# Wait for all tests to complete
182-
echo ""
183-
echo "Waiting for all tests to complete..."
184-
for pid in "${pids[@]}"; do
185-
wait "$pid"
186-
done
187-
188-
# Restore all original manifests
189-
echo ""
190-
echo "Restoring original manifest files..."
191-
for tutorial in "${tutorial_paths[@]}"; do
192-
if [ -f "$tutorial/manifest.yaml.backup" ]; then
193-
mv "$tutorial/manifest.yaml.backup" "$tutorial/manifest.yaml"
194-
echo "Restored $tutorial/manifest.yaml"
195-
fi
196-
done
197-
198-
# Collect results
199-
for i in "${!tutorial_paths[@]}"; do
200-
if [ -f "/tmp/passed_$i.txt" ]; then
201-
passed_tests+=($(cat "/tmp/passed_$i.txt"))
202-
elif [ -f "/tmp/failed_$i.txt" ]; then
203-
failed_tests+=($(cat "/tmp/failed_$i.txt"))
204-
fi
205-
done
206-
207-
208-
# Stop the background log poller
209-
echo "🛑 Stopping AgentEx log poller (PID: $LOG_POLLER_PID)"
210-
kill $LOG_POLLER_PID 2>/dev/null || echo "Log poller already stopped"
211-
212-
# Print summary
213-
echo ""
214-
echo "========================================="
215-
echo "TEST SUMMARY"
216-
echo "========================================="
217-
echo "Total: ${#tutorial_paths[@]}"
218-
echo "Passed: ${#passed_tests[@]}"
219-
echo "Failed: ${#failed_tests[@]}"
220-
221-
if [ ${#failed_tests[@]} -gt 0 ]; then
222-
echo ""
223-
echo "Failed tests:"
224-
for test in "${failed_tests[@]}"; do
225-
echo " ❌ $test"
226-
done
227-
else
228-
echo ""
229-
echo "🎉 All tests passed!"
230-
fi
231-
232-
# Show comprehensive pytest summary
233-
echo ""
234-
echo "========================================="
235-
echo "PYTEST RESULTS SUMMARY"
236-
echo "========================================="
237-
238-
# Collect all pytest results
239-
total_passed_tests=0
240-
total_failed_tests=0
241-
total_pytest_tests=0
242-
243-
echo ""
244-
echo "📊 PYTEST EXECUTION SUMMARY:"
245-
echo "----------------------------------------"
246-
247-
for i in "${!tutorial_paths[@]}"; do
248-
tutorial="${tutorial_paths[$i]}"
249-
output_file="/tmp/test_output_$i.log"
250-
251-
if [ -f "$output_file" ]; then
252-
# Extract pytest summary from each test
253-
if grep -q "PYTEST OUTPUT" "$output_file"; then
254-
pytest_section=$(sed -n '/========== PYTEST OUTPUT ==========/,/========== END PYTEST OUTPUT ==========/p' "$output_file")
255-
256-
# Count individual test results in this tutorial
257-
test_passed=$(echo "$pytest_section" | grep -c "PASSED" || echo "0")
258-
test_failed=$(echo "$pytest_section" | grep -c "FAILED" || echo "0")
259-
test_errors=$(echo "$pytest_section" | grep -c "ERROR" || echo "0")
260-
261-
if [ "$test_passed" -gt 0 ] || [ "$test_failed" -gt 0 ] || [ "$test_errors" -gt 0 ]; then
262-
tutorial_total=$((test_passed + test_failed + test_errors))
263-
total_pytest_tests=$((total_pytest_tests + tutorial_total))
264-
total_passed_tests=$((total_passed_tests + test_passed))
265-
total_failed_tests=$((total_failed_tests + test_failed + test_errors))
266-
267-
if [ "$test_failed" -gt 0 ] || [ "$test_errors" -gt 0 ]; then
268-
echo "❌ $tutorial: $tutorial_total tests ($test_passed passed, $test_failed failed, $test_errors errors)"
269-
else
270-
echo "✅ $tutorial: $tutorial_total tests ($test_passed passed)"
271-
fi
272-
else
273-
echo "⚠️ $tutorial: No pytest results found"
274-
fi
275-
else
276-
echo "⚠️ $tutorial: No pytest output detected"
277-
fi
278-
else
279-
echo "⚠️ $tutorial: No output file found"
280-
fi
281-
done
282-
283-
echo "----------------------------------------"
284-
echo "🏁 OVERALL PYTEST SUMMARY:"
285-
echo " Total pytest tests run: $total_pytest_tests"
286-
echo " Total pytest tests passed: $total_passed_tests"
287-
echo " Total pytest tests failed: $total_failed_tests"
288-
echo ""
289-
290-
# Show detailed results for failed tests only
291-
if [ ${#failed_tests[@]} -gt 0 ]; then
292-
echo "❌ DETAILED FAILURE ANALYSIS:"
293-
echo "----------------------------------------"
294-
for i in "${!tutorial_paths[@]}"; do
295-
if [ -f "/tmp/failed_$i.txt" ]; then
296-
test_name=$(cat "/tmp/failed_$i.txt")
297-
output_file=$(cat "/tmp/failed_output_$i.txt")
298-
echo ""
299-
echo "📋 FAILED: $test_name"
300-
echo "----------------------------------------"
301-
if [ -f "$output_file" ]; then
302-
# Show just the pytest failure details
303-
if grep -q "PYTEST OUTPUT" "$output_file"; then
304-
echo "Pytest failure details:"
305-
sed -n '/========== PYTEST OUTPUT ==========/,/========== END PYTEST OUTPUT ==========/p' "$output_file"
306-
else
307-
echo "No pytest output found. Showing last 40 lines:"
308-
tail -40 "$output_file"
309-
fi
310-
else
311-
echo "No output file found"
312-
fi
313-
echo "----------------------------------------"
314-
fi
315-
done
316-
fi
317-
318-
if [ ${#failed_tests[@]} -gt 0 ]; then
319-
exit 1
320-
fi
321-
322-
- name: Debug Logs for Failed Tests
323-
if: always()
324-
run: |
325-
cd scale-agentex/agentex
326-
327-
echo ""
328-
echo "================================================================================"
329-
echo "🐛 DEBUG LOGS (for troubleshooting)"
330-
echo "================================================================================"
331-
332-
# Show collected AgentEx server container logs
333-
echo ""
334-
echo "========================================="
335-
echo "AGENTEX SERVER LOGS (COLLECTED DURING TESTS)"
336-
echo "========================================="
337-
if [ -f "/tmp/agentex_container_logs.txt" ]; then
338-
echo "📋 AgentEx server logs collected during test execution:"
339-
echo "----------------------------------------"
340-
cat /tmp/agentex_container_logs.txt
341-
echo "----------------------------------------"
342-
else
343-
echo "⚠️ No AgentEx container logs file found"
344-
echo "📋 Attempting to get current container logs:"
345-
echo "----------------------------------------"
346-
docker logs agentex --tail=50 2>/dev/null || {
347-
echo "❌ Failed to get logs from 'agentex' container"
348-
echo "Available containers:"
349-
docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}"
350-
}
351-
echo "----------------------------------------"
352-
fi
353-
354-
# Show tutorial agent logs for failed tests
355-
cd ../../examples/tutorials
356-
357-
# Check if there were any failed tests
358-
failed_tests=()
359-
for i in {0..20}; do # Check up to 20 possible tests
360-
if [ -f "/tmp/failed_$i.txt" ]; then
361-
failed_tests+=($(cat "/tmp/failed_$i.txt"))
362-
fi
363-
done
364-
365-
if [ ${#failed_tests[@]} -gt 0 ]; then
366-
echo ""
367-
echo "========================================="
368-
echo "TUTORIAL AGENT LOGS FOR FAILED TESTS"
369-
echo "========================================="
370-
371-
for test in "${failed_tests[@]}"; do
372-
test_name=$(basename "$test")
373-
logfile="/tmp/agentex-${test_name}.log"
374-
375-
echo ""
376-
echo "📋 Tutorial agent logs for $test:"
377-
echo "----------------------------------------"
378-
379-
if [ -f "$logfile" ]; then
380-
echo "Tutorial agent logs (last 100 lines):"
381-
tail -100 "$logfile"
382-
else
383-
echo "⚠️ No agent log file found at: $logfile"
384-
echo "Available agent log files:"
385-
ls -la /tmp/agentex-*.log 2>/dev/null || echo " (none)"
386-
fi
387-
echo "----------------------------------------"
388-
done
389-
else
390-
echo ""
391-
echo "✅ No failed tests - no agent logs to show"
392-
fi
105+
echo "Testing tutorial: ${{ matrix.tutorial }}"
106+
AGENTEX_API_BASE_URL="http://localhost:5003" \
107+
./run_agent_test.sh --build-cli "${{ matrix.tutorial }}"

0 commit comments

Comments
 (0)