Skip to content

Commit 51c3fac

Browse files
RoxyFarhadRoxyFarhad
authored
Merge pull request #173 from scaleapi/RF/feat/integration-pipeline
Adding the Integrations Tests
2 parents 73e2dd9 + dfbf3e2 commit 51c3fac

File tree

7 files changed

+513
-134
lines changed

7 files changed

+513
-134
lines changed

‎.github/workflows/agentex-tutorials-test.yml‎

Lines changed: 315 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,315 @@
1+
name: Test Tutorial Agents
2+
3+
on:
4+
pull_request:
5+
branches: [ main ]
6+
push:
7+
branches: [ main ]
8+
workflow_dispatch:
9+
10+
jobs:
11+
find-tutorials:
12+
runs-on: ubuntu-latest
13+
outputs:
14+
tutorials: ${{ steps.get-tutorials.outputs.tutorials }}
15+
steps:
16+
- name: Checkout agentex-python repo
17+
uses: actions/checkout@v4
18+
19+
- name: Find all tutorials
20+
id: get-tutorials
21+
run: |
22+
cd examples/tutorials
23+
# Find all tutorials and exclude specific temporal ones
24+
all_tutorials=$(find . -name "manifest.yaml" -exec dirname {} \; | sort | sed 's|^\./||')
25+
26+
# Filter out the specified temporal tutorials that are being updated
27+
filtered_tutorials=$(echo "$all_tutorials" | grep -v -E "(temporal)")
28+
29+
# Convert to JSON array
30+
tutorials=$(echo "$filtered_tutorials" | jq -R -s -c 'split("\n") | map(select(length > 0))')
31+
32+
echo "tutorials=$tutorials" >> $GITHUB_OUTPUT
33+
echo "All tutorials found: $(echo "$all_tutorials" | wc -l)"
34+
echo "Filtered tutorials: $(echo "$filtered_tutorials" | wc -l)"
35+
echo "Excluded tutorials:"
36+
echo "$all_tutorials" | grep -E "(10_temporal/050_|10_temporal/070_|10_temporal/080_)" || echo " (none matched exclusion pattern)"
37+
echo "Final tutorial list: $tutorials"
38+
39+
test-tutorial:
40+
needs: find-tutorials
41+
runs-on: ubuntu-latest
42+
timeout-minutes: 15
43+
strategy:
44+
matrix:
45+
tutorial: ${{ fromJson(needs.find-tutorials.outputs.tutorials) }}
46+
fail-fast: false
47+
name: test-${{ matrix.tutorial }}
48+
49+
steps:
50+
- name: Checkout agentex-python repo
51+
uses: actions/checkout@v4
52+
53+
- name: Install UV
54+
run: |
55+
curl -LsSf https://astral.sh/uv/install.sh | sh
56+
echo "$HOME/.local/bin" >> $GITHUB_PATH
57+
58+
- name: Pull latest AgentEx image
59+
run: |
60+
echo "🐳 Pulling latest Scale AgentEx Docker image..."
61+
docker pull ghcr.io/scaleapi/scale-agentex/agentex:latest
62+
echo "✅ Successfully pulled AgentEx Docker image"
63+
64+
- name: Checkout scale-agentex repo
65+
uses: actions/checkout@v4
66+
with:
67+
repository: scaleapi/scale-agentex
68+
path: scale-agentex
69+
70+
- name: Configure Docker Compose for pulled image and host networking
71+
run: |
72+
cd scale-agentex/agentex
73+
echo "🔧 Configuring AgentEx container to use pulled image and host networking..."
74+
75+
# Install yq for YAML manipulation
76+
sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
77+
sudo chmod +x /usr/local/bin/yq
78+
79+
# Override to use pulled image instead of building
80+
yq eval '.services.agentex.image = "ghcr.io/scaleapi/scale-agentex/agentex:latest"' -i docker-compose.yml
81+
yq eval 'del(.services.agentex.build)' -i docker-compose.yml
82+
83+
# Add extra_hosts to agentex service to make host.docker.internal work
84+
yq eval '.services.agentex.extra_hosts = ["host.docker.internal:host-gateway"]' -i docker-compose.yml
85+
86+
echo "✅ Configured docker-compose to use pulled image with host access"
87+
88+
- name: Start AgentEx Server
89+
run: |
90+
cd scale-agentex/agentex
91+
echo "🚀 Starting AgentEx server and dependencies..."
92+
93+
# Start all services
94+
docker compose up -d
95+
96+
echo "⏳ Waiting for dependencies to be healthy..."
97+
98+
# Wait for services to be healthy
99+
for i in {1..30}; do
100+
if docker compose ps | grep -q "healthy"; then
101+
echo "✅ Dependencies are healthy"
102+
break
103+
fi
104+
echo " Attempt $i/30: Waiting for services..."
105+
sleep 5
106+
done
107+
108+
# Wait specifically for AgentEx server to be ready
109+
echo "⏳ Waiting for AgentEx server to be ready..."
110+
for i in {1..30}; do
111+
if curl -s --max-time 5 http://localhost:5003/health >/dev/null 2>&1; then
112+
echo "✅ AgentEx server is ready"
113+
break
114+
fi
115+
echo " Attempt $i/30: Waiting for AgentEx server..."
116+
sleep 5
117+
done
118+
119+
- name: Build AgentEx SDK
120+
run: |
121+
echo "🔨 Building AgentEx SDK wheel..."
122+
uv build
123+
echo "✅ SDK built successfully"
124+
ls -la dist/
125+
126+
- name: Test Tutorial
127+
id: run-test
128+
working-directory: ./examples/tutorials
129+
env:
130+
OPENAI_API_KEY: ${{ secrets.TUTORIAL_OPENAI_API_KEY }}
131+
HEALTH_CHECK_PORT: 8080 # Use non-privileged port for temporal worker health checks
132+
run: |
133+
echo "Testing tutorial: ${{ matrix.tutorial }}"
134+
AGENTEX_API_BASE_URL="http://localhost:5003" \
135+
./run_agent_test.sh --build-cli "${{ matrix.tutorial }}"
136+
137+
- name: Print agent logs on failure
138+
if: failure()
139+
working-directory: ./examples/tutorials
140+
run: |
141+
echo "🚨 Test failed for tutorial: ${{ matrix.tutorial }}"
142+
echo "📋 Printing agent logs..."
143+
144+
# Look for agent log files in the tutorial directory
145+
if find "${{ matrix.tutorial }}" -name "*.log" -type f 2>/dev/null | grep -q .; then
146+
echo "Found agent log files:"
147+
find "${{ matrix.tutorial }}" -name "*.log" -type f -exec echo "=== {} ===" \; -exec cat {} \;
148+
else
149+
echo "No .log files found, checking for other common log locations..."
150+
fi
151+
152+
# Check for any output files or dumps
153+
if find "${{ matrix.tutorial }}" -name "agent_output*" -o -name "debug*" -o -name "*.out" 2>/dev/null | grep -q .; then
154+
echo "Found other output files:"
155+
find "${{ matrix.tutorial }}" -name "agent_output*" -o -name "debug*" -o -name "*.out" -exec echo "=== {} ===" \; -exec cat {} \;
156+
fi
157+
158+
# Print the last 50 lines of any python processes that might still be running
159+
echo "🔍 Checking for running python processes..."
160+
ps aux | grep python || echo "No python processes found"
161+
162+
- name: Record test result
163+
id: test-result
164+
if: always()
165+
run: |
166+
# Create results directory
167+
mkdir -p test-results
168+
169+
# Determine result
170+
if [ "${{ steps.run-test.outcome }}" == "success" ]; then
171+
result="passed"
172+
echo "result=passed" >> $GITHUB_OUTPUT
173+
echo "tutorial=${{ matrix.tutorial }}" >> $GITHUB_OUTPUT
174+
else
175+
result="failed"
176+
echo "result=failed" >> $GITHUB_OUTPUT
177+
echo "tutorial=${{ matrix.tutorial }}" >> $GITHUB_OUTPUT
178+
fi
179+
180+
# Save result to file for artifact upload
181+
# Create a safe filename from tutorial path
182+
safe_name=$(echo "${{ matrix.tutorial }}" | tr '/' '_' | tr -d ' ')
183+
echo "$result" > "test-results/result-${safe_name}.txt"
184+
echo "${{ matrix.tutorial }}" > "test-results/tutorial-${safe_name}.txt"
185+
echo "safe_name=${safe_name}" >> $GITHUB_OUTPUT
186+
187+
- name: Upload test result
188+
if: always()
189+
uses: actions/upload-artifact@v4
190+
with:
191+
name: test-result-${{ steps.test-result.outputs.safe_name }}
192+
path: test-results/
193+
retention-days: 1
194+
195+
test-summary:
196+
if: always()
197+
needs: [find-tutorials, test-tutorial]
198+
runs-on: ubuntu-latest
199+
name: Test Summary
200+
steps:
201+
- name: Download all test results
202+
uses: actions/download-artifact@v4
203+
with:
204+
pattern: test-result-*
205+
path: all-results/
206+
merge-multiple: true
207+
continue-on-error: true
208+
209+
- name: Generate Test Summary
210+
run: |
211+
echo "# 🧪 Tutorial Tests Summary" >> $GITHUB_STEP_SUMMARY
212+
echo "" >> $GITHUB_STEP_SUMMARY
213+
214+
# Initialize counters
215+
passed_count=0
216+
failed_count=0
217+
skipped_count=0
218+
total_count=0
219+
220+
# Get all tutorials that were supposed to run
221+
tutorials='${{ needs.find-tutorials.outputs.tutorials }}'
222+
223+
if [ -d "all-results" ] && [ "$(ls -A all-results 2>/dev/null)" ]; then
224+
echo "📊 Processing individual test results from artifacts..."
225+
226+
echo "## Test Results" >> $GITHUB_STEP_SUMMARY
227+
echo "" >> $GITHUB_STEP_SUMMARY
228+
echo "| Tutorial | Status | Result |" >> $GITHUB_STEP_SUMMARY
229+
echo "|----------|--------|--------|" >> $GITHUB_STEP_SUMMARY
230+
231+
# Process each result file
232+
for result_file in all-results/result-*.txt; do
233+
if [ -f "$result_file" ]; then
234+
# Extract the safe name from filename
235+
safe_name=$(basename "$result_file" .txt | sed 's/result-//')
236+
237+
# Get corresponding tutorial name file
238+
tutorial_file="all-results/tutorial-${safe_name}.txt"
239+
240+
if [ -f "$tutorial_file" ]; then
241+
tutorial_name=$(cat "$tutorial_file")
242+
result=$(cat "$result_file")
243+
244+
total_count=$((total_count + 1))
245+
246+
if [ "$result" = "passed" ]; then
247+
echo "| \`$tutorial_name\` | ✅ | Passed |" >> $GITHUB_STEP_SUMMARY
248+
passed_count=$((passed_count + 1))
249+
else
250+
echo "| \`$tutorial_name\` | ❌ | Failed |" >> $GITHUB_STEP_SUMMARY
251+
failed_count=$((failed_count + 1))
252+
fi
253+
fi
254+
fi
255+
done
256+
257+
# Check for any tutorials that didn't have results (skipped/cancelled)
258+
echo "$tutorials" | jq -r '.[]' | while read expected_tutorial; do
259+
safe_expected=$(echo "$expected_tutorial" | tr '/' '_' | tr -d ' ')
260+
if [ ! -f "all-results/result-${safe_expected}.txt" ]; then
261+
echo "| \`$expected_tutorial\` | ⏭️ | Skipped/Cancelled |" >> $GITHUB_STEP_SUMMARY
262+
skipped_count=$((skipped_count + 1))
263+
total_count=$((total_count + 1))
264+
fi
265+
done
266+
267+
else
268+
echo "⚠️ No individual test results found. This could mean:"
269+
echo "- Test jobs were cancelled before completion"
270+
echo "- Artifacts failed to upload"
271+
echo "- No tutorials were found to test"
272+
echo ""
273+
274+
overall_result="${{ needs.test-tutorial.result }}"
275+
echo "Overall job status: **$overall_result**"
276+
277+
if [[ "$overall_result" == "success" ]]; then
278+
echo "✅ All tests appear to have passed based on job status."
279+
elif [[ "$overall_result" == "failure" ]]; then
280+
echo "❌ Some tests appear to have failed based on job status."
281+
echo ""
282+
echo "💡 **Tip:** Check individual job logs for specific failure details."
283+
elif [[ "$overall_result" == "cancelled" ]]; then
284+
echo "⏭️ Tests were cancelled."
285+
else
286+
echo "❓ Test status is unclear: $overall_result"
287+
fi
288+
289+
# Don't show detailed breakdown when we don't have individual results
290+
tutorial_count=$(echo "$tutorials" | jq -r '. | length')
291+
echo ""
292+
echo "Expected tutorial count: $tutorial_count"
293+
fi
294+
295+
# Only show detailed statistics if we have individual results
296+
if [ -d "all-results" ] && [ "$(ls -A all-results 2>/dev/null)" ]; then
297+
echo "" >> $GITHUB_STEP_SUMMARY
298+
echo "## Summary Statistics" >> $GITHUB_STEP_SUMMARY
299+
echo "" >> $GITHUB_STEP_SUMMARY
300+
echo "- **Total Tests:** $total_count" >> $GITHUB_STEP_SUMMARY
301+
echo "- **Passed:** $passed_count ✅" >> $GITHUB_STEP_SUMMARY
302+
echo "- **Failed:** $failed_count ❌" >> $GITHUB_STEP_SUMMARY
303+
echo "- **Skipped:** $skipped_count ⏭️" >> $GITHUB_STEP_SUMMARY
304+
echo "" >> $GITHUB_STEP_SUMMARY
305+
306+
if [ $failed_count -eq 0 ] && [ $passed_count -gt 0 ]; then
307+
echo "🎉 **All tests passed!**" >> $GITHUB_STEP_SUMMARY
308+
elif [ $failed_count -gt 0 ]; then
309+
echo "⚠️ **Some tests failed.** Check individual job logs for details." >> $GITHUB_STEP_SUMMARY
310+
echo "" >> $GITHUB_STEP_SUMMARY
311+
echo "💡 **Tip:** Look for the 'Print agent logs on failure' step in failed jobs for debugging information." >> $GITHUB_STEP_SUMMARY
312+
else
313+
echo "ℹ️ **Tests were cancelled or skipped.**" >> $GITHUB_STEP_SUMMARY
314+
fi
315+
fi

‎examples/tutorials/10_async/00_base/000_hello_acp/tests/test_agent.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ async def test_send_event_and_poll(self, client: AsyncAgentex, agent_id: str):
9696
):
9797
assert isinstance(message, TaskMessage)
9898
if message.content and message.content.type == "text" and message.content.author == "agent":
99-
assert "Hello! I've received your message" in message.content.content
99+
assert "Hello! I've received your task" in message.content.content
100100
break
101101

102102

0 commit comments

Comments
 (0)