cleaning up

Author: RoxyFarhad

.github/workflows/agentex-tutorials-test.yml

@@ -20,7 +20,7 @@ jobs:
           all_tutorials=$(find . -name "manifest.yaml" -exec dirname {} \; | sort | sed 's|^\./||')
 
           # Filter out the specified temporal tutorials that are being updated
-          filtered_tutorials=$(echo "$all_tutorials" | grep -v -E "(10_temporal/050_|10_temporal/070_|10_temporal/080_)")
+          filtered_tutorials=$(echo "$all_tutorials" | grep -v -E "(temporal)")
 
           # Convert to JSON array
           tutorials=$(echo "$filtered_tutorials" | jq -R -s -c 'split("\n") | map(select(length > 0))')
@@ -116,8 +116,24 @@ jobs:
           HEALTH_CHECK_PORT: 8080 # Use non-privileged port for temporal worker health checks
         run: |
           echo "Testing tutorial: ${{ matrix.tutorial }}"
+
+          # Create output files
+          SANITIZED_NAME=$(echo "${{ matrix.tutorial }}" | sed 's/\//-/g')
+          OUTPUT_FILE="/tmp/test-output-$SANITIZED_NAME.txt"
+          STATUS_FILE="/tmp/test-status-$SANITIZED_NAME.txt"
+
+          # Run the test, show output live, and also capture to file
+          set +e  # Don't exit on error
           AGENTEX_API_BASE_URL="http://localhost:5003" \
-            ./run_agent_test.sh --build-cli "${{ matrix.tutorial }}"
+            ./run_agent_test.sh --build-cli "${{ matrix.tutorial }}" 2>&1 | tee "$OUTPUT_FILE"
+          EXIT_CODE=${PIPESTATUS[0]}  # Get exit code from the first command in pipeline
+          set -e  # Re-enable exit on error
+
+          # Write the exit code to status file
+          echo "$EXIT_CODE" > "$STATUS_FILE"
+
+          # Exit with the original exit code to make the job succeed/fail correctly
+          exit $EXIT_CODE
 
       - name: Upload Test Results
         if: always()
@@ -128,7 +144,13 @@ jobs:
 
           # Create a temporary directory with the sanitized name
           mkdir -p "test-results-$SANITIZED_NAME"
-          cp /tmp/agentex-*.log "test-results-$SANITIZED_NAME/" 2>/dev/null || echo "No log files to copy"
+
+          # Copy the test output and status files
+          cp "/tmp/test-output-$SANITIZED_NAME.txt" "test-results-$SANITIZED_NAME/" 2>/dev/null || echo "No output file to copy"
+          cp "/tmp/test-status-$SANITIZED_NAME.txt" "test-results-$SANITIZED_NAME/" 2>/dev/null || echo "No status file to copy"
+
+          # Also copy agent logs if they exist
+          cp /tmp/agentex-*.log "test-results-$SANITIZED_NAME/" 2>/dev/null || echo "No agent log files to copy"
 
           # Upload using the actions/upload-artifact action
           echo "artifact-name=test-results-$SANITIZED_NAME" >> $GITHUB_ENV
@@ -161,17 +183,6 @@ jobs:
           # Get tutorial list from needs context
           tutorials='${{ needs.find-tutorials.outputs.tutorials }}'
 
-          # Debug: Show what we're working with
-          echo "🔍 DEBUG: Tutorial list from find-tutorials job:"
-          echo "$tutorials"
-          echo ""
-          echo "🔍 DEBUG: Downloaded artifacts:"
-          ls -la test-results/ || echo "No test-results directory found"
-          echo ""
-          echo "🔍 DEBUG: Artifact contents:"
-          find test-results/ -type f -name "*.log" || echo "No log files found"
-          echo ""
-
           # Initialize counters
           total_tutorials=0
           passed_tutorials=0
@@ -184,27 +195,31 @@ jobs:
           echo "## 📊 Overall Results" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
 
-          # Process each tutorial result
-          for tutorial_dir in test-results/test-results-*/; do
-            if [ -d "$tutorial_dir" ]; then
-              # Extract sanitized name and convert back to original tutorial path
-              sanitized_name=$(basename "$tutorial_dir" | sed 's/test-results-//')
-              tutorial_name=$(echo "$sanitized_name" | sed 's/-/\//g')
-              total_tutorials=$((total_tutorials + 1))
-
-              # Check if there are any log files in this directory
-              if find "$tutorial_dir" -name "*.log" -type f | grep -q .; then
-                # Determine success/failure based on pytest-specific failure patterns
-                if find "$tutorial_dir" -name "*.log" -exec grep -l "FAILED.*::" {} \; | head -1 >/dev/null || \
-                   find "$tutorial_dir" -name "*.log" -exec grep -l "=== FAILURES ===" {} \; | head -1 >/dev/null || \
-                   find "$tutorial_dir" -name "*.log" -exec grep -l "AssertionError" {} \; | head -1 >/dev/null; then
-                  failed_tutorials=$((failed_tutorials + 1))
-                  failed_tests+=("$tutorial_name")
-                else
-                  passed_tutorials=$((passed_tutorials + 1))
-                  passed_tests+=("$tutorial_name")
-                fi
+          # Process each tutorial and check its status file
+          echo "$tutorials" | jq -r '.[]' | while read -r tutorial_name; do
+            total_tutorials=$((total_tutorials + 1))
+
+            # Convert tutorial name to sanitized format
+            sanitized_name=$(echo "$tutorial_name" | sed 's/\//-/g')
+            tutorial_dir="test-results/test-results-$sanitized_name"
+            status_file="$tutorial_dir/test-status-$sanitized_name.txt"
+
+            if [ -f "$status_file" ]; then
+              exit_code=$(cat "$status_file")
+              if [ "$exit_code" = "0" ]; then
+                passed_tutorials=$((passed_tutorials + 1))
+                passed_tests+=("$tutorial_name")
+                echo "✅ $tutorial_name: PASSED (exit code: $exit_code)"
+              else
+                failed_tutorials=$((failed_tutorials + 1))
+                failed_tests+=("$tutorial_name")
+                echo "❌ $tutorial_name: FAILED (exit code: $exit_code)"
               fi
+            else
+              # No status file found, assume failed
+              failed_tutorials=$((failed_tutorials + 1))
+              failed_tests+=("$tutorial_name")
+              echo "❌ $tutorial_name: FAILED (no status file)"
             fi
           done
 
@@ -226,35 +241,26 @@ jobs:
             echo "" >> $GITHUB_STEP_SUMMARY
           fi
 
-          # Show pytest failures only for failed tests
+          # Show test output for failed tests
           if [ $failed_tutorials -gt 0 ]; then
             echo "## ❌ Failed Tutorials ($failed_tutorials)" >> $GITHUB_STEP_SUMMARY
             echo "" >> $GITHUB_STEP_SUMMARY
             echo '```' >> $GITHUB_STEP_SUMMARY
 
-            # Extract and append pytest failures from each failed test
+            # Show the full output for each failed test
             for test in "${failed_tests[@]}"; do
-              # Find the log file for this test (convert back to sanitized name)
               sanitized_test_name=$(echo "$test" | sed 's/\//-/g')
-              log_file=$(find "test-results/test-results-$sanitized_test_name" -name "*.log" | head -1)
-              if [ -f "$log_file" ]; then
+              output_file="test-results/test-results-$sanitized_test_name/test-output-$sanitized_test_name.txt"
+
+              if [ -f "$output_file" ]; then
                 echo "================================================================================================" >> $GITHUB_STEP_SUMMARY
                 echo "FAILED: $test" >> $GITHUB_STEP_SUMMARY
                 echo "================================================================================================" >> $GITHUB_STEP_SUMMARY
-
-                # Extract pytest output between the delimiters, or show pytest summary if no delimiters
-                if grep -q "========== PYTEST OUTPUT ==========" "$log_file"; then
-                  sed -n '/========== PYTEST OUTPUT ==========/,/========== END PYTEST OUTPUT ==========/p' "$log_file" | \
-                    sed '1d;$d' >> $GITHUB_STEP_SUMMARY
-                else
-                  # If no delimiters, try to extract pytest-related lines
-                  grep -E "(FAILED|ERROR|AssertionError|collected.*items|=====.*=====|::.*FAILED)" "$log_file" >> $GITHUB_STEP_SUMMARY || \
-                    echo "No pytest output found in log file" >> $GITHUB_STEP_SUMMARY
-                fi
+                cat "$output_file" >> $GITHUB_STEP_SUMMARY
                 echo "" >> $GITHUB_STEP_SUMMARY
               else
                 echo "================================================================================================" >> $GITHUB_STEP_SUMMARY
-                echo "FAILED: $test (No log file found)" >> $GITHUB_STEP_SUMMARY
+                echo "FAILED: $test (No output file found)" >> $GITHUB_STEP_SUMMARY
                 echo "================================================================================================" >> $GITHUB_STEP_SUMMARY
                 echo "" >> $GITHUB_STEP_SUMMARY
               fi

examples/tutorials/10_agentic/00_base/040_other_sdks/tests/test_agent.py

@@ -114,7 +114,7 @@ async def test_send_event_and_poll_simple_query(self, client: AsyncAgentex, agen
                 break
 
         # Verify state has been updated by polling the states for 10 seconds
-        for i in range(10):
+        for i in range(20):
             if i == 9:
                 raise Exception("Timeout waiting for state updates")
             states = await client.states.list(agent_id=agent_id, task_id=task.id)
@@ -187,7 +187,12 @@ async def test_multi_turn_conversation_with_state(self, client: AsyncAgentex, ag
             sleep_interval=1.0,
         ):
             assert isinstance(message, TaskMessage)
-            if message.content and message.content.type == "text" and message.content.author == "agent" and message.content.content:
+            if (
+                message.content
+                and message.content.type == "text"
+                and message.content.author == "agent"
+                and message.content.content
+            ):
                 break
 
         ## keep polling the states for 10 seconds for the input_list and turn_number to be updated
@@ -216,7 +221,12 @@ async def test_multi_turn_conversation_with_state(self, client: AsyncAgentex, ag
             timeout=30,
             sleep_interval=1.0,
         ):
-            if message.content and message.content.type == "text" and message.content.author == "agent" and message.content.content:
+            if (
+                message.content
+                and message.content.type == "text"
+                and message.content.author == "agent"
+                and message.content.content
+            ):
                 response_text = message.content.content.lower()
                 assert "blue" in response_text
                 found_response = True
@@ -273,7 +283,10 @@ async def stream_messages() -> None:
                 # For full messages, content is at the top level
                 # For delta messages, we need to check parent_task_message
                 if msg_type == "full":
-                    if event.get("content", {}).get("type") == "text" and event.get("content", {}).get("author") == "user":
+                    if (
+                        event.get("content", {}).get("type") == "text"
+                        and event.get("content", {}).get("author") == "user"
+                    ):
                         user_message_found = True
                 elif msg_type == "done":
                     break

examples/tutorials/run_agent_test.sh

@@ -263,7 +263,7 @@ run_test() {
 
 
     # Run the tests with retry mechanism
-    local max_retries=3
+    local max_retries=5
     local retry_count=0
     local exit_code=1