run-examples.yml

---
name: Run Examples Scripts

on:
    pull_request:
        types: [labeled]
    workflow_dispatch:
        inputs:
            reason:
                description: Reason for manual trigger
                required: true
                default: ''
    schedule:
        - cron: 30 22 * * * # Runs at 10:30pm UTC every day

permissions:
    contents: read
    pull-requests: write
    issues: write

jobs:
    test-examples:
        # Schedule trigger only runs in the main repository, not in forks
        if: github.event.label.name == 'test-examples' || github.event_name == 'workflow_dispatch' || (github.event_name == 'schedule' && 
            github.repository == 'OpenHands/software-agent-sdk')
        runs-on: ubuntu-24.04
        timeout-minutes: 60
        steps:
            - name: Wait for agent server to finish build
              if: github.event_name == 'pull_request'
              uses: lewagon/wait-on-check-action@v1.6.1
              with:
                  ref: ${{ github.event.pull_request.head.ref }}
                  check-name: Build & Push (python-amd64)
                  repo-token: ${{ secrets.GITHUB_TOKEN }}
                  wait-interval: 10

            - name: Checkout
              uses: actions/checkout@v6
              with:
                  ref: ${{ github.event.pull_request.head.ref }}
                  repository: ${{ github.event.pull_request.head.repo.full_name }}

            - name: Install uv
              uses: astral-sh/setup-uv@v7
              with:
                  enable-cache: true
                  python-version: '3.13'

            - name: Install Node.js
              uses: actions/setup-node@v6
              with:
                  node-version: '22'

            - name: Setup Apptainer
              uses: eWaterCycle/setup-apptainer@v2
              with:
                  apptainer-version: 1.3.6

            - name: Install Chromium
              run: |
                  sudo apt-get update
                  sudo apt-get install -y chromium-browser

            - name: Install dependencies
              run: uv sync --frozen --group dev

            - name: Run examples
              shell: bash
              env:
                  LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
                  LLM_MODEL: openhands/claude-haiku-4-5-20251001
                  LLM_BASE_URL: https://llm-proxy.app.all-hands.dev
                  RUNTIME_API_KEY: ${{ secrets.RUNTIME_API_KEY }}
                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
                  PR_NUMBER: ${{ github.event.pull_request.number }}
                  REPO_OWNER: ${{ github.repository_owner }}
                  REPO_NAME: ${{ github.event.repository.name }}
                  GITHUB_SHA: ${{ github.event.pull_request.head.sha }}
                  OPENHANDS_CLOUD_API_KEY: ${{ secrets.ALLHANDS_BOT_OPENHANDS_SAAS_API_KEY }}
                  # ACP agents (Claude Code, Codex) route through LiteLLM proxy
                  ANTHROPIC_BASE_URL: https://llm-proxy.app.all-hands.dev
                  ANTHROPIC_API_KEY: ${{ secrets.LLM_API_KEY }}
                  OPENAI_BASE_URL: https://llm-proxy.app.all-hands.dev
                  OPENAI_API_KEY: ${{ secrets.LLM_API_KEY }}
              run: |
                  RESULTS_DIR=".example-test-results"
                  REPORT_PATH="examples_report.md"
                  rm -rf "$RESULTS_DIR"
                  mkdir -p "$RESULTS_DIR"

                  update_comment() {
                      if [ -z "$API_URL" ]; then
                          echo "Skipping PR comment update because API_URL is unset."
                          return
                      fi

                      local comment_body="$1"
                      local payload
                      local response

                      payload=$(jq -n --arg body "$comment_body" '{body: $body}')

                      if [ -z "$COMMENT_ID" ]; then
                          echo "Creating PR comment..."
                          if ! response=$(curl -sSf -X POST \
                              -H "Authorization: token ${GITHUB_TOKEN}" \
                              -H "Accept: application/vnd.github.v3+json" \
                              -H "Content-Type: application/json" \
                              "${API_URL}" \
                              -d "$payload"); then
                              echo "::error::Failed to create PR comment."
                              exit 1
                          fi
                          COMMENT_ID=$(echo "$response" | jq -r '.id // ""')
                          if [ -z "$COMMENT_ID" ]; then
                              echo "::error::GitHub API response did not include a comment id: $response"
                              exit 1
                          fi
                          echo "Created comment with ID: $COMMENT_ID"
                      else
                          echo "Updating PR comment (ID: $COMMENT_ID)..."
                          if ! curl -sSf -X PATCH \
                              -H "Authorization: token ${GITHUB_TOKEN}" \
                              -H "Accept: application/vnd.github.v3+json" \
                              -H "Content-Type: application/json" \
                              "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/issues/comments/${COMMENT_ID}" \
                              -d "$payload" > /dev/null; then
                              echo "::error::Failed to update PR comment (ID: $COMMENT_ID)."
                              exit 1
                          fi
                      fi
                  }

                  API_URL=""
                  COMMENT_ID=""

                  if [ "${{ github.event_name }}" = "pull_request" ]; then
                      API_URL="https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/issues/${PR_NUMBER}/comments"
                      initial_comment="## 🔄 Running Examples with \`${LLM_MODEL}\`"
                      initial_comment+=$'\n\n'
                      initial_comment+="_Run in progress..._"
                      initial_comment+=$'\n'
                      update_comment "$initial_comment"
                  fi

                  EXIT_CODE=0
                  uv run pytest tests/examples/test_examples.py \
                      --run-examples \
                      --examples-results-dir "$RESULTS_DIR" \
                      -n 4 || EXIT_CODE=$?

                  TIMESTAMP="$(date -u '+%Y-%m-%d %H:%M:%S UTC')"
                  WORKFLOW_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"

                  uv run python scripts/render_examples_report.py \
                      --results-dir "$RESULTS_DIR" \
                      --model "$LLM_MODEL" \
                      --workflow-url "$WORKFLOW_URL" \
                      --timestamp "$TIMESTAMP" \
                      --output "$REPORT_PATH"

                  COMMENT_BODY="$(cat "$REPORT_PATH")"
                  echo "$COMMENT_BODY"

                  if [ "${{ github.event_name }}" = "pull_request" ]; then
                      echo "Publishing PR comment..."
                      update_comment "$COMMENT_BODY"
                  fi

                  if [ $EXIT_CODE -ne 0 ]; then
                      exit $EXIT_CODE
                  fi
            - name: Read examples report for issue comment
              if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
              id: read_report
              shell: bash
              run: |
                  if [ -f examples_report.md ]; then
                      REPORT_CONTENT=$(cat examples_report.md)
                      echo "report<<EOF" >> "$GITHUB_OUTPUT"
                      echo "$REPORT_CONTENT" >> "$GITHUB_OUTPUT"
                      echo "EOF" >> "$GITHUB_OUTPUT"
                  else
                      echo "report=Report file not found" >> "$GITHUB_OUTPUT"
                  fi

            - name: Comment with results on tracker issue
              if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
              uses: KeisukeYamashita/create-comment@v1
              with:
                  number: 976
                  unique: false
                  comment: |
                      **Trigger:** ${{ github.event_name == 'schedule' && 'Nightly Scheduled Run' || format('Manual Trigger: {0}', github.event.inputs.reason) }}
                      **Commit:** ${{ github.sha }}
                      **Workflow Run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

                      ${{ steps.read_report.outputs.report }}