chore(deps): bump lewagon/wait-on-check-action from 1.6.0 to 1.6.1 #456
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Run Condenser Tests | |
| on: | |
| # Use pull_request_target to access secrets even on fork PRs | |
| # This is safe because we only run when the 'condenser-test' label is added by a maintainer | |
| pull_request_target: | |
| types: | |
| - labeled | |
| workflow_dispatch: | |
| inputs: | |
| reason: | |
| description: Reason for manual trigger | |
| required: true | |
| default: '' | |
| env: | |
| N_PROCESSES: 2 # Fewer parallel processes for condenser tests (only 2 LLMs) | |
| jobs: | |
| post-initial-comment: | |
| if: > | |
| github.event_name == 'pull_request_target' && | |
| github.event.label.name == 'condenser-test' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - name: Comment on PR | |
| uses: KeisukeYamashita/create-comment@v1 | |
| with: | |
| unique: false | |
| comment: | | |
| Hi! I started running the condenser tests on your PR. You will receive a comment with the results shortly. | |
| Note: These are non-blocking tests that validate condenser functionality across different LLMs. | |
| run-condenser-tests: | |
| # Security: Only run when condenser-test label is present or via workflow_dispatch | |
| # This prevents automatic execution on fork PRs without maintainer approval | |
| if: | | |
| always() && ( | |
| ( | |
| github.event_name == 'pull_request_target' && | |
| github.event.label.name == 'condenser-test' | |
| ) || | |
| github.event_name == 'workflow_dispatch' | |
| ) | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| contents: read | |
| id-token: write | |
| pull-requests: write | |
| strategy: | |
| matrix: | |
| python-version: ['3.13'] | |
| job-config: | |
| # Only run against 2 LLMs for condenser tests: | |
| # - Claude Opus 4.5 (primary - supports thinking blocks) | |
| # - GPT-5.1 Codex Max (secondary - cross-LLM validation) | |
| - name: Claude Opus 4.5 | |
| run-suffix: opus_condenser_run | |
| llm-config: | |
| model: litellm_proxy/anthropic/claude-opus-4-5-20251101 | |
| extended_thinking: true | |
| - name: GPT-5.1 Codex Max | |
| run-suffix: gpt51_condenser_run | |
| llm-config: | |
| model: litellm_proxy/gpt-5.1-codex-max | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| with: | |
| # For pull_request_target: checkout fork PR code (requires explicit repository) | |
| # For other events: fallback to current repository and ref | |
| repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | |
| ref: ${{ github.event.pull_request.head.sha || github.ref }} | |
| # Security: Don't persist credentials to prevent untrusted PR code from using them | |
| persist-credentials: false | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| version: latest | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Python dependencies using uv | |
| run: | | |
| uv sync --dev | |
| uv pip install pytest | |
| - name: Run condenser test evaluation for ${{ matrix.job-config.name }} | |
| env: | |
| LLM_CONFIG: ${{ toJson(matrix.job-config.llm-config) }} | |
| LLM_API_KEY: ${{ secrets.LLM_API_KEY }} | |
| LLM_BASE_URL: https://llm-proxy.app.all-hands.dev | |
| run: | | |
| set -eo pipefail | |
| AGENT_SDK_VERSION=$(git rev-parse --short HEAD) | |
| EVAL_NOTE="${AGENT_SDK_VERSION}_${{ matrix.job-config.run-suffix }}" | |
| echo "Running condenser tests only (c*.py pattern)" | |
| uv run python tests/integration/run_infer.py \ | |
| --llm-config "$LLM_CONFIG" \ | |
| --num-workers $N_PROCESSES \ | |
| --eval-note "$EVAL_NOTE" \ | |
| --test-type condenser | |
| # get condenser tests JSON results | |
| RESULTS_FILE=$(find tests/integration/outputs/*${{ matrix.job-config.run-suffix }}* -name "results.json" -type f | head -n 1) | |
| echo "RESULTS_FILE: $RESULTS_FILE" | |
| if [ -f "$RESULTS_FILE" ]; then | |
| echo "JSON_RESULTS_FILE=$RESULTS_FILE" >> $GITHUB_ENV | |
| else | |
| echo "JSON_RESULTS_FILE=" >> $GITHUB_ENV | |
| fi | |
| - name: Wait a little bit | |
| run: sleep 10 | |
| - name: Create archive of evaluation outputs | |
| run: | | |
| TIMESTAMP=$(date +'%y-%m-%d-%H-%M') | |
| cd tests/integration/outputs # Change to the outputs directory | |
| tar -czvf ../../../condenser_tests_${{ matrix.job-config.run-suffix }}_${TIMESTAMP}.tar.gz *${{ matrix.job-config.run-suffix }}* # Include result directories for this model | |
| - name: Upload evaluation results as artifact | |
| uses: actions/upload-artifact@v7 | |
| id: upload_results_artifact | |
| with: | |
| name: condenser-test-outputs-${{ matrix.job-config.run-suffix }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: condenser_tests_${{ matrix.job-config.run-suffix }}_*.tar.gz | |
| - name: Save test results for consolidation | |
| run: | | |
| # Copy the structured JSON results file for consolidation | |
| mkdir -p test_results_summary | |
| if [ -n "${{ env.JSON_RESULTS_FILE }}" ] && [ -f "${{ env.JSON_RESULTS_FILE }}" ]; then | |
| # Copy the JSON results file directly | |
| cp "${{ env.JSON_RESULTS_FILE }}" "test_results_summary/${{ matrix.job-config.run-suffix }}_results.json" | |
| echo "✓ Copied JSON results file for consolidation" | |
| else | |
| echo "✗ No JSON results file found" | |
| exit 1 | |
| fi | |
| - name: Upload test results summary | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: test-results-${{ matrix.job-config.run-suffix }} | |
| path: test_results_summary/${{ matrix.job-config.run-suffix }}_results.json | |
| consolidate-results: | |
| needs: run-condenser-tests | |
| if: | | |
| always() && ( | |
| ( | |
| github.event_name == 'pull_request_target' && | |
| github.event.label.name == 'condenser-test' | |
| ) || | |
| github.event_name == 'workflow_dispatch' | |
| ) | |
| runs-on: ubuntu-24.04 | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| with: | |
| # When using pull_request_target, explicitly checkout the PR branch | |
| # This ensures we use the scripts from the actual PR code | |
| ref: ${{ github.event.pull_request.head.sha || github.ref }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| version: latest | |
| python-version: '3.13' | |
| - name: Install Python dependencies using uv | |
| run: | | |
| uv sync --dev | |
| - name: Download all test results | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: test-results-* | |
| merge-multiple: true | |
| path: all_results | |
| - name: Download all condenser test artifacts | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: condenser-test-outputs-* | |
| path: artifacts | |
| - name: Consolidate test results | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| MANUAL_REASON: ${{ github.event.inputs.reason }} | |
| COMMIT_SHA: ${{ github.sha }} | |
| PYTHONPATH: ${{ github.workspace }} | |
| GITHUB_SERVER_URL: ${{ github.server_url }} | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| GITHUB_RUN_ID: ${{ github.run_id }} | |
| run: | | |
| uv run python tests/integration/utils/consolidate_json_results.py \ | |
| --results-dir all_results \ | |
| --artifacts-dir artifacts \ | |
| --output-file consolidated_results.json | |
| echo "Consolidated results generated successfully" | |
| uv run python tests/integration/utils/generate_markdown_report.py \ | |
| --input-file consolidated_results.json \ | |
| --output-file consolidated_report.md | |
| - name: Upload consolidated report | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: consolidated-condenser-report | |
| path: consolidated_report.md | |
| - name: Create consolidated PR comment | |
| if: github.event_name == 'pull_request_target' | |
| run: | | |
| # Add header to clarify these are non-blocking tests | |
| echo "## Condenser Test Results (Non-Blocking)" > final_report.md | |
| echo "" >> final_report.md | |
| echo "> These tests validate condenser functionality and do not block PR merges." >> final_report.md | |
| echo "" >> final_report.md | |
| cat consolidated_report.md >> final_report.md | |
| # Sanitize @OpenHands mentions to prevent self-mention loops | |
| COMMENT_BODY=$(uv run python -c "from openhands.sdk.utils.github import sanitize_openhands_mentions; import sys; print(sanitize_openhands_mentions(sys.stdin.read()), end='')" < final_report.md) | |
| # Use GitHub CLI to create comment with explicit PR number | |
| echo "$COMMENT_BODY" | gh pr comment ${{ github.event.pull_request.number }} --body-file - | |
| env: | |
| GH_TOKEN: ${{ github.token }} |