Skip to content

Commit 1825e67

Browse files
xingyaowwopenhands-agentenyst
authored
feat: add experimental QA changes workflow (#2717)
Co-authored-by: openhands <openhands@all-hands.dev> Co-authored-by: Engel Nyst <engel.nyst@gmail.com>
1 parent 79f77af commit 1825e67

File tree

2 files changed

+141
-0
lines changed

2 files changed

+141
-0
lines changed

‎.github/workflows/qa-changes-by-openhands.yml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
---
2+
# EXPERIMENTAL: Automated QA validation of PR changes using OpenHands.
3+
#
4+
# Unlike pr-review (which reads diffs and posts code-review comments),
5+
# this workflow actually runs the code — setting up the environment,
6+
# executing tests, exercising changed behavior, and posting a structured
7+
# QA report as a PR comment.
8+
#
9+
# This is an early experiment; expect rough edges. The plugin source is
10+
# pinned to the extensions feature branch while we iterate.
11+
name: QA Changes by OpenHands [experimental]
12+
13+
on:
14+
pull_request:
15+
types: [opened, ready_for_review, labeled, review_requested]
16+
17+
permissions:
18+
contents: read
19+
pull-requests: write
20+
issues: write
21+
22+
jobs:
23+
qa-changes:
24+
# Only run for same-repo PRs (secrets aren't available for forks).
25+
# Trigger conditions mirror pr-review, but use the 'qa-this' label
26+
# and openhands-agent reviewer request.
27+
if: |
28+
github.event.pull_request.head.repo.full_name == github.repository && (
29+
(github.event.action == 'opened' && github.event.pull_request.draft == false && github.event.pull_request.author_association != 'FIRST_TIME_CONTRIBUTOR' && github.event.pull_request.author_association != 'NONE') ||
30+
(github.event.action == 'ready_for_review' && github.event.pull_request.author_association != 'FIRST_TIME_CONTRIBUTOR' && github.event.pull_request.author_association != 'NONE') ||
31+
github.event.label.name == 'qa-this' ||
32+
github.event.requested_reviewer.login == 'openhands-agent' ||
33+
github.event.requested_reviewer.login == 'all-hands-bot'
34+
)
35+
concurrency:
36+
group: qa-changes-${{ github.event.pull_request.number }}
37+
cancel-in-progress: true
38+
runs-on: ubuntu-24.04
39+
timeout-minutes: 30
40+
steps:
41+
- name: Run QA Changes
42+
# EXPERIMENTAL: pointing at feature branch while iterating
43+
uses: OpenHands/extensions/plugins/qa-changes@feat/qa-changes-plugin
44+
with:
45+
llm-model: litellm_proxy/claude-sonnet-4-5-20250929
46+
llm-base-url: https://llm-proxy.app.all-hands.dev
47+
max-budget: '10.0'
48+
timeout-minutes: '30'
49+
max-iterations: '500'
50+
# EXPERIMENTAL: use the feature branch of extensions
51+
extensions-version: feat/qa-changes-plugin
52+
llm-api-key: ${{ secrets.LLM_API_KEY }}
53+
github-token: ${{ secrets.PAT_TOKEN }}
54+
lmnr-api-key: ${{ secrets.LMNR_SKILLS_API_KEY }}

‎.github/workflows/qa-changes-evaluation.yml‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
---
2+
name: QA Changes Evaluation [experimental]
3+
4+
# This workflow evaluates how well QA validation performed.
5+
# It runs when a PR is closed to assess QA effectiveness.
6+
#
7+
# Security note: pull_request_target is safe here because this workflow
8+
# never checks out or executes PR code. It only:
9+
# 1. Downloads artifacts produced by a trusted workflow run
10+
# 2. Runs evaluation scripts from the extensions repo (main/pinned branch)
11+
12+
on:
13+
pull_request_target:
14+
types: [closed]
15+
16+
permissions:
17+
contents: read
18+
pull-requests: read
19+
20+
jobs:
21+
evaluate:
22+
runs-on: ubuntu-24.04
23+
env:
24+
PR_NUMBER: ${{ github.event.pull_request.number }}
25+
REPO_NAME: ${{ github.repository }}
26+
PR_MERGED: ${{ github.event.pull_request.merged }}
27+
28+
steps:
29+
- name: Download QA trace artifact
30+
id: download-trace
31+
uses: dawidd6/action-download-artifact@v19
32+
continue-on-error: true
33+
with:
34+
workflow: qa-changes-by-openhands.yml
35+
name: qa-changes-trace-${{ github.event.pull_request.number }}
36+
path: trace-info
37+
search_artifacts: true
38+
if_no_artifact_found: warn
39+
40+
- name: Check if trace file exists
41+
id: check-trace
42+
run: |
43+
if [ -f "trace-info/laminar_trace_info.json" ]; then
44+
echo "trace_exists=true" >> $GITHUB_OUTPUT
45+
echo "Found trace file for PR #$PR_NUMBER"
46+
else
47+
echo "trace_exists=false" >> $GITHUB_OUTPUT
48+
echo "No trace file found for PR #$PR_NUMBER - skipping evaluation"
49+
fi
50+
51+
# EXPERIMENTAL: pinned to feature branch while qa-changes plugin is in development.
52+
# Switch to @main (and remove ref:) once the plugin is merged.
53+
- name: Checkout extensions repository
54+
if: steps.check-trace.outputs.trace_exists == 'true'
55+
uses: actions/checkout@v6
56+
with:
57+
repository: OpenHands/extensions
58+
ref: feat/qa-changes-plugin
59+
path: extensions
60+
61+
- name: Set up Python
62+
if: steps.check-trace.outputs.trace_exists == 'true'
63+
uses: actions/setup-python@v6
64+
with:
65+
python-version: '3.12'
66+
67+
- name: Install dependencies
68+
if: steps.check-trace.outputs.trace_exists == 'true'
69+
run: pip install lmnr
70+
71+
- name: Run evaluation
72+
if: steps.check-trace.outputs.trace_exists == 'true'
73+
env:
74+
# Script expects LMNR_PROJECT_API_KEY; org secret is named LMNR_SKILLS_API_KEY
75+
LMNR_PROJECT_API_KEY: ${{ secrets.LMNR_SKILLS_API_KEY }}
76+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
77+
run: |
78+
python extensions/plugins/qa-changes/scripts/evaluate_qa_changes.py \
79+
--trace-file trace-info/laminar_trace_info.json
80+
81+
- name: Upload evaluation logs
82+
uses: actions/upload-artifact@v7
83+
if: always() && steps.check-trace.outputs.trace_exists == 'true'
84+
with:
85+
name: qa-changes-evaluation-${{ github.event.pull_request.number }}
86+
path: '*.log'
87+
retention-days: 30

0 commit comments

Comments
 (0)