merge: update settings schema branch with main

- merge latest main into the SDK settings-schema branch
- type settings metadata for pyright compatibility
- bump openhands-sdk to 1.15.0 for API breakage policy

Co-authored-by: openhands <openhands@all-hands.dev>

Author: openhands-agent

.github/actions/pr-review/action.yml

@@ -330,9 +330,16 @@ def check_model(
             **kwargs,
         )
 
-        content = response.choices[0].message.content if response.choices else None
+        response_content = (
+            response.choices[0].message.content if response.choices else None
+        )
+        reasoning_content = (
+            getattr(response.choices[0].message, "reasoning_content", None)
+            if response.choices
+            else None
+        )
 
-        if content:
+        if response_content or reasoning_content:
             return True, f"✓ {display_name}: OK"
         else:
             # Check if there's any other data in the response for diagnostics

.github/run-eval/resolve_model_config.py

@@ -2,10 +2,14 @@
 name: PR Review by OpenHands
 
 on:
-    # TEMPORARY MITIGATION (Clinejection hardening)
-    #
-    # We temporarily avoid `pull_request_target` here. We'll restore it after the PR review
-    # workflow is fully hardened for untrusted execution.
+    # Use pull_request so workflow changes can be validated in PRs.
+    # This workflow requires secrets, so the job only runs for same-repo PRs.
+    # It runs when:
+    #   1. A new PR is opened (non-draft), OR
+    #   2. A draft PR is marked as ready for review, OR
+    #   3. A maintainer adds the 'review-this' label, OR
+    #   4. A maintainer requests openhands-agent or all-hands-bot as a reviewer
+    # Adding labels and requesting reviewers still requires write access.
     pull_request:
         types: [opened, ready_for_review, labeled, review_requested]
 
@@ -16,38 +20,32 @@ permissions:
 
 jobs:
     pr-review:
-        # Note: fork PRs will not have access to repository secrets under `pull_request`.
-        # Skip forks to avoid noisy failures until we restore a hardened `pull_request_target` flow.
+        # Run when one of the following conditions is met:
+        #   1. A new non-draft PR is opened by a non-first-time contributor, OR
+        #   2. A draft PR is converted to ready for review by a non-first-time contributor, OR
+        #   3. 'review-this' label is added, OR
+        #   4. openhands-agent or all-hands-bot is requested as a reviewer
+        # Note: FIRST_TIME_CONTRIBUTOR and NONE PRs require manual trigger via label/reviewer request.
         if: |
-            github.event.pull_request.head.repo.full_name == github.repository &&
-            (
-                (github.event.action == 'opened' && github.event.pull_request.draft == false) ||
-                github.event.action == 'ready_for_review' ||
-                (github.event.action == 'labeled' && github.event.label.name == 'review-this') ||
-                (
-                    github.event.action == 'review_requested' &&
-                    (
-                        github.event.requested_reviewer.login == 'openhands-agent' ||
-                        github.event.requested_reviewer.login == 'all-hands-bot'
-                    )
-                )
+            github.event.pull_request.head.repo.full_name == github.repository && (
+                (github.event.action == 'opened' && github.event.pull_request.draft == false && github.event.pull_request.author_association != 'FIRST_TIME_CONTRIBUTOR' && github.event.pull_request.author_association != 'NONE') ||
+                (github.event.action == 'ready_for_review' && github.event.pull_request.author_association != 'FIRST_TIME_CONTRIBUTOR' && github.event.pull_request.author_association != 'NONE') ||
+                github.event.label.name == 'review-this' ||
+                github.event.requested_reviewer.login == 'openhands-agent' ||
+                github.event.requested_reviewer.login == 'all-hands-bot'
             )
         concurrency:
             group: pr-review-${{ github.event.pull_request.number }}
             cancel-in-progress: true
         runs-on: ubuntu-24.04
         steps:
             - name: Run PR Review
-              uses: OpenHands/software-agent-sdk/.github/actions/pr-review@main
+              uses: OpenHands/extensions/plugins/pr-review@main
               with:
-                  # LLM model(s) to use. Can be comma-separated for A/B testing
-                  # - one model will be randomly selected per review
                   llm-model: litellm_proxy/claude-sonnet-4-5-20250929
                   llm-base-url: https://llm-proxy.app.all-hands.dev
                   # Review style: roasted (other option: standard)
                   review-style: roasted
-                  # Use the PR's head commit SHA to test SDK changes on the SDK repo itself
-                  sdk-version: ${{ github.event.pull_request.head.sha }}
                   llm-api-key: ${{ secrets.LLM_API_KEY }}
                   github-token: ${{ secrets.ALLHANDS_BOT_GITHUB_PAT }}
                   lmnr-api-key: ${{ secrets.LMNR_SKILLS_API_KEY }}

.github/workflows/pr-review-by-openhands.yml

@@ -1,15 +1,13 @@
 ---
 name: PR Review Evaluation
 
-# This workflow runs when a PR is merged or closed to evaluate how well
-# the review agent's comments were addressed.
-# 
-# It creates an evaluation trace in Laminar that can be processed by a
-# signal to determine review effectiveness.
+# This workflow evaluates how well PR review comments were addressed.
+# It runs when a PR is closed to assess review effectiveness.
 #
-# Prerequisites:
-# - PR must have been reviewed by pr-review-by-openhands.yml first
-# - Trace info artifact must exist from the review workflow
+# Security note: pull_request_target is safe here because:
+# 1. Only triggers on PR close (not on code changes)
+# 2. Does not checkout PR code - only downloads artifacts from trusted workflow runs
+# 3. Runs evaluation scripts from the extensions repo, not from the PR
 
 on:
     pull_request_target:
@@ -21,17 +19,13 @@ permissions:
 
 jobs:
     evaluate:
-        # Only run if:
-        # 1. This is a merged PR, AND
-        # 2. The PR was previously reviewed (has the trace artifact)
         runs-on: ubuntu-24.04
         env:
             PR_NUMBER: ${{ github.event.pull_request.number }}
             REPO_NAME: ${{ github.repository }}
             PR_MERGED: ${{ github.event.pull_request.merged }}
+
         steps:
-            # Note: actions/download-artifact@v5 only works within the same workflow run.
-            # We use dawidd6/action-download-artifact to download from a different workflow.
             - name: Download review trace artifact
               id: download-trace
               uses: dawidd6/action-download-artifact@v6
@@ -43,8 +37,6 @@ jobs:
                   search_artifacts: true
                   if_no_artifact_found: warn
 
-            # Check if the trace file actually exists (the artifact download may
-            # succeed but with no matching artifact, only issuing a warning)
             - name: Check if trace file exists
               id: check-trace
               run: |
@@ -53,53 +45,41 @@ jobs:
                     echo "Found trace file for PR #$PR_NUMBER"
                   else
                     echo "trace_exists=false" >> $GITHUB_OUTPUT
-                    echo "No trace file found for PR #$PR_NUMBER"
-                    echo "This PR may not have been reviewed by the agent, skipping evaluation"
+                    echo "No trace file found for PR #$PR_NUMBER - skipping evaluation"
                   fi
 
-            - name: Checkout software-agent-sdk repository
+            # Always checkout main branch for security - cannot test script changes in PRs
+            - name: Checkout extensions repository
               if: steps.check-trace.outputs.trace_exists == 'true'
               uses: actions/checkout@v5
               with:
-                  repository: OpenHands/software-agent-sdk
-                  path: software-agent-sdk
+                  repository: OpenHands/extensions
+                  path: extensions
 
             - name: Set up Python
               if: steps.check-trace.outputs.trace_exists == 'true'
               uses: actions/setup-python@v6
               with:
-                  python-version: '3.13'
-
-            - name: Install uv
-              if: steps.check-trace.outputs.trace_exists == 'true'
-              uses: astral-sh/setup-uv@v7
-              with:
-                  enable-cache: true
+                  python-version: '3.12'
 
             - name: Install dependencies
               if: steps.check-trace.outputs.trace_exists == 'true'
-              run: |
-                  # Install lmnr SDK for Laminar integration
-                  uv pip install --system lmnr
+              run: pip install lmnr
 
             - name: Run evaluation
               if: steps.check-trace.outputs.trace_exists == 'true'
               env:
+                  # Script expects LMNR_PROJECT_API_KEY; org secret is named LMNR_SKILLS_API_KEY
                   LMNR_PROJECT_API_KEY: ${{ secrets.LMNR_SKILLS_API_KEY }}
                   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
               run: |
-                  # Copy trace info to working directory
-                  cp trace-info/laminar_trace_info.json .
-
-                  # Run the evaluation script
-                  uv run python software-agent-sdk/examples/03_github_workflows/02_pr_review/evaluate_review.py
+                  python extensions/plugins/pr-review/scripts/evaluate_review.py \
+                      --trace-file trace-info/laminar_trace_info.json
 
             - name: Upload evaluation logs
               uses: actions/upload-artifact@v5
               if: always() && steps.check-trace.outputs.trace_exists == 'true'
               with:
                   name: pr-review-evaluation-${{ github.event.pull_request.number }}
-                  path: |
-                      *.log
-                      *.json
+                  path: '*.log'
                   retention-days: 30

.github/workflows/pr-review-evaluation.yml

@@ -19,7 +19,6 @@ on:
                     - swebench
                     - swtbench
                     - commit0
-                    - multiswebench
                     - swebenchmultimodal
             sdk_ref:
                 description: SDK commit/ref to evaluate (must be a semantic version like v1.0.0 unless 'Allow unreleased branches' is checked)