diff --git a/.github/workflows/conventional-commits.yaml b/.github/workflows/verify-conventional-commits.yaml
similarity index 100%
rename from .github/workflows/conventional-commits.yaml
rename to .github/workflows/verify-conventional-commits.yaml
diff --git a/.github/workflows/verify-test-generation-prompts.yaml b/.github/workflows/verify-test-generation-prompts.yaml
new file mode 100644
index 000000000..2e1301884
--- /dev/null
+++ b/.github/workflows/verify-test-generation-prompts.yaml
@@ -0,0 +1,97 @@
+name: Verify test generation prompts
+
+on:
+ pull_request:
+ paths:
+ - ".github/workflows/verify-test-generation-prompts.yml"
+ - "shiny/pytest/_generate/**"
+ workflow_dispatch:
+
+concurrency:
+ group: "prompt-test-generation-${{ github.event.pull_request.number || 'dispatch' }}"
+ cancel-in-progress: true
+
+env:
+ PYTHON_VERSION: "3.13"
+ ATTEMPTS: 3
+ PYTHONUNBUFFERED: 1
+
+jobs:
+ verify-test-generation-prompts:
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ env.PYTHON_VERSION }}
+
+ - name: Setup py-shiny
+ id: install
+ uses: ./.github/py-shiny/setup
+
+ - name: Install Test Generator Dependencies
+ run: |
+ make ci-install-ai-deps
+
+ - name: Run Evaluation and Tests 3 Times
+ env:
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+ PYTHONUNBUFFERED: 1
+ timeout-minutes: 25
+ run: |
+ make run-test-ai-evaluation
+
+ - name: Upload test results
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: test-results-${{ github.run_id }}
+ path: |
+ test-results-inspect-ai/
+ retention-days: 7
+
+ - name: Process Results
+ timeout-minutes: 2
+ run: |
+ # Results are already averaged by the bash script, just verify they exist
+ if [ ! -f "test-results-inspect-ai/summary.json" ]; then
+ echo "No averaged summary found at test-results-inspect-ai/summary.json"
+ ls -la test-results-inspect-ai/
+ exit 1
+ else
+ echo "Using averaged results from all attempts"
+ cat test-results-inspect-ai/summary.json
+ fi
+
+ - name: Check Quality Gate
+ timeout-minutes: 2
+ run: |
+ if [ ! -f "test-results-inspect-ai/summary.json" ]; then
+ echo "Summary file not found at test-results-inspect-ai/summary.json"
+ ls -la test-results-inspect-ai/
+ exit 1
+ else
+ echo "Found summary file, checking quality gate..."
+ python tests/inspect-ai/utils/scripts/quality_gate.py test-results-inspect-ai/
+ fi
+
+ - name: Prepare Comment Body
+ if: github.event_name == 'pull_request'
+ timeout-minutes: 1
+ run: |
+ python tests/inspect-ai/scripts/prepare_comment.py test-results-inspect-ai/summary.json
+
+ - name: Comment PR Results
+ if: github.event_name == 'pull_request'
+ uses: marocchino/sticky-pull-request-comment@v2
+ with:
+ header: inspect-ai-results
+ path: comment_body.txt
diff --git a/.github/workflows/verify-testing-docs-on-change.yml b/.github/workflows/verify-testing-docs-on-change.yml
new file mode 100644
index 000000000..7667a581b
--- /dev/null
+++ b/.github/workflows/verify-testing-docs-on-change.yml
@@ -0,0 +1,93 @@
+name: Verify testing documentation for changes
+
+on:
+ pull_request:
+ paths:
+ - ".github/workflows/verify-testing-docs-on-change.yml"
+ - "docs/_quartodoc-testing.yml"
+ - "shiny/playwright/controller/**"
+
+permissions:
+ contents: write
+ pull-requests: write
+
+jobs:
+ verify-testing-docs:
+ runs-on: ubuntu-latest
+ if: github.event_name == 'pull_request'
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Setup py-shiny
+ id: install
+ uses: ./.github/py-shiny/setup
+
+ - name: Install dependencies
+ run: |
+ make ci-install-docs
+
+ - name: Update testing docs and check for changes
+ id: check-docs-changes
+ run: |
+ # Store the current state of the documentation file
+ cp shiny/pytest/_generate/_data/testing-documentation.json testing-documentation-before.json
+
+ # Run the make command to update testing docs
+ make update-testing-docs
+
+ if [[ ! -f testing-documentation-before.json || ! -f shiny/pytest/_generate/_data/testing-documentation.json ]]; then
+ echo "One or both documentation files are missing."
+ exit 1
+ fi
+
+ # Check if the documentation file has changed
+ if diff -q testing-documentation-before.json shiny/pytest/_generate/_data/testing-documentation.json > /dev/null 2>&1; then
+ echo "docs_changed=true" >> $GITHUB_OUTPUT
+ echo "The generated documentation is out of sync with the current controller changes."
+ echo "\n\n"
+ diff -q testing-documentation-before.json shiny/pytest/_generate/_data/testing-documentation.json || true
+ echo "\n\n"
+ else
+ echo "docs_changed=false" >> $GITHUB_OUTPUT
+ echo "Documentation file is up to date"
+ fi
+
+ - name: Comment on PR about testing docs update
+ if: steps.check-docs-changes.outputs.docs_changed == 'true'
+ uses: marocchino/sticky-pull-request-comment@v2
+ with:
+ header: testing-docs-update
+ message: |
+ 🚨 **Testing Documentation Out of Sync**
+
+ We detected changes in the `shiny/playwright/controller` directory that affect the testing documentation used by the `shiny add test` command.
+
+ **The generated documentation is out of sync with your controller changes. Please run:**
+
+ ```bash
+ make update-testing-docs
+ ```
+
+ **Then commit the updated `shiny/pytest/_generate/_data/testing-documentation.json` file.**
+
+ Additional details
+
+ The updated documentation file ensures that the AI test generator has access to the latest controller API documentation.
+
+ (.*?)", method_text, re.DOTALL
+ )
+ if param_code_match:
+ code_content = param_code_match.group(1)
+ params = re.findall(
+ r'(.*?)', code_content
+ )
+ return ", ".join(params)
+ return ""
+
+ params_table_text = params_match.group(1)
+ lines = params_table_text.strip().split("\n")
+
+ if len(lines) < 3:
+ return ""
+
+ param_lines = lines[2:]
+ parameters: List[str] = []
+ for line in param_lines:
+ parts = [p.strip() for p in line.strip().split("|") if p.strip()]
+ if len(parts) >= 2:
+ name = parts[0].replace("`", "")
+ type_str = re.sub(r"\[(.*?)\]\(.*?\)", r"\1", parts[1])
+ type_str = type_str.replace("`", "").replace("\n", " ")
+ parameters.append(f"{name} ({type_str})")
+
+ return ", ".join(parameters)
+
+
+def parse_qmd_content(content: str) -> Optional[Dict[str, Any]]:
+ """
+ Parses the content of a .qmd file to extract controller and method information.
+ """
+ data: Dict[str, Any] = {}
+ lines = content.strip().split("\n")
+
+ controller_match = re.match(r"# ([\w\.]+) {.*}", lines[0])
+ if not controller_match:
+ return None
+
+ data["controller_name"] = controller_match.group(1)
+ methods: List[Dict[str, Any]] = []
+ data["methods"] = methods
+
+ try:
+ methods_table_start_index = next(
+ i for i, line in enumerate(lines) if "## Methods" in line
+ )
+ except StopIteration:
+ return data
+
+ first_method_def_index = len(lines)
+ try:
+ first_method_def_index = next(
+ i
+ for i, line in enumerate(lines)
+ if line.startswith("### ") and i > methods_table_start_index
+ )
+ except StopIteration:
+ pass
+
+ methods_table_lines = lines[methods_table_start_index + 3 : first_method_def_index]
+ for line in methods_table_lines:
+ if not line.strip().startswith("|"):
+ continue
+ parts = [p.strip() for p in line.strip().split("|") if p.strip()]
+ if len(parts) < 2:
+ continue
+ method_name_md, description = parts[0], parts[1]
+ method_name_match = re.search(r"\[([\w_]+)\]", method_name_md)
+ if not method_name_match:
+ continue
+ method_name = method_name_match.group(1)
+
+ parameters_str = ""
+ method_detail_regex = re.compile(
+ r"### " + re.escape(method_name) + r" {.*?}(.*?)(?=\n### |\Z)", re.DOTALL
+ )
+ method_detail_match = method_detail_regex.search(content)
+
+ if method_detail_match:
+ method_text = method_detail_match.group(1)
+ parameters_str = parse_parameters_from_text(method_text)
+
+ methods.append(
+ {
+ "name": method_name,
+ "description": description.strip(),
+ "parameters": parameters_str,
+ }
+ )
+ return data
+
+
+def convert_xml_to_json(xml_file_path: Path) -> str:
+ """
+ Parses an XML file containing multiple .qmd docs and converts it to a
+ structured JSON object containing controller and method information.
+ """
+ try:
+ with open(xml_file_path, "r", encoding="utf-8") as f:
+ xml_content = f.read()
+
+ def cdata_replacer(match: re.Match[str]) -> str:
+ path = match.group(1)
+ content = match.group(2)
+ content = content.replace("]]>", "]]>")
+ return f''
+
+ xml_content_cdata = re.sub(
+ r'(.*?)',
+ cdata_replacer,
+ xml_content,
+ flags=re.DOTALL,
+ )
+
+ rooted_xml_content = f"{xml_content_cdata}"
+
+ root = ET.fromstring(rooted_xml_content)
+
+ except (ET.ParseError, FileNotFoundError) as e:
+ return json.dumps({"error": str(e)}, indent=2)
+
+ all_controllers_data: List[Dict[str, Any]] = []
+ files_element = root.find("files")
+
+ if files_element is None:
+ return json.dumps({"error": "No element found in XML"}, indent=2)
+
+ for file_elem in files_element.findall("file"):
+ path = file_elem.get("path")
+ if path and (
+ path.startswith("playwright.controller.") or path == "run.ShinyAppProc.qmd"
+ ):
+ content = file_elem.text
+ if content:
+ controller_data = parse_qmd_content(content)
+ if controller_data and controller_data.get("methods"):
+ all_controllers_data.append(controller_data)
+
+ all_controllers_data.sort(key=lambda x: x.get("controller_name", ""))
+
+ return json.dumps(all_controllers_data, indent=2)
+
+
+def main() -> None:
+ """Main entry point for the application."""
+ args = parse_arguments()
+
+ try:
+ input_path, output_path = validate_arguments(args)
+ except SystemExit:
+ return
+
+ print(f"Starting conversion of '{input_path}' to '{output_path}'")
+
+ try:
+ json_output_string = convert_xml_to_json(input_path)
+
+ with open(output_path, "w", encoding="utf-8") as f:
+ f.write(json_output_string)
+
+ print(f"Conversion complete. Output saved to '{output_path}'")
+
+ except Exception as e:
+ print(f"Error during conversion: {e}", file=sys.stderr)
+ sys.exit(1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/inspect-ai/utils/scripts/process_results.py b/tests/inspect-ai/utils/scripts/process_results.py
new file mode 100644
index 000000000..0e7b0796c
--- /dev/null
+++ b/tests/inspect-ai/utils/scripts/process_results.py
@@ -0,0 +1,90 @@
+import json
+import sys
+from pathlib import Path
+from typing import Any, Dict, List, Union
+
+
+def process_inspect_results(result_file_path: Union[str, Path]) -> None:
+ """Process a single Inspect AI result file and generate a summary."""
+ input_path = Path(result_file_path)
+
+ # 1. Validate that the input path is a valid .json file
+ if not input_path.is_file() or input_path.suffix.lower() != ".json":
+ print(f"Error: The provided path is not a valid .json file: {input_path}")
+ sys.exit(1)
+
+ print(f"Processing file: {input_path.name}")
+
+ # 2. Load the JSON data with error handling
+ with open(input_path, "r", encoding="utf-8") as f:
+ try:
+ data: Dict[str, Any] = json.load(f)
+ except json.JSONDecodeError as e:
+ print(f"Error decoding JSON from file {input_path}: {e}")
+ sys.exit(1)
+
+ # 3. Extract the list of samples from the top-level 'samples' key
+ samples: List[Dict[str, Any]] = data.get("samples", [])
+ if not isinstance(samples, list):
+ print(f"Error: 'samples' key in {input_path} is not a list.")
+ sys.exit(1)
+
+ total_tests = len(samples)
+
+ if total_tests == 0:
+ print(f"No samples found in the result file: {input_path}")
+
+ # 4. Correctly count tests based on the 'value' within scores.model_graded_qa
+ passed_tests = sum(
+ 1
+ for s in samples
+ if s.get("scores", {}).get("model_graded_qa", {}).get("value") == "C"
+ )
+ partial_tests = sum(
+ 1
+ for s in samples
+ if s.get("scores", {}).get("model_graded_qa", {}).get("value") == "P"
+ )
+ failed_tests = sum(
+ 1
+ for s in samples
+ if s.get("scores", {}).get("model_graded_qa", {}).get("value") == "I"
+ )
+
+ # Calculate pass rate including both Complete and Partial grades
+ passing_tests = passed_tests + partial_tests
+ pass_rate = (passing_tests / total_tests) * 100 if total_tests > 0 else 0
+
+ # Generate summary dictionary
+ summary = {
+ "total": total_tests,
+ "passed": passed_tests,
+ "partial": partial_tests,
+ "failed": failed_tests,
+ "pass_rate": pass_rate,
+ "quality_gate_passed": pass_rate >= 80, # 80% threshold
+ "details": (
+ f"Complete: {passed_tests}, Partial: {partial_tests}, "
+ f"Incomplete: {failed_tests}, Passing: {passing_tests}/{total_tests}"
+ ),
+ }
+
+ # 5. Save the summary in the same directory as the input file
+ summary_file_path = input_path.parent / "summary.json"
+ with open(summary_file_path, "w") as f:
+ json.dump(summary, f, indent=2)
+
+ print(f"\nSummary saved to: {summary_file_path}")
+ print(
+ f"Processed {total_tests} tests: {passed_tests} complete, "
+ f"{partial_tests} partial, {failed_tests} incomplete"
+ )
+ print(f"Pass rate (Complete + Partial): {pass_rate:.1f}%")
+
+
+if __name__ == "__main__":
+ if len(sys.argv) != 2:
+ print("Usage: python process_results.py ")
+ sys.exit(1)
+
+ process_inspect_results(sys.argv[1])
diff --git a/tests/inspect-ai/utils/scripts/quality_gate.py b/tests/inspect-ai/utils/scripts/quality_gate.py
new file mode 100644
index 000000000..8c9fab7bb
--- /dev/null
+++ b/tests/inspect-ai/utils/scripts/quality_gate.py
@@ -0,0 +1,48 @@
+import json
+import sys
+from pathlib import Path
+from typing import Any, Dict, Union
+
+
+def check_quality_gate(results_dir: Union[str, Path], threshold: float = 80) -> None:
+ """
+ Check if evaluation results meet quality gate.
+
+ The quality gate is based on the pass_rate from the summary.json file.
+ Pass rate includes both 'Complete' (C) and 'Partial' (P) grades.
+ Tests with 'Incomplete' (I) grade do not count towards the pass rate.
+
+ Args:
+ results_dir: Directory containing the summary.json file
+ threshold: Minimum pass rate percentage required (default: 80%)
+ """
+
+ summary_path = Path(results_dir) / "summary.json"
+
+ if not summary_path.exists():
+ print("Summary file not found")
+ sys.exit(1)
+
+ with open(summary_path, "r") as f:
+ summary: Dict[str, Any] = json.load(f)
+
+ pass_rate = summary.get("pass_rate", 0)
+
+ if pass_rate >= threshold:
+ print(
+ f"✅ Quality gate PASSED: {pass_rate:.1f}% >= {threshold}% (Complete + Partial grades)"
+ )
+ sys.exit(0)
+ else:
+ print(
+ f"❌ Quality gate FAILED: {pass_rate:.1f}% < {threshold}% (Complete + Partial grades)"
+ )
+ sys.exit(1)
+
+
+if __name__ == "__main__":
+ if len(sys.argv) != 2:
+ print("Usage: python quality_gate.py ")
+ sys.exit(1)
+
+ check_quality_gate(sys.argv[1])
diff --git a/tests/playwright/ai_generated_apps/bookmark/navsets/test_navsets_express_bookmarking.py b/tests/playwright/ai_generated_apps/bookmark/navsets/test_navsets_express_bookmarking.py
index e0df4fee5..09bb7f644 100644
--- a/tests/playwright/ai_generated_apps/bookmark/navsets/test_navsets_express_bookmarking.py
+++ b/tests/playwright/ai_generated_apps/bookmark/navsets/test_navsets_express_bookmarking.py
@@ -35,13 +35,18 @@ def test_navsets_bookmarking_demo(
# Non-module navsets
navset_collection = controller.NavsetTab(page, "navsets_collection")
navset_collection.set(navset_name)
- navset_cont = navset_controller(page, f"{navset_name}_{navset_variant}")
+ navset_cont = navset_controller(
+ page, f"{navset_name}_{navset_variant}" # pyright: ignore[reportCallIssue]
+ )
navset_cont.set(f"{navset_name}_c")
# Module navsets
mod_navset_collection = controller.NavsetTab(page, "first-navsets_collection")
mod_navset_collection.set(navset_name)
- mod_navset_cont = navset_controller(page, f"first-{navset_name}_{navset_variant}")
+ mod_navset_cont = navset_controller(
+ page,
+ f"first-{navset_name}_{navset_variant}", # pyright: ignore[reportCallIssue]
+ ) # pyright: ignore[reportCallIssue]
mod_navset_cont.set(f"{navset_name}_b")
existing_url = page.url
diff --git a/tests/playwright/ai_generated_apps/bookmark/navsets/test_navsets_hidden_bookmarking.py b/tests/playwright/ai_generated_apps/bookmark/navsets/test_navsets_hidden_bookmarking.py
index 44e95ce24..c7a6c9d93 100644
--- a/tests/playwright/ai_generated_apps/bookmark/navsets/test_navsets_hidden_bookmarking.py
+++ b/tests/playwright/ai_generated_apps/bookmark/navsets/test_navsets_hidden_bookmarking.py
@@ -29,7 +29,7 @@ def test_navset_hidden_bookmarking(
navset_collection = controller.NavsetTab(page, "navsets_collection")
navset_collection.set(navset_name)
navset_id = f"{navset_name}_{navset_variant}"
- navset_cont = navset_controller(page, navset_id)
+ navset_cont = navset_controller(page, navset_id) # pyright: ignore[reportCallIssue]
navset_btn = controller.InputActionButton(page, f"{navset_id}_button")
navset_btn.click()
navset_btn.click()
@@ -37,7 +37,9 @@ def test_navset_hidden_bookmarking(
# Module navsets
mod_navset_collection = controller.NavsetTab(page, "first-navsets_collection")
mod_navset_collection.set(navset_name)
- mod_navset_cont = navset_controller(page, f"first-{navset_id}")
+ mod_navset_cont = navset_controller(
+ page, f"first-{navset_id}" # pyright: ignore[reportCallIssue]
+ )
mod_navset_btn = controller.InputActionButton(page, f"first-{navset_id}_button")
mod_navset_btn.click()