Add AI-powered Shiny test generation and evaluation

Introduces an AI-powered test generator (with CLI integration), evaluation suite with sample apps and scripts, and utility tools for documentation and quality control. Updates the CLI to support 'shiny generate test' for automated test creation using Anthropic or OpenAI models. Adds extensive documentation and example apps for robust evaluation and development workflows.

Author: karangattu

.github/workflows/testing-docs-update.yml

@@ -0,0 +1,62 @@
+name: Testing Documentation Update
+
+on:
+  push:
+    paths:
+      - 'docs/api/testing/**'
+      - 'docs/_quartodoc-testing.yml'
+  pull_request:
+    paths:
+      - 'docs/api/testing/**'
+      - 'docs/_quartodoc-testing.yml'
+
+jobs:
+  update-testing-docs:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '25'
+          cache: 'npm'
+
+      - name: Install repomix
+        run: npm install -g repomix
+
+      - name: Verify repomix installation
+        run: repomix --version
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          make install-deps
+          make install
+
+      - name: Run testing documentation processing
+        run: |
+          echo "Processing testing documentation changes..."
+          repomix docs/api/testing -o shiny/testing/utils/scripts/repomix-output-testing.xml
+          python shiny/testing/utils/scripts/process_docs.py --input shiny/testing/utils/scripts/repomix-output-testing.xml --output shiny/testing/generator/data/docs/documentation_testing.json
+
+      - name: Check for changes
+        id: git-check
+        run: |
+          git diff --exit-code || echo "changes=true" >> $GITHUB_OUTPUT
+
+      - name: Commit and push changes
+        if: steps.git-check.outputs.changes == 'true'
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "GitHub Action"
+          git add .
+          git commit -m "Auto-update testing documentation" || exit 0
+          git push

pyrightconfig.json

@@ -11,6 +11,9 @@
     "tests/playwright/deploys/*/app.py",
     "shiny/templates",
     "tests/playwright/ai_generated_apps",
+    "shiny/testing/evaluation",
+    "shiny/testing/generator",
+    "shiny/testing/utils",
   ],
   "typeCheckingMode": "strict",
   "reportImportCycles": "none",

shiny/_main.py

@@ -530,6 +530,11 @@ def add() -> None:
     pass
 
 
+@main.group(help="""Generate files for your Shiny app using AI.""")
+def generate() -> None:
+    pass
+
+
 @add.command(
     help="""Add a test file for a specified Shiny app.
 
@@ -564,6 +569,52 @@ def test(
     add_test_file(app_file=app, test_file=test_file)
 
 
+@generate.command(
+    "test",
+    help="""Generate AI-powered test file for a specified Shiny app.
+
+Generate a comprehensive test file for a specified app using AI. The generator
+will analyze your app code and create appropriate test cases with assertions.
+
+After creating the test file, you can use `pytest` to run the tests:
+
+        pytest TEST_FILE
+""",
+)
+@click.option(
+    "--app",
+    "-a",
+    type=str,
+    help="Path to the app file for which you want to generate a test file.",
+)
+@click.option(
+    "--output",
+    "-o",
+    type=str,
+    help="Path for the generated test file. If not provided, will be auto-generated.",
+)
+@click.option(
+    "--provider",
+    type=click.Choice(["anthropic", "openai"]),
+    default="anthropic",
+    help="AI provider to use for test generation.",
+)
+@click.option(
+    "--model",
+    type=str,
+    help="Specific model to use (optional). Examples: haiku3.5, sonnet, gpt-4.1, o3-mini",
+)
+def test_generate(
+    app: str | None,
+    output: str | None,
+    provider: str,
+    model: str | None,
+) -> None:
+    from ._main_generate_test import generate_test_file
+
+    generate_test_file(app_file=app, output_file=output, provider=provider, model=model)
+
+
 @main.command(
     help="""Create a Shiny application from a template.
 

shiny/_main_generate_test.py

@@ -0,0 +1,124 @@
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import click
+import questionary
+
+from ._main_utils import cli_action, cli_bold, cli_code, path_rel_wd
+
+
+def generate_test_file(
+    *,
+    app_file: str | None,
+    output_file: str | None,
+    provider: str,
+    model: str | None,
+):
+    """Generate AI-powered test file for a Shiny app."""
+
+    # Get app file path
+    if app_file is None:
+
+        def path_exists(x: str) -> bool | str:
+            if not isinstance(x, (str, Path)):
+                return False
+            path = Path(x)
+            if path.is_dir():
+                return "Please provide a file path to your Shiny app"
+            return path.exists() or f"Shiny app file can not be found: {x}"
+
+        app_file_val = questionary.path(
+            "Enter the path to the app file:",
+            default=path_rel_wd("app.py"),
+            validate=path_exists,
+        ).ask()
+    else:
+        app_file_val = app_file
+
+    # User quit early
+    if app_file_val is None:
+        sys.exit(1)
+
+    app_path = Path(app_file_val)
+
+    # Make sure app file exists
+    if not app_path.exists():
+        click.echo(f"❌ Error: App file does not exist: {app_path}")
+        sys.exit(1)
+
+    # Get output file path if not provided
+    if output_file is None:
+        suggested_output = app_path.parent / f"test_{app_path.stem}.py"
+
+        def output_path_valid(x: str) -> bool | str:
+            if not isinstance(x, (str, Path)):
+                return False
+            path = Path(x)
+            if path.is_dir():
+                return "Please provide a file path for your test file."
+            if path.exists():
+                return "Test file already exists. Please provide a new file name."
+            if not path.name.startswith("test_"):
+                return "Test file must start with 'test_'"
+            return True
+
+        output_file_val = questionary.path(
+            "Enter the path for the generated test file:",
+            default=str(suggested_output),
+            validate=output_path_valid,
+        ).ask()
+    else:
+        output_file_val = output_file
+
+    # User quit early
+    if output_file_val is None:
+        sys.exit(1)
+
+    output_path = Path(output_file_val)
+
+    # Validate output file
+    if output_path.exists():
+        click.echo(f"❌ Error: Test file already exists: {output_path}")
+        sys.exit(1)
+
+    if not output_path.name.startswith("test_"):
+        click.echo("❌ Error: Test file must start with 'test_'")
+        sys.exit(1)
+
+    # Import and use the test generator
+    try:
+        # Import the test generator from the new testing module structure
+        from .testing import ShinyTestGenerator
+    except ImportError as e:
+        click.echo(f"❌ Error: Could not import ShinyTestGenerator: {e}")
+        click.echo("Make sure the shiny testing dependencies are installed.")
+        sys.exit(1)
+
+    click.echo(f"🤖 Generating test using {provider} provider...")
+    if model:
+        click.echo(f"📝 Using model: {model}")
+
+    try:
+        # Create the generator
+        generator = ShinyTestGenerator(provider=provider)  # type: ignore
+
+        # Generate the test
+        _, test_file_path = generator.generate_test_from_file(
+            app_file_path=str(app_path),
+            model=model,
+            output_file=str(output_path),
+        )
+
+        click.echo(f"✅ Test file generated successfully: {test_file_path}")
+        click.echo()
+        click.echo(cli_action(cli_bold("Next steps:")))
+        click.echo(
+            f"- Run {cli_code('pytest ' + str(test_file_path))} to run the generated test"
+        )
+        click.echo("- Review and customize the test as needed")
+
+    except Exception as e:
+        click.echo(f"❌ Error generating test: {e}")
+        sys.exit(1)

shiny/testing/README.md

@@ -0,0 +1,81 @@
+# Shiny Testing Framework
+
+This directory contains the comprehensive testing framework for Shiny applications, including AI-powered test generation, evaluation tools, and utility scripts.
+
+## Components
+
+### 1. Generator (`generator/`)
+
+The core AI-powered test generation system that creates comprehensive test files for Shiny applications.
+
+**Key Features:**
+
+- Support for multiple AI providers (Anthropic, OpenAI)
+- Model selection and configuration
+- Template-based test generation
+- File and code-based input processing
+
+**Usage:**
+
+```python
+from shiny.testing import ShinyTestGenerator
+
+generator = ShinyTestGenerator(provider="anthropic")
+test_code, test_file = generator.generate_test_from_file("app.py")
+```
+
+### 2. Evaluation (`evaluation/`)
+
+Framework for evaluating the performance and quality of the test generator.
+
+**Components:**
+
+- **apps/**: Collection of diverse Shiny applications for testing
+- **scripts/**: Evaluation execution and metadata management
+- **results/**: Storage for evaluation outcomes and analysis
+
+**Usage:**
+
+```bash
+python evaluation/scripts/evaluation.py
+```
+
+### 3. Utils (`utils/`)
+
+Utility tools for processing documentation, analyzing results, and quality gating.
+
+**Key Scripts:**
+
+- `process_docs.py`: Convert XML documentation to JSON format
+- `process_results.py`: Analyze evaluation results and generate summaries
+- `quality_gate.py`: Validate performance against quality thresholds
+
+## CLI Integration
+
+The test generator is integrated into the Shiny CLI:
+
+```bash
+# Generate test with interactive prompts
+shiny generate test
+
+# Generate test with specific parameters
+shiny generate test --app app.py --output test_app.py --provider anthropic
+
+# Use different models
+shiny generate test --app app.py --provider openai --model gpt-4.1-nano
+```
+
+## Getting Started
+
+1. **Install Dependencies**: Ensure you have the required AI provider SDKs and API keys
+2. **Generate Tests**: Use the CLI or Python API to generate tests
+3. **Run Evaluations**: Use the evaluation framework to assess generator performance
+4. **Quality Control**: Use utility scripts for processing and validation
+
+## Development Workflow
+
+1. **Add Test Apps**: Place new evaluation apps in `evaluation/apps/`
+2. **Update Documentation**: Modify `generator/data/docs/` for API changes
+3. **Run Evaluations**: Execute evaluation scripts to test performance
+4. **Process Results**: Use utility scripts to analyze outcomes
+5. **Quality Gate**: Validate results meet quality standards

shiny/testing/__init__.py

@@ -0,0 +1,3 @@
+from .generator import ShinyTestGenerator
+
+__all__ = ["ShinyTestGenerator"]

shiny/testing/evaluation/__init__.py

@@ -0,0 +1,5 @@
+"""
+Evaluation Module
+
+Contains evaluation apps, scripts, and results for testing the Shiny test generator.
+"""