Add local examples testing option

Author: htahir1

qualityflow/README.md

@@ -97,6 +97,16 @@ python run.py
 - Compare LLM vs baseline test approaches
 - Experiment with different configurations
 
+### Local Testing Option
+
+For offline development or controlled testing, use the local examples:
+
+```bash
+python run.py --config configs/experiment.local.yaml
+```
+
+This uses the included `examples/toy_lib/` code instead of cloning external repositories.
+
 ## ⚙️ Configuration
 
 ### Key Parameters
@@ -125,14 +135,17 @@ steps:
 ### Pipeline Options
 
 ```bash
-# Use fake provider (no API key needed)
-python run.py  # Uses config defaults
+# Default: uses remote repository (requests library)
+python run.py  
 
-# Force fresh execution (no caching) 
-python run.py --no-cache
+# Local testing with included examples
+python run.py --config configs/experiment.local.yaml
 
-# Use different config
+# High-quality test generation
 python run.py --config configs/experiment.strict.yaml
+
+# Force fresh execution (no caching) 
+python run.py --no-cache
 ```
 
 ## 🔬 Advanced Usage
@@ -213,7 +226,8 @@ qualityflow/
 │
 ├── configs/                          # Pipeline configurations
 │   ├── experiment.default.yaml       # Standard experiment settings
-│   └── experiment.strict.yaml        # High-quality gates
+│   ├── experiment.strict.yaml        # High-quality gates
+│   └── experiment.local.yaml         # Local examples testing
 │
 ├── pipelines/                        # Pipeline definitions
 │   └── generate_and_evaluate.py      # Main pipeline
@@ -233,9 +247,9 @@ qualityflow/
 │   └── unit_test_strict_v2.jinja    # Comprehensive test generation
 │
 ├── examples/                         # Demo code for testing
-│   └── toy_lib/                     # Sample library
-│       ├── calculator.py
-│       └── string_utils.py
+│   └── toy_lib/                     # Sample library with test-friendly code
+│       ├── calculator.py           # Calculator class with edge cases
+│       └── string_utils.py         # String utilities with validation
 │
 └── run.py                           # Main entry point
 ```
@@ -246,6 +260,7 @@ qualityflow/
 - **Prompt Templates**: Jinja2 templates for LLM test generation  
 - **Configuration**: YAML-driven experiment settings
 - **Test Generation**: Both LLM-based and heuristic approaches for comparison
+- **Example Code**: Sample Python modules (`toy_lib`) designed for effective test generation demonstration
 
 ## 🚀 Production Deployment
 

qualityflow/configs/experiment.local.yaml

@@ -0,0 +1,40 @@
+# QualityFlow Local Examples Configuration
+# Use local toy_lib examples instead of remote repositories
+
+# Pipeline configuration
+name: "generate_and_evaluate"
+version: "1.0"
+
+# Source configuration - using local examples
+steps:
+  select_input:
+    parameters:
+      # Use local examples instead of remote repo
+      repo_url: "local"
+      ref: "main"
+      target_glob: "examples/**/*.py"  # Target the toy_lib examples
+  
+  analyze_code:
+    parameters:
+      strategy: "all"  # Include all example files
+      max_files: 5  # Process all toy_lib files
+  
+  # LLM generation configuration
+  gen_tests_agent:
+    parameters:
+      provider: "fake"  # Use fake provider by default for local testing
+      model: "gpt-4o-mini"
+      prompt_path: "prompts/unit_test_v1.jinja"
+      max_tests_per_file: 3
+      max_files: 5  # Process all toy_lib files
+  
+  # Baseline test generation
+  gen_tests_baseline:
+    parameters:
+      enabled: true
+      max_files: 5  # Match agent max_files
+
+# Resource configuration
+settings:
+  docker:
+    requirements: requirements.txt

qualityflow/pipelines/generate_and_evaluate.py

@@ -32,10 +32,12 @@ def generate_and_evaluate() -> None:
     workspace_dir, commit_sha = fetch_source(spec)
 
     # Step 3: Analyze and select code files
-    code_summary = analyze_code(workspace_dir, commit_sha)
+    code_summary = analyze_code(workspace_dir, commit_sha, spec)
 
     # Step 4: Generate tests using LLM agent
-    agent_tests_dir, prompt_used = gen_tests_agent(workspace_dir, code_summary)
+    agent_tests_dir, test_summary = gen_tests_agent(
+        workspace_dir, code_summary
+    )
 
     # Step 5: Generate baseline tests (optional)
     baseline_tests_dir = gen_tests_baseline(workspace_dir, code_summary)
@@ -52,7 +54,7 @@ def generate_and_evaluate() -> None:
     report(
         workspace_dir,
         commit_sha,
-        prompt_used,
+        test_summary,
         agent_results,
         baseline_results,
     )

qualityflow/run.py

@@ -40,7 +40,7 @@ def main(config: Union[str, None], no_cache: bool):
     except Exception:
         # Fallback to current working directory
         default_config = Path.cwd() / "configs" / "experiment.default.yaml"
-    
+
     chosen_config = config or str(default_config)
 
     try:

qualityflow/steps/analyze_code.py

@@ -28,7 +28,7 @@ class SelectionStrategy(str, Enum):
 def analyze_code(
     workspace_dir: Path,
     commit_sha: str,
-    target_glob: str = "src/**/*.py",
+    source_spec: Dict[str, str],
     strategy: SelectionStrategy = SelectionStrategy.LOW_COVERAGE,
     max_files: int = 10,
 ) -> Annotated[Dict, "code_summary"]:
@@ -38,14 +38,19 @@ def analyze_code(
     Args:
         workspace_dir: Path to workspace directory
         commit_sha: Git commit SHA
-        target_glob: Glob pattern for target files
+        source_spec: Source specification containing target_glob and other settings
         strategy: File selection strategy
         max_files: Maximum number of files to select
 
     Returns:
         Code summary dictionary containing selected files and metadata
     """
-    logger.info(f"Analyzing code in {workspace_dir} with strategy {strategy}")
+    # Extract target_glob from source spec
+    target_glob = source_spec.get("target_glob", "src/**/*.py")
+
+    logger.info(
+        f"Analyzing code in {workspace_dir} with strategy {strategy} and glob {target_glob}"
+    )
 
     workspace_path = Path(workspace_dir)
 

qualityflow/steps/fetch_source.py

@@ -21,7 +21,7 @@ def fetch_source(
     source_spec: Dict[str, str],
 ) -> Tuple[Annotated[Path, "workspace_dir"], Annotated[str, "commit_sha"]]:
     """
-    Fetch and materialize workspace from git repository.
+    Fetch and materialize workspace from git repository or use local examples.
 
     Args:
         source_spec: Source specification from select_input step
@@ -32,9 +32,53 @@ def fetch_source(
     repo_url = source_spec["repo_url"]
     ref = source_spec["ref"]
 
+    # Handle local examples case
+    if repo_url == "local":
+        logger.info("Using local QualityFlow examples")
+        try:
+            # Get the project root (QualityFlow directory)
+            current_file = Path(__file__).resolve()
+            project_root = (
+                current_file.parent.parent
+            )  # Go up from steps/ to project root
+
+            # Create temporary workspace and copy examples
+            workspace_dir = tempfile.mkdtemp(
+                prefix="qualityflow_local_workspace_"
+            )
+            workspace_path = Path(workspace_dir)
+
+            # Copy examples directory to the temporary workspace
+            import shutil
+
+            examples_src = project_root / "examples"
+            examples_dest = workspace_path / "examples"
+
+            if examples_src.exists():
+                shutil.copytree(examples_src, examples_dest)
+                logger.info(
+                    f"Copied examples from {examples_src} to {examples_dest}"
+                )
+            else:
+                logger.warning(
+                    f"Examples directory not found at {examples_src}"
+                )
+
+            commit_sha = "local-examples"
+            logger.info(f"Local workspace ready at {workspace_path}")
+            return workspace_path, commit_sha
+
+        except Exception as e:
+            logger.error(f"Failed to set up local workspace: {e}")
+            # Fallback to current working directory
+            workspace_dir = tempfile.mkdtemp(
+                prefix="qualityflow_fallback_workspace_"
+            )
+            return Path(workspace_dir), "local-fallback"
+
     logger.info(f"Fetching source from {repo_url}@{ref}")
 
-    # Create temporary workspace
+    # Create temporary workspace for remote repositories
     workspace_dir = tempfile.mkdtemp(prefix="qualityflow_workspace_")
     workspace_path = Path(workspace_dir)