refactoring

tom-sapletta-com · tom-sapletta-com · commit 2556e8afb821 · 2026-03-30T17:59:39.000+02:00
diff --git a/benchmarks/llm_benchmark.py b/benchmarks/llm_benchmark.py
@@ -1097,7 +1097,7 @@ def generate_command_errors_report(results: BenchmarkResults) -> str:
     lines.append("")
     lines.append(f"Generated: `{ts}`")
     lines.append("")
-    lines.append("Źródło: `examples/benchmark_nlp2cmd.py` + `benchmark_results.json`")
+    lines.append("Źródło: `benchmarks/llm_benchmark.py` + `benchmark_results.json`")
     lines.append("")
 
     failures = [
diff --git a/docs/development/BENCHMARKING.md b/docs/development/BENCHMARKING.md
@@ -29,13 +29,13 @@ make benchmark-clean
 
 ```bash
 # Run the main benchmark script
-python3 examples/02_benchmarks/performance_testing/benchmark.py
+PYTHONPATH=src python3 benchmarks/llm_benchmark.py
 
 # Run benchmark WITHOUT cache (forces fresh LLM calls for every query)
-python3 examples/02_benchmarks/performance_testing/benchmark.py --no-cache
+PYTHONPATH=src python3 benchmarks/llm_benchmark.py --no-cache
 
 # Run the sequential commands example
-python3 examples/02_benchmarks/sequential_testing/run_sequential.py
+python3 examples/02_benchmarks/sequential_testing/benchmark.py
 ```
 
 ### Benchmark Without Cache
@@ -47,7 +47,7 @@ For true LLM performance testing without cache influence:
 make benchmark
 
 # Benchmark without cache (pure LLM performance)
-python3 examples/02_benchmarks/performance_testing/benchmark.py --no-cache
+PYTHONPATH=src python3 benchmarks/llm_benchmark.py --no-cache
 
 # The --no-cache flag disables:
 # - Cache lookups (exact, fuzzy, similarity)
@@ -107,7 +107,7 @@ Here's a sample benchmark output:
 2. **benchmark_results.csv**: CSV file suitable for plotting in Excel or other tools
 3. **sequential_benchmark_results.json**: Results from the sequential commands example
 
-When using the LLM benchmark (`examples/02_benchmarks/performance_testing/benchmark.py`), additional files are generated in `benchmark_output/`:
+When using the canonical LLM benchmark (`benchmarks/llm_benchmark.py`), additional files are generated in `benchmark_output/`:
 
 1. **benchmark_results.json**: Raw per-query results + aggregated accuracy stats
 2. **benchmark_results.html**: Interactive charts
@@ -152,7 +152,7 @@ Based on benchmark results, you can:
 
 ### Adding New Commands
 
-Edit `examples/02_benchmarks/performance_testing/benchmark.py` and modify the command lists:
+Edit `benchmarks/llm_benchmark.py` for the canonical benchmark, or `examples/02_benchmarks/performance_testing/benchmark.py` for the lightweight adapter example, and modify the command lists:
 
 ```python
 commands = {
@@ -210,7 +210,7 @@ Add benchmarking to your CI pipeline:
 # .github/workflows/benchmark.yml
 - name: Run Benchmark
   run: |
-    python3 examples/02_benchmarks/performance_testing/benchmark.py
+    python3 benchmarks/llm_benchmark.py
     
 - name: Upload Results
   uses: actions/upload-artifact@v3
diff --git a/docs/reference/examples-guide.md b/docs/reference/examples-guide.md
@@ -170,16 +170,21 @@ python examples/_dynamic_orchestrator.py --prompt "Create a dashboard with chart
 
 ## 📊 Performance Benchmarks
 
-**Location:** `examples/02_benchmarks/`
+**Lightweight examples:** `examples/02_benchmarks/`
+
+**Canonical benchmark suite:** `benchmarks/`
 
 ```bash
-# Run performance tests
-cd examples/02_benchmarks/performance_testing
-python benchmark.py
+# Run lightweight performance examples
+python3 examples/02_benchmarks/performance_testing/benchmark.py
 
 # Sequential testing
-cd examples/02_benchmarks/sequential_testing
-python run_sequential.py
+python3 examples/02_benchmarks/sequential_testing/benchmark.py
+
+# Run canonical benchmark scripts
+PYTHONPATH=src python3 benchmarks/llm_benchmark.py
+PYTHONPATH=src python3 benchmarks/learning_benchmark.py
+PYTHONPATH=src python3 benchmarks/thermodynamic_benchmark.py
 ```
 
 ## 🔧 Domain-Specific Examples
diff --git a/examples/02_benchmarks/README.md b/examples/02_benchmarks/README.md
@@ -1,6 +1,8 @@
 # Benchmarks - Performance Testing
 
-This section contains tools for benchmarking and testing NLP2CMD performance.
+This section contains lightweight examples for benchmarking and testing NLP2CMD performance.
+
+For the canonical benchmark suite, see `./benchmarks/` at the repository root.
 
 ## Categories
 
diff --git a/examples/02_benchmarks/performance_testing/README.md b/examples/02_benchmarks/performance_testing/README.md
@@ -1,6 +1,8 @@
 # Performance Testing Benchmark
 
-Comprehensive performance benchmarking for NLP2CMD.
+Lightweight performance benchmarking example for NLP2CMD.
+
+For the canonical benchmark suite, see `benchmarks/` at the repository root.
 
 ## Overview
 
diff --git a/examples/02_benchmarks/performance_testing/benchmark.py b/examples/02_benchmarks/performance_testing/benchmark.py
@@ -18,9 +18,9 @@
 from datetime import datetime
 from typing import List, Dict, Any
 
-# Add src to path
-sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
-sys.path.append(str(Path(__file__).resolve().parents[2]))
+PROJECT_ROOT = Path(__file__).resolve().parents[3]
+sys.path.insert(0, str(PROJECT_ROOT / "src"))
+sys.path.insert(0, str(PROJECT_ROOT / "examples"))
 
 from _example_helpers import print_separator
 
diff --git a/examples/02_benchmarks/sequential_testing/benchmark.py b/examples/02_benchmarks/sequential_testing/benchmark.py
@@ -7,17 +7,17 @@
 batch processing of commands.
 
 Usage:
-    python examples/benchmark_sequential_commands.py
+    python3 examples/02_benchmarks/sequential_testing/benchmark.py
 """
 
 import sys
 import time
 from pathlib import Path
 from typing import List, Tuple
 
-# Add src to path
-sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
-sys.path.append(str(Path(__file__).resolve().parents[2]))
+PROJECT_ROOT = Path(__file__).resolve().parents[3]
+sys.path.insert(0, str(PROJECT_ROOT / "src"))
+sys.path.insert(0, str(PROJECT_ROOT / "examples"))
 
 from _example_helpers import print_separator
 
diff --git a/examples/05_advanced_features/dynamic_schemas/demo_intelligent_nlp2cmd.py b/examples/05_advanced_features/dynamic_schemas/demo_intelligent_nlp2cmd.py
@@ -239,7 +239,7 @@ def demo_intelligent_nlp2cmd():
     
     # Initialize system
     nlp = IntelligentNLP2CMD(
-        storage_dir="./migrated_schemas"
+        storage_dir=str(PROJECT_ROOT / "generated" / "migrated_schemas")
     )
     
     # Test queries
diff --git a/examples/05_advanced_features/dynamic_schemas/demo_persistent_storage.py b/examples/05_advanced_features/dynamic_schemas/demo_persistent_storage.py
@@ -18,7 +18,7 @@ def demonstrate_persistent_storage():
     print("=" * 60)
     
     # Initialize registry with persistent storage
-    storage_dir = "./my_command_schemas"
+    storage_dir = str(PROJECT_ROOT / "generated" / "my_command_schemas")
     registry = DynamicSchemaRegistry(
         use_per_command_storage=True,
         storage_dir=storage_dir,
diff --git a/examples/05_advanced_features/dynamic_schemas/demo_schema_flow.py b/examples/05_advanced_features/dynamic_schemas/demo_schema_flow.py
@@ -30,11 +30,12 @@ def demonstrate_schema_flow():
     
     # Step 1: Initialize registry
     print("\n1. Initializing Schema Registry...")
+    command_schemas_dir = PROJECT_ROOT / "command_schemas"
     registry = DynamicSchemaRegistry(
         use_per_command_storage=True,
-        storage_dir="./command_schemas"
+        storage_dir=str(command_schemas_dir)
     )
-    print(f"   Storage location: ./command_schemas")
+    print(f"   Storage location: {command_schemas_dir}")
     print(f"   Loaded schemas: {len(registry.schemas)}")
     
     # Step 2: Extract schema from command
@@ -54,7 +55,7 @@ def demonstrate_schema_flow():
     
     # Step 3: Show stored schema file
     print(f"\n3. Schema stored in file system...")
-    schema_file = Path("./command_schemas/commands") / f"{command}.json"
+    schema_file = command_schemas_dir / "commands" / f"{command}.json"
     if schema_file.exists():
         print(f"   File: {schema_file}")
         print(f"   Size: {schema_file.stat().st_size} bytes")
diff --git a/examples/05_advanced_features/dynamic_schemas/demo_version_detection.py b/examples/05_advanced_features/dynamic_schemas/demo_version_detection.py
@@ -2,8 +2,6 @@
 """Practical demonstration of version-aware command generation."""
 
 import subprocess
-import json
-from pathlib import Path
 
 def demonstrate_version_detection():
     """Demonstrate practical version detection and command adaptation."""
diff --git a/examples/05_advanced_features/dynamic_schemas/schema_flow_demo.py b/examples/05_advanced_features/dynamic_schemas/schema_flow_demo.py
@@ -27,9 +27,10 @@ def show_schema_extraction_flow():
     print("-" * 40)
     from nlp2cmd.schema_extraction import DynamicSchemaRegistry
     
+    command_schemas_dir = PROJECT_ROOT / "command_schemas"
     registry = DynamicSchemaRegistry(
         use_per_command_storage=True,
-        storage_dir="./command_schemas"
+        storage_dir=str(command_schemas_dir)
     )
     
     # Extract schema
@@ -43,9 +44,9 @@ def show_schema_extraction_flow():
     
     print("\n📋 STEP 3: Store Schema")
     print("-" * 40)
-    schema_file = Path("./command_schemas/commands/docker.json")
+    schema_file = command_schemas_dir / "commands" / "docker.json"
     if schema_file.exists():
-        print(f"✓ Stored in: {schema_file}")
+        print(f"   Storage location: {command_schemas_dir}")
         with open(schema_file) as f:
             data = json.load(f)
         print(f"✓ File size: {schema_file.stat().st_size} bytes")
diff --git a/examples/05_advanced_features/dynamic_schemas/simple_schema_demo.py b/examples/05_advanced_features/dynamic_schemas/simple_schema_demo.py
@@ -21,9 +21,10 @@ def main():
     
     # 1. Load schemas from storage
     print("\n1. Loading schemas from storage...")
+    command_schemas_dir = PROJECT_ROOT / "command_schemas"
     registry = DynamicSchemaRegistry(
         use_per_command_storage=True,
-        storage_dir="./command_schemas"
+        storage_dir=str(command_schemas_dir)
     )
     print(f"   Loaded {len(registry.schemas)} schemas")
     
@@ -51,8 +52,9 @@ def main():
     # Note: SchemaDrivenAppSpecAdapter loads schemas from validated_schemas.json
     # Let's copy our schemas there first
     import shutil
-    if Path('./command_schemas/index.json').exists():
-        shutil.copy('./command_schemas/index.json', './validated_schemas.json')
+    validated_schemas_path = command_schemas_dir / 'exports' / 'validated_schemas.json'
+    if (command_schemas_dir / 'index.json').exists():
+        shutil.copy(command_schemas_dir / 'index.json', validated_schemas_path)
     
     adapter = SchemaDrivenAppSpecAdapter()
     nlp = NLP2CMD(adapter=adapter)
diff --git a/examples/06_tools_and_utilities/migration_tools/README.md b/examples/06_tools_and_utilities/migration_tools/README.md
@@ -0,0 +1,15 @@
+# Migration Tools
+
+Tools and walkthroughs for migrating schema storage and data layouts.
+
+## Files
+
+- `guide.py` — migration guide from JSON/YAML to TOON format
+- `demo_versioned_schemas.py` — versioned schema storage and migration demo
+
+## Run
+
+```bash
+python guide.py
+python demo_versioned_schemas.py
+```
diff --git a/examples/06_tools_and_utilities/migration_tools/demo_versioned_schemas.py b/examples/06_tools_and_utilities/migration_tools/demo_versioned_schemas.py
@@ -14,15 +14,15 @@
 from nlp2cmd.schema_extraction import ExtractedSchema, CommandSchema, CommandParameter
 
 
-def migrate_existing_schemas(source_file: str = "./validated_schemas.json"):
+def migrate_existing_schemas(source_file: str = str(PROJECT_ROOT / "command_schemas" / "exports" / "validated_schemas.json")):
     """Migrate existing schemas to versioned storage."""
     
     print("=" * 60)
     print("Migrating Existing Schemas to Versioned Storage")
     print("=" * 60)
     
     # Initialize versioned store
-    store = VersionedSchemaStore("./migrated_schemas")
+    store = VersionedSchemaStore(str(PROJECT_ROOT / "generated" / "migrated_schemas"))
     
     # Load existing schemas
     print(f"\nLoading schemas from {source_file}...")
@@ -379,7 +379,7 @@ def main():
     
     # Show storage structure
     print("\nStorage structure:")
-    storage_path = Path("./migrated_schemas")
+    storage_path = PROJECT_ROOT / "generated" / "migrated_schemas"
     if storage_path.exists():
         for item in sorted(storage_path.rglob("*")):
             if item.is_file():
@@ -391,7 +391,7 @@ def main():
     
     # Clean up option
     if input("\nDelete demo files? (y/N): ").lower() == 'y':
-        shutil.rmtree("./migrated_schemas")
+        shutil.rmtree(storage_path)
         print("Demo files cleaned up.")
 
 
diff --git a/examples/README.md b/examples/README.md
@@ -7,7 +7,7 @@ This directory contains organized examples demonstrating various capabilities of
 Each numbered folder represents a category of examples:
 
 - **[01_basics](./01_basics/)** - Fundamental examples and getting started guides
-- **[02_benchmarks](./02_benchmarks/)** - Performance testing and benchmarking tools
+- **[02_benchmarks](./02_benchmarks/)** - Lightweight performance testing and benchmarking examples
 - **[03_integrations](./03_integrations/)** - Integration examples with external systems
 - **[04_domain_specific](./04_domain_specific/)** - Domain-specific use cases and applications
 - **[05_advanced_features](./05_advanced_features/)** - Advanced features and experimental capabilities
@@ -17,6 +17,8 @@ Each numbered folder represents a category of examples:
 
 Each example folder contains its own README.md with specific instructions. Navigate to any folder to explore examples in that category.
 
+For the canonical benchmark runners, see the top-level `benchmarks/` directory.
+
 ## Running Examples
 
 Most examples can be run directly from their respective folders:
diff --git a/src/nlp2cmd/generation/schema/adapter.py b/src/nlp2cmd/generation/schema/adapter.py
@@ -36,7 +36,10 @@ def __init__(
     def _load_schemas(self):
         """Load schemas from various sources."""
         # Load from validated schemas if available
+        project_root = Path(__file__).resolve().parents[4]
         validated_candidates = [
+            project_root / "command_schemas" / "exports" / "validated_schemas.json",
+            project_root / "validated_schemas.json",
             Path("./command_schemas/exports/validated_schemas.json"),
             Path("./validated_schemas.json"),
         ]

Original file line number	Diff line number	Diff line change
`@@ -239,7 +239,7 @@ def demo_intelligent_nlp2cmd():`
`239`	`239`
`240`	`240`	`# Initialize system`
`241`	`241`	`nlp = IntelligentNLP2CMD(`
`242`		`- storage_dir="./migrated_schemas"`
	`242`	`+ storage_dir=str(PROJECT_ROOT / "generated" / "migrated_schemas")`
`243`	`243`	`)`
`244`	`244`
`245`	`245`	`# Test queries`