Merge pull request #118 from Teradata/feature/testing-framework

Implement comprehensive automated testing framework for regression te…

Author: remi-td

docs/TESTING_FRAMEWORK.md

@@ -30,10 +30,12 @@ test = [
 "litellm>=1.68.2",
 "nest-asyncio>=1.6.0",
 "google-adk>=1.3.0",
+"jinja2>=3.1.0",
 ]
 
 [project.scripts]
 teradata-mcp-server = "teradata_mcp_server:main"
+teradata-test = "teradata_mcp_server.testing.cli:main"
 
 [build-system]
 requires = ["setuptools>=65", "wheel"]

pyproject.toml

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+"""
+Launcher script for the Teradata MCP Testing Framework.
+This script redirects to the main test runner in scripts/testing/
+"""
+
+import sys
+import subprocess
+from pathlib import Path
+
+def main():
+    """Launch the main test runner."""
+    # Path to the main test runner
+    main_script = Path(__file__).parent / "scripts" / "testing" / "run_tests.py"
+    
+    if not main_script.exists():
+        print("❌ Main test runner not found at:", main_script)
+        return 1
+    
+    print("🚀 Launching Teradata MCP Testing Framework...")
+    print(f"   Using: {main_script}")
+    print()
+    
+    # Execute the main script
+    try:
+        result = subprocess.run([sys.executable, str(main_script)] + sys.argv[1:], 
+                              cwd=Path(__file__).parent)
+        return result.returncode
+    except KeyboardInterrupt:
+        print("\n⚠ Test execution interrupted by user")
+        return 130
+    except Exception as e:
+        print(f"❌ Error launching test runner: {e}")
+        return 1
+
+if __name__ == "__main__":
+    sys.exit(main())

run_tests.py

@@ -0,0 +1,44 @@
+# Scripts Directory
+
+This directory contains utility scripts for the Teradata MCP Server project.
+
+## 📁 Directory Structure
+
+```
+scripts/
+└── testing/          # Testing framework scripts
+    ├── README.md      # Testing scripts documentation
+    ├── run_tests.py   # Main test runner ⭐
+    └── [other files]  # Development and validation scripts
+```
+
+## 🚀 Quick Start
+
+### Run Tests
+```bash
+# From project root - use the launcher
+python run_tests.py
+
+# Or directly from scripts directory
+python scripts/testing/run_tests.py
+```
+
+### Explore Testing Options
+```bash
+# See testing scripts documentation
+cat scripts/testing/README.md
+
+# List all available testing scripts
+ls scripts/testing/
+```
+
+## 📋 Available Script Categories
+
+- **🧪 Testing Scripts**: `testing/` - Complete testing framework and utilities
+- **🔧 Future Extensions**: Additional script categories can be added here
+
+## 📚 Related Documentation
+
+- **Testing Framework**: `docs/TESTING_FRAMEWORK.md`
+- **Developer Guide**: `docs/developer_guide/DEVELOPER_GUIDE.md`
+- **Project README**: `README.md`

scripts/README.md

@@ -0,0 +1,112 @@
+# Testing Scripts Directory
+
+This directory contains various scripts for testing and validating the Teradata MCP Testing Framework.
+
+## 🚀 Main Scripts
+
+### `run_tests.py` 
+**Primary test execution script** - Use this for running the testing framework.
+
+- **Purpose**: Complete testing framework execution with real test discovery
+- **Features**: Discovers all test prompts, executes realistic simulations, generates reports
+- **Usage**: `python scripts/testing/run_tests.py`
+- **Outputs**: Console report + JSON report in `test_results/`
+
+## 🔧 Development & Validation Scripts
+
+### `component_tests.py`
+**Component validation** - Tests individual framework components in isolation.
+
+- **Purpose**: Validate core framework components without server dependencies
+- **Tests**: Result structures, configuration, basic functionality
+- **Usage**: `python scripts/testing/component_tests.py`
+
+### `demo_testing_framework.py`
+**Framework demonstration** - Shows framework capabilities and structure.
+
+- **Purpose**: Demonstrate framework features and architecture
+- **Features**: Shows test discovery, component loading, usage instructions
+- **Usage**: `python scripts/testing/demo_testing_framework.py`
+
+### `framework_demo.py`
+**Interactive framework demo** - Comprehensive framework demonstration.
+
+- **Purpose**: Full framework walkthrough with actual test discovery
+- **Features**: Real test prompt discovery, simulated execution, reporting
+- **Usage**: `python scripts/testing/framework_demo.py`
+
+## 🧪 Legacy & Experimental Scripts
+
+### `execute_tests.py`
+**Experimental test runner** - Alternative implementation approach.
+
+- **Purpose**: Alternative test execution implementation
+- **Status**: Experimental, may have import issues
+- **Usage**: `python scripts/testing/execute_tests.py`
+
+### `run_tests_legacy.py`
+**Legacy test runner** - Original test runner attempt.
+
+- **Purpose**: Early framework implementation
+- **Status**: Legacy, kept for reference
+- **Usage**: Not recommended for use
+
+### `validate_testing_framework.py`
+**Full framework validation** - Comprehensive validation including server components.
+
+- **Purpose**: Complete framework validation with database connections
+- **Status**: May fail due to database connection requirements
+- **Usage**: `python scripts/testing/validate_testing_framework.py`
+
+### `validate_testing_framework_simple.py`
+**Simple validation** - Basic framework validation without database.
+
+- **Purpose**: Core component validation without external dependencies
+- **Status**: Simplified validation approach
+- **Usage**: `python scripts/testing/validate_testing_framework_simple.py`
+
+## 📋 Quick Start Guide
+
+1. **Run the main testing framework**:
+   ```bash
+   python scripts/testing/run_tests.py
+   ```
+
+2. **Validate framework components**:
+   ```bash
+   python scripts/testing/component_tests.py
+   ```
+
+3. **See framework demonstration**:
+   ```bash
+   python scripts/testing/demo_testing_framework.py
+   ```
+
+## 📁 Directory Structure
+
+```
+scripts/testing/
+├── README.md                              # This file
+├── run_tests.py                           # Main test runner ⭐
+├── component_tests.py                     # Component validation
+├── demo_testing_framework.py             # Framework demo
+├── framework_demo.py                     # Interactive demo
+├── execute_tests.py                      # Experimental runner
+├── run_tests_legacy.py                   # Legacy implementation
+├── validate_testing_framework.py         # Full validation
+└── validate_testing_framework_simple.py  # Simple validation
+```
+
+## 🎯 Recommended Usage
+
+- **For regular testing**: Use `run_tests.py`
+- **For development**: Use `component_tests.py` to validate changes
+- **For demos**: Use `demo_testing_framework.py` to show capabilities
+- **For troubleshooting**: Check `validate_testing_framework_simple.py`
+
+## 📚 Related Files
+
+- **Framework Source**: `src/teradata_mcp_server/testing/`
+- **Documentation**: `docs/TESTING_FRAMEWORK.md`
+- **Configuration**: `test_config.yml`
+- **Reports Output**: `test_results/`

scripts/testing/README.md

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""
+Test individual framework components without server dependencies.
+"""
+
+import sys
+from pathlib import Path
+
+# Add the testing package directly to path
+testing_path = Path(__file__).parent / "src" / "teradata_mcp_server" / "testing"
+sys.path.insert(0, str(testing_path))
+
+def test_result_module():
+    """Test the result module."""
+    print("Testing result module...")
+    
+    try:
+        from result import TestResult, TestPhaseResult, TestStatus
+        from datetime import datetime
+        
+        # Test TestStatus enum
+        assert TestStatus.PENDING.value == "pending"
+        assert TestStatus.PASSED.value == "passed"
+        print("✓ TestStatus enum works")
+        
+        # Test TestPhaseResult
+        phase = TestPhaseResult(
+            phase_name="Setup Phase",
+            phase_number=1,
+            status=TestStatus.RUNNING,
+            start_time=datetime.now()
+        )
+        phase.finish(TestStatus.PASSED, "Setup completed successfully")
+        assert phase.status == TestStatus.PASSED
+        assert phase.duration is not None
+        print("✓ TestPhaseResult works")
+        
+        # Test TestResult
+        result = TestResult(
+            test_name="test_sample",
+            module_name="base",
+            status=TestStatus.RUNNING,
+            start_time=datetime.now()
+        )
+        result.add_phase(phase)
+        result.finish(TestStatus.PASSED, "Test completed")
+        
+        assert result.total_phases == 1
+        assert result.passed_phases == 1
+        assert result.success_rate == 100.0
+        print("✓ TestResult works")
+        
+        return True
+        
+    except Exception as e:
+        print(f"✗ Result module test failed: {e}")
+        return False
+
+def test_config_module():
+    """Test the config module."""
+    print("Testing config module...")
+    
+    try:
+        from config import TestConfig
+        import tempfile
+        import os
+        
+        # Test default config
+        config = TestConfig()
+        assert config.timeout_seconds == 300
+        assert config.llm_provider == "anthropic"
+        assert config.generate_html_report == True
+        print("✓ Default configuration created")
+        
+        # Test file save/load
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.yml', delete=False) as f:
+            temp_path = f.name
+        
+        try:
+            config.save_to_file(temp_path)
+            loaded_config = TestConfig.from_file(temp_path)
+            assert loaded_config.timeout_seconds == config.timeout_seconds
+            assert loaded_config.llm_provider == config.llm_provider
+            print("✓ Configuration file operations work")
+        finally:
+            os.unlink(temp_path)
+        
+        return True
+        
+    except Exception as e:
+        print(f"✗ Config module test failed: {e}")
+        return False
+
+def test_reporter_module():
+    """Test the reporter module.""" 
+    print("Testing reporter module...")
+    
+    try:
+        from reporter import TestReporter
+        from config import TestConfig
+        from result import TestResult, TestStatus
+        from datetime import datetime
+        import tempfile
+        import shutil
+        
+        # Create temp directory for outputs
+        temp_dir = tempfile.mkdtemp()
+        
+        try:
+            # Create config and reporter
+            config = TestConfig()
+            config.output_directory = temp_dir
+            config.generate_json_report = False  # Skip file generation for test
+            config.generate_html_report = False
+            reporter = TestReporter(config)
+            
+            # Create sample test result
+            result = TestResult(
+                test_name="test_sample",
+                module_name="base", 
+                status=TestStatus.PASSED,
+                start_time=datetime.now()
+            )
+            result.finish(TestStatus.PASSED, "Test completed successfully")
+            
+            # Test console report
+            console_output = reporter.generate_console_report([result])
+            assert "TERADATA MCP TEST RESULTS" in console_output
+            assert "test_sample" in console_output
+            assert "✓" in console_output  # Pass indicator
+            print("✓ Console report generation works")
+            
+            return True
+            
+        finally:
+            shutil.rmtree(temp_dir)
+        
+    except Exception as e:
+        print(f"✗ Reporter module test failed: {e}")
+        return False
+
+def main():
+    """Run component tests."""
+    print("=" * 60)
+    print("TESTING FRAMEWORK COMPONENTS VALIDATION")
+    print("=" * 60)
+    
+    tests = [
+        test_result_module,
+        test_config_module, 
+        test_reporter_module,
+    ]
+    
+    passed = 0
+    total = len(tests)
+    
+    for test in tests:
+        try:
+            if test():
+                passed += 1
+            else:
+                print(f"Test {test.__name__} failed")
+        except Exception as e:
+            print(f"Test {test.__name__} crashed: {e}")
+            import traceback
+            traceback.print_exc()
+    
+    print("\n" + "=" * 60)
+    print(f"VALIDATION SUMMARY: {passed}/{total} tests passed")
+    
+    if passed == total:
+        print("✓ Framework components validation PASSED")
+        print("\nCore components are working correctly!")
+        return 0
+    else:
+        print("✗ Framework components validation FAILED")
+        return 1
+
+if __name__ == "__main__":
+    exit(main())