issue/593 - added verbose test mode

Author: wooway777

test/infinicore/framework/base.py

@@ -1,5 +1,6 @@
 import torch
 import infinicore
+import traceback  # Add import for traceback
 
 from abc import ABC, abstractmethod
 from typing import List, Dict, Any, Optional
@@ -216,14 +217,19 @@ def __str__(self):
 class TestConfig:
     """Test configuration"""
 
-    def __init__(self, debug=False, bench=False, num_prerun=10, num_iterations=1000):
+    def __init__(
+        self,
+        debug=False,
+        bench=False,
+        num_prerun=10,
+        num_iterations=1000,
+        verbose=False,
+    ):
         self.debug = debug
         self.bench = bench
         self.num_prerun = num_prerun
         self.num_iterations = num_iterations
-
-
-# In base.py - update the TestRunner class
+        self.verbose = verbose
 
 
 class TestRunner:
@@ -273,16 +279,11 @@ def run_tests(self, devices, test_func, test_type="Test"):
                         # Test was skipped due to both operators not being implemented
                         skip_msg = f"{test_case} - {InfiniDeviceNames[device]} - Both operators not implemented"
                         self.skipped_tests.append(skip_msg)
-                        print(
-                            f"\033[93m⚠\033[0m Skipped - both operators not implemented"
-                        )
                     elif status == "partial":
                         # Test was partially executed (one operator not implemented)
                         partial_msg = f"{test_case} - {InfiniDeviceNames[device]} - One operator not implemented"
                         self.partial_tests.append(partial_msg)
-                        print(
-                            f"\033[93m⚠\033[0m Partial - one operator not implemented"
-                        )
+
                     # Failed tests are handled in the exception handler below
 
                 except Exception as e:
@@ -291,6 +292,12 @@ def run_tests(self, devices, test_func, test_type="Test"):
                     )
                     print(f"\033[91m✗\033[0m {error_msg}")
                     self.failed_tests.append(error_msg)
+
+                    # In verbose mode, print full traceback and stop execution
+                    if self.config.verbose:
+                        traceback.print_exc()
+                        return False  # Stop test execution immediately
+
                     if self.config.debug:
                         raise
 
@@ -312,34 +319,16 @@ def print_summary(self):
 
         print(f"\n{'='*60}")
         print("TEST SUMMARY")
-        print(f"{'='*60}")
         print(f"Total tests: {total_tests}")
         print(f"\033[92mPassed: {passed_count}\033[0m")
 
-        # Display partial tests (one operator not implemented)
-        if self.partial_tests:
-            print(
-                f"\033[93mPartial (one operator not implemented): {partial_count}\033[0m"
-            )
-            for test in self.partial_tests:
-                print(f"  - {test}")
-
-        # Display skipped tests (both operators not implemented)
-        if self.skipped_tests:
-            print(
-                f"\033[93mSkipped (both operators not implemented): {skipped_count}\033[0m"
-            )
-            for test in self.skipped_tests:
-                print(f"  - {test}")
-
+        result = True
         # Display failed tests
         if self.failed_tests:
             print(f"\033[91mFailed: {failed_count}\033[0m")
-            for failure in self.failed_tests:
-                print(f"  - {failure}")
 
             # Return False only if there are actual test failures
-            return False
+            result = False
         else:
             # Calculate success rate based on actual executed tests
             executed_tests = passed_count + partial_count + failed_count
@@ -352,10 +341,11 @@ def print_summary(self):
                 print(
                     f"\n\033[93mTests completed with some implementations missing\033[0m"
                 )
-                return True  # Skipped/partial tests don't count as failures
             else:
                 print(f"\n\033[92mAll tests passed!\033[0m")
-                return True
+
+        print(f"{'='*60}")
+        return result
 
 
 class BaseOperatorTest(ABC):
@@ -537,6 +527,9 @@ def run_test(self, device, test_case, config):
             if torch_result is None:
                 torch_implemented = False
         except NotImplementedError:
+            if config.verbose:
+                traceback.print_exc()
+                return False  # Stop test execution immediately
             torch_implemented = False
             torch_result = None
 
@@ -545,6 +538,9 @@ def run_test(self, device, test_case, config):
             if infini_result is None:
                 infini_implemented = False
         except NotImplementedError:
+            if config.verbose:
+                traceback.print_exc()
+                return False  # Stop test execution immediately
             infini_implemented = False
             infini_result = None
 

test/infinicore/framework/config.py

@@ -1,7 +1,6 @@
 import argparse
 from .devices import InfiniDeviceEnum
 
-# hardware_info.py
 """
 Shared hardware platform information for the InfiniCore testing framework
 """
@@ -61,6 +60,9 @@ def get_args():
   # Run with debug mode on multiple devices
   python test_operator.py --cpu --nvidia --debug
 
+  # Run with verbose mode to stop on first error with full traceback
+  python test_operator.py --cpu --nvidia --verbose
+
   # Run performance profiling with custom iterations
   python test_operator.py --nvidia --bench --num_prerun 50 --num_iterations 5000
 
@@ -90,6 +92,11 @@ def get_args():
         action="store_true",
         help="Enable debug mode for detailed tensor comparison",
     )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable verbose mode to stop on first error with full traceback",
+    )
 
     # Device options using shared hardware info
     hardware_group = get_hardware_args_group(parser)

test/infinicore/framework/runner.py

@@ -28,6 +28,7 @@ def run(self):
             bench=self.args.bench,
             num_prerun=self.args.num_prerun,
             num_iterations=self.args.num_iterations,
+            verbose=self.args.verbose,  # Pass verbose flag to TestConfig
         )
 
         runner = TestRunner(self.operator_test.test_cases, config)

test/infinicore/run.py

@@ -122,6 +122,9 @@ def run_all_op_tests(ops_dir=None, specific_ops=None, extra_args=None):
 
     results = {}
 
+    # Check if verbose mode is enabled
+    verbose_mode = extra_args and "--verbose" in extra_args
+
     for test_file in operator_test_files:
         test_name = test_file.stem
 
@@ -199,14 +202,27 @@ def run_all_op_tests(ops_dir=None, specific_ops=None, extra_args=None):
                 f"{status_icon}  {test_name}: {status_text} (return code: {returncode})"
             )
 
+            # In verbose mode, stop execution on first failure
+            if verbose_mode and not success and returncode not in [-2, -3]:
+                break
+
         except Exception as e:
             print(f"💥 {test_name}: ERROR - {str(e)}")
             results[test_name] = (False, -1, "", str(e))
 
+            # In verbose mode, stop execution on any exception
+            if verbose_mode:
+                print(f"\n{'!'*60}")
+                print(
+                    f"VERBOSE MODE: Stopping execution due to exception in {test_name}"
+                )
+                print(f"{'!'*60}")
+                break
+
     return results
 
 
-def print_summary(results):
+def print_summary(results, verbose_mode=False, total_expected_tests=0):
     """Print a comprehensive summary of test results."""
     print(f"\n{'='*80}")
     print("CUMULATIVE TEST SUMMARY")
@@ -242,7 +258,11 @@ def print_summary(results):
 
     total = len(results)
 
-    print(f"Total tests: {total}")
+    print(f"Total tests run: {total}")
+    if total_expected_tests > 0 and total < total_expected_tests:
+        print(f"Total tests expected: {total_expected_tests}")
+        print(f"Tests not executed: {total_expected_tests - total}")
+
     print(f"Passed: {passed}")
     print(f"Failed: {failed}")
 
@@ -290,6 +310,10 @@ def print_summary(results):
             success_rate = passed / executed_tests * 100
             print(f"\nSuccess rate: {success_rate:.1f}%")
 
+    if verbose_mode and total < total_expected_tests:
+        print(f"\n💡 Verbose mode: Execution stopped after first failure")
+        print(f"   {total_expected_tests - total} tests were not executed")
+
     if failed == 0:
         if skipped > 0 or partial > 0:
             print(f"\n⚠️  Tests completed with some operators not implemented")
@@ -358,6 +382,11 @@ def generate_help_epilog(ops_dir):
     epilog_parts.append("  # Run with debug mode on multiple devices")
     epilog_parts.append("  python run.py --cpu --nvidia --debug")
     epilog_parts.append("")
+    epilog_parts.append(
+        "  # Run with verbose mode to stop on first error with full traceback"
+    )
+    epilog_parts.append("  python run.py --cpu --nvidia --verbose")
+    epilog_parts.append("")
     epilog_parts.append("  # List available tests without running")
     epilog_parts.append("  python run.py --list")
     epilog_parts.append("")
@@ -386,6 +415,12 @@ def generate_help_epilog(ops_dir):
     epilog_parts.append(
         "  - --bench option is disabled in batch mode (run individual tests for benchmarking)"
     )
+    epilog_parts.append(
+        "  - --verbose mode stops execution on first error and shows full traceback"
+    )
+    epilog_parts.append(
+        "  - In verbose mode, subsequent tests are skipped after first failure"
+    )
 
     return "\n".join(epilog_parts)
 
@@ -413,6 +448,11 @@ def main():
         action="store_true",
         help="List all available test files without running them",
     )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable verbose mode to stop on first error with full traceback (passed to individual tests)",
+    )
 
     from framework import get_hardware_args_group
 
@@ -442,6 +482,10 @@ def main():
             print(f"Error: Ops directory '{ops_dir}' does not exist.")
             sys.exit(1)
 
+    # Add verbose flag to extra arguments if specified
+    if args.verbose and "--verbose" not in unknown_args:
+        unknown_args.append("--verbose")
+
     # Show what extra arguments will be passed
     if unknown_args:
         print(f"Passing extra arguments to test scripts: {unknown_args}")
@@ -453,6 +497,9 @@ def main():
     print(f"Operating directory: {ops_dir}")
     print(f"Available operators: {len(available_operators)}")
 
+    if args.verbose:
+        print(f"Verbose mode: ENABLED (will stop on first error with full traceback)")
+
     if args.ops:
         # Validate requested operators
         valid_ops = []
@@ -469,10 +516,13 @@ def main():
 
         if valid_ops:
             print(f"Testing operators: {', '.join(valid_ops)}")
+            total_expected_tests = len(valid_ops)
         else:
             print("No valid operators specified. Running all available tests.")
+            total_expected_tests = len(available_operators)
     else:
         print("Testing all available operators")
+        total_expected_tests = len(available_operators)
 
     print()
 
@@ -484,7 +534,7 @@ def main():
     )
 
     # Print summary and exit with appropriate code
-    all_passed = print_summary(results)
+    all_passed = print_summary(results, args.verbose, total_expected_tests)
 
     # Check if there were any tests with missing implementations
     has_missing_implementations = any(
@@ -495,6 +545,18 @@ def main():
         print(f"\n⚠️  Note: Some operators are not fully implemented")
         print(f"   Run individual tests for details on missing implementations")
 
+    if args.verbose and not all_passed:
+        print(
+            f"\n💡 Verbose mode tip: Use individual test commands for detailed debugging:"
+        )
+        failed_ops = [
+            name
+            for name, (success, _, _, _) in results.items()
+            if not success and name in results
+        ]
+        for op in failed_ops[:3]:  # Show first 3 failed operators
+            print(f"   python {ops_dir / (op + '.py')} --verbose")
+
     sys.exit(0 if all_passed else 1)