Merge pull request #788 from InfiniTensor/issue/787

issue/787 - Split run ops test logic and fix kwargs name in report

Author: wooway777

test/infinicore/framework/__init__.py

@@ -1,5 +1,5 @@
 from .base import TestConfig, TestRunner, BaseOperatorTest
-from .test_case import TestCase, TestResult
+from .entities import TestCase
 from .benchmark import BenchmarkUtils, BenchmarkResult
 from .config import (
     add_common_test_args,
@@ -9,35 +9,44 @@
 )
 from .datatypes import to_torch_dtype, to_infinicore_dtype
 from .devices import InfiniDeviceEnum, InfiniDeviceNames, torch_device_map
+from .results import TestTiming, OperatorResult, CaseResult, TestSummary
 from .runner import GenericTestRunner
 from .tensor import TensorSpec, TensorInitializer
-from .utils import (
+from .executor import TestExecutor
+from .utils.compare_utils import (
     compare_results,
     create_test_comparator,
     debug,
     get_tolerance,
+)
+from .utils.json_utils import save_json_report
+from .utils.tensor_utils import (
     infinicore_tensor_from_torch,
-    rearrange_tensor,
     convert_infinicore_to_torch,
+    rearrange_tensor,
     is_broadcast,
+    is_integer_dtype,
     is_complex_dtype,
     is_floating_dtype,
-    is_integer_dtype,
 )
 
+
 __all__ = [
     # Core types and classes
     "BaseOperatorTest",
+    "CaseResult",
     "GenericTestRunner",
     "InfiniDeviceEnum",
     "InfiniDeviceNames",
+    "OperatorResult",
     "TensorInitializer",
     "TensorSpec",
     "TestCase",
     "TestConfig",
-    "TestResult",
+    "TestExecutor",
+    "TestSummary",
     "TestRunner",
-    "TestReporter",
+    "TestTiming",
     # Core functions
     "add_common_test_args",
     "compare_results",
@@ -50,6 +59,8 @@
     "get_tolerance",
     "infinicore_tensor_from_torch",
     "rearrange_tensor",
+    # Json utilites
+    "save_json_report",
     # Utility functions
     "to_infinicore_dtype",
     "to_torch_dtype",

test/infinicore/framework/base.py

@@ -8,15 +8,15 @@
 import traceback
 from abc import ABC, abstractmethod
 
-from .test_case import TestCase, TestResult
+from .results import CaseResult
 from .datatypes import to_torch_dtype, to_infinicore_dtype
 from .devices import InfiniDeviceNames, torch_device_map
 from .tensor import TensorSpec, TensorInitializer
-from .utils import (
+from .utils.tensor_utils import (
     clone_torch_tensor,
-    create_test_comparator,
     infinicore_tensor_from_torch,
 )
+from .utils.compare_utils import create_test_comparator
 from .benchmark import BenchmarkUtils
 
 
@@ -84,7 +84,7 @@ def run_tests(self, devices, test_func, test_type="Test"):
                 try:
                     print(f"{test_case}")
 
-                    # Execute test and get TestResult object
+                    # Execute test and get CaseResult object
                     test_result = test_func(device, test_case, self.config)
                     self.test_results.append(test_result)
 
@@ -118,8 +118,8 @@ def run_tests(self, devices, test_func, test_type="Test"):
                     print(f"\033[91m✗\033[0m {error_msg}")
                     self.failed_tests.append(error_msg)
 
-                    # Create a failed TestResult
-                    failed_result = TestResult(
+                    # Create a failed CaseResult
+                    failed_result = CaseResult(
                         success=False,
                         return_code=-1,
                         error_message=str(e),
@@ -400,12 +400,12 @@ def run_test(self, device, test_case, config):
             config: Test configuration
 
         Returns:
-            TestResult: Test result object containing status and timing information
+            CaseResult: Test case result object containing status and timing information
         """
         device_str = torch_device_map[device]
 
-        # Initialize test result
-        test_result = TestResult(
+        # Initialize test case result
+        test_result = CaseResult(
             success=False,
             return_code=-1,  # Default to failure
             test_case=test_case,

test/infinicore/framework/benchmark.py

@@ -5,7 +5,7 @@
 import time
 import torch
 import infinicore
-from .utils import synchronize_device
+from .utils.tensor_utils import synchronize_device
 
 
 class BenchmarkUtils:

test/infinicore/framework/test_case.py test/infinicore/framework/entities.pytest/infinicore/framework/test_case.py renamed to test/infinicore/framework/entities.py

@@ -7,21 +7,6 @@
 from .tensor import TensorSpec
 
 
-@dataclass
-class TestResult:
-    """Test result data structure"""
-
-    success: bool
-    return_code: int  # 0: success, -1: failure, -2: skipped, -3: partial
-    torch_host_time: float = 0.0
-    torch_device_time: float = 0.0
-    infini_host_time: float = 0.0
-    infini_device_time: float = 0.0
-    error_message: str = ""
-    test_case: Any = None
-    device: Any = None
-
-
 class TestCase:
     """Test case with all configuration included"""
 

test/infinicore/framework/executor.py

@@ -0,0 +1,81 @@
+import sys
+import importlib.util
+from io import StringIO
+from contextlib import contextmanager
+from .results import OperatorResult, TestSummary
+
+
+@contextmanager
+def capture_output():
+    """Context manager: captures stdout and stderr."""
+    new_out, new_err = StringIO(), StringIO()
+    old_out, old_err = sys.stdout, sys.stderr
+    try:
+        sys.stdout, sys.stderr = new_out, new_err
+        yield new_out, new_err
+    finally:
+        sys.stdout, sys.stderr = old_out, old_err
+
+
+class TestExecutor:
+    def execute(self, file_path) -> OperatorResult:
+        result = OperatorResult(name=file_path.stem)
+
+        try:
+            # 1. Dynamically import the module
+            module = self._import_module(file_path)
+
+            # 2. Look for TestRunner
+            if not hasattr(module, "GenericTestRunner"):
+                raise ImportError("No GenericTestRunner found in module")
+
+            # 3. Look for TestClass (subclass of BaseOperatorTest)
+            test_class = self._find_test_class(module)
+            if not test_class:
+                raise ImportError("No BaseOperatorTest subclass found")
+
+            test_instance = test_class()
+            runner_class = module.GenericTestRunner
+            runner = runner_class(test_instance.__class__)
+
+            # 4. Execute and capture output
+            with capture_output() as (out, err):
+                success, internal_runner = runner.run()
+
+            # 5. Populate results
+            result.success = success
+            result.stdout = out.getvalue()
+            result.stderr = err.getvalue()
+
+            # Extract detailed results from internal_runner
+            test_results = internal_runner.get_test_results() if internal_runner else []
+
+            test_summary = TestSummary()
+            test_summary.process_operator_result(result, test_results)
+
+        except Exception as e:
+            result.success = False
+            result.error_message = str(e)
+            result.stderr += f"\nExecutor Error: {str(e)}"
+            result.return_code = -1
+
+        return result
+
+    def _import_module(self, path):
+        module_name = f"op_test_{path.stem}"
+        spec = importlib.util.spec_from_file_location(module_name, path)
+        if not spec or not spec.loader:
+            raise ImportError(f"Could not load spec from {path}")
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[module_name] = module
+        spec.loader.exec_module(module)
+        return module
+
+    def _find_test_class(self, module):
+        for attr_name in dir(module):
+            attr = getattr(module, attr_name)
+            if isinstance(attr, type) and hasattr(attr, "__bases__"):
+                # Simple check for base class name
+                if any("BaseOperatorTest" in str(b) for b in attr.__bases__):
+                    return attr
+        return None