stdout comparison

Update test_results.py

Update parse_test_output.py

Update equivalence.py

Author: KRRT7

codeflash/verification/equivalence.py

@@ -1,6 +1,7 @@
+import difflib
 import sys
 
-from codeflash.cli_cmds.console import logger
+from codeflash.cli_cmds.console import console, logger
 from codeflash.verification.comparator import comparator
 from codeflash.verification.test_results import TestResults, TestType, VerificationType
 
@@ -61,6 +62,12 @@ def compare_test_results(original_results: TestResults, candidate_results: TestR
                 cdd_test_result.return_value,
             )
             break
+        if (original_test_result.stdout and cdd_test_result.stdout) and not comparator(
+            original_test_result.stdout, cdd_test_result.stdout
+        ):
+            are_equal = False
+            break
+
         if original_test_result.test_type in [TestType.EXISTING_UNIT_TEST, TestType.CONCOLIC_COVERAGE_TEST] and (
             cdd_test_result.did_pass != original_test_result.did_pass
         ):

codeflash/verification/parse_test_output.py

@@ -42,6 +42,10 @@ def parse_func(file_path: Path) -> XMLParser:
     return parse(file_path, xml_parser)
 
 
+matches_re = re.compile(r"!######(.*?):(.*?)([^\.:]*?):(.*?):(.*?):(.*?)######!")
+cleaner_re = re.compile(r"!######(.*?)######!")
+
+
 def parse_test_return_values_bin(file_location: Path, test_files: TestFiles, test_config: TestConfig) -> TestResults:
     test_results = TestResults()
     if not file_location.exists():
@@ -259,7 +263,13 @@ def parse_test_xml(
                     message = testcase.result[0].message.lower()
                     if "timed out" in message:
                         timed_out = True
-            matches = re.findall(r"!######(.*?):(.*?)([^\.:]*?):(.*?):(.*?):(.*?)######!", testcase.system_out or "")
+
+            sys_stdout = testcase.system_out or ""
+            matches = matches_re.findall(sys_stdout)
+
+            if sys_stdout:
+                sys_stdout = cleaner_re.sub("", sys_stdout)
+
             if not matches or not len(matches):
                 test_results.add(
                     FunctionTestInvocation(
@@ -278,6 +288,7 @@ def parse_test_xml(
                         test_type=test_type,
                         return_value=None,
                         timed_out=timed_out,
+                        stdout=sys_stdout,
                     )
                 )
 
@@ -306,6 +317,7 @@ def parse_test_xml(
                             test_type=test_type,
                             return_value=None,
                             timed_out=timed_out,
+                            stdout=sys_stdout,
                         )
                     )
 
@@ -393,6 +405,7 @@ def merge_test_results(
                         verification_type=VerificationType(result_bin.verification_type)
                         if result_bin.verification_type
                         else None,
+                        stdout=xml_result.stdout,
                     )
                 )
         elif xml_results.test_results[0].id.iteration_id is not None:
@@ -422,6 +435,7 @@ def merge_test_results(
                         verification_type=VerificationType(bin_result.verification_type)
                         if bin_result.verification_type
                         else None,
+                        stdout=xml_result.stdout,
                     )
                 )
         else:
@@ -448,6 +462,7 @@ def merge_test_results(
                         verification_type=VerificationType(bin_result.verification_type)
                         if bin_result.verification_type
                         else None,
+                        stdout=xml_result.stdout,
                     )
                 )
 

codeflash/verification/test_results.py

@@ -87,6 +87,7 @@ class FunctionTestInvocation:
     return_value: Optional[object]  # The return value of the function invocation
     timed_out: Optional[bool]
     verification_type: Optional[str] = VerificationType.FUNCTION_CALL
+    stdout: Optional[str] = None
 
     @property
     def unique_invocation_loop_id(self) -> str:

pyproject.toml

@@ -175,7 +175,8 @@ ignore = [
     "TD002",
     "TD003",
     "TD004",
-    "PLR2004"
+    "PLR2004",
+    "UP007" # remove once we drop 3.9 support.
 ]
 
 [tool.ruff.lint.flake8-type-checking]