Compare unit tests using file content hashes (#75)

We can pass when a test case isn't found but not so important

Author: aurelf

.github/workflows/main.yml

@@ -19,4 +19,4 @@ jobs:
     # build and test with multiple LLVM versions
     - run: docker build --build-arg LLVM_VERSION=${{ matrix.llvm_versions }} -t symqemu .
     - run: docker run -t symqemu make -C build check
-    - run: docker run -t symqemu sh -c "cd tests/symqemu && python3 -m unittest test.py"
+    - run: docker run -t symqemu sh -c "cd tests/symqemu && python3.11 -m unittest test.py"

.gitignore

@@ -20,3 +20,5 @@ GTAGS
 *.swp
 *.patch
 *.gcov
+
+**/generated_outputs/

Dockerfile

@@ -6,7 +6,7 @@ RUN apt update && apt install -y \
     libglib2.0-dev \
     llvm \
     git \
-    python3 \
+    python3.11 \
     python3-pip \
     cmake \
     wget \

tests/symqemu/test.py

@@ -1,10 +1,10 @@
-import filecmp
 import pathlib
 import shutil
 import unittest
 import tempfile
 import subprocess
 import os
+import hashlib
 
 import util
 
@@ -19,33 +19,30 @@ def run_symqemu_and_assert_correct_result(self, binary_name):
 
         util.run_symqemu_on_test_binary(binary_name=binary_name, generated_test_cases_output_dir=symqemu_gen_output_dir)
 
-        # `filecmp.dircmp` does a "shallow" comparison, but this is not a problem here because
-        # the timestamps should always be different, so the actual content of the files will be compared.
-        # See https://docs.python.org/3/library/filecmp.html#filecmp.dircmp
-        expected_vs_actual_output_comparison = filecmp.dircmp(symqemu_gen_output_dir, symqemu_ref_output_dir)
-
-        for diff_file in expected_vs_actual_output_comparison.diff_files:
-            ref_file = symqemu_ref_output_dir / diff_file
-            gen_file = symqemu_gen_output_dir / diff_file
-
-            tmp_ref = tempfile.NamedTemporaryFile("w+")
-            subprocess.run(["xxd", f"{ref_file}"], stdout=tmp_ref, check=True)
-
-            tmp_gen = tempfile.NamedTemporaryFile("w+")
-            subprocess.run(["xxd", f"{gen_file}"], stdout=tmp_gen, check=True)
-
-            wdiff = subprocess.run(["wdiff", f"{tmp_ref.name}", f"{tmp_gen.name}"], capture_output=True)
-            colordiff = subprocess.run(["colordiff"], input=wdiff.stdout, capture_output=True)
-
-            print(f"===== Diff found in {diff_file} ======")
-            print(f"{colordiff.stdout.decode('utf-8').strip()}")
-            print(f"=================================")
-            print()
-
-        self.assertEqual(expected_vs_actual_output_comparison.diff_files, [])
-        self.assertEqual(expected_vs_actual_output_comparison.left_only, [])
-        self.assertEqual(expected_vs_actual_output_comparison.right_only, [])
-        self.assertEqual(expected_vs_actual_output_comparison.funny_files, [])
+        expected_hashes = {}
+        for ref_file in symqemu_ref_output_dir.iterdir():
+            with open(ref_file, 'rb', buffering=0) as f:
+                f_hash = hashlib.file_digest(f, "sha256").hexdigest()
+                expected_hashes[f_hash] = [False, ref_file]
+
+        testcase_not_found = False
+        for gen_file in symqemu_gen_output_dir.iterdir():
+            with open(gen_file, 'rb', buffering=0) as f:
+                f_hash = hashlib.file_digest(f, "sha256").hexdigest()
+                ret = expected_hashes.get(f_hash)
+                if ret is not None:
+                    ret[0] = True
+                    expected_hashes[f_hash] = ret
+                else:
+                    print(f"Error: content of file {gen_file} not found in expected testcases.");
+                    testcase_not_found = True
+
+        for (is_found, fname) in expected_hashes.values():
+            # (is_found, fname) = tuple(ret)
+            if not is_found:
+                print(f"Warning: expected testcase {fname} has not been generated.")
+
+        self.assertFalse(testcase_not_found)
 
     def test_simple(self):
         self.run_symqemu_and_assert_correct_result('simple')