From f44da94d76a7332b0457f2be3cc3972f72e40c0a Mon Sep 17 00:00:00 2001 From: Romain Malmain Date: Fri, 2 May 2025 20:09:41 +0200 Subject: [PATCH 1/2] compare unit tests using hashes --- .github/workflows/main.yml | 2 +- .gitignore | 2 ++ Dockerfile | 2 +- tests/symqemu/test.py | 39 +++++++++++++------------------------- 4 files changed, 17 insertions(+), 28 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5dd3602f478..1ba573d1e9d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,4 +19,4 @@ jobs: # build and test with multiple LLVM versions - run: docker build --build-arg LLVM_VERSION=${{ matrix.llvm_versions }} -t symqemu . - run: docker run -t symqemu make -C build check - - run: docker run -t symqemu sh -c "cd tests/symqemu && python3 -m unittest test.py" + - run: docker run -t symqemu sh -c "cd tests/symqemu && python3.11 -m unittest test.py" diff --git a/.gitignore b/.gitignore index 61fa39967b5..21999550a7c 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,5 @@ GTAGS *.swp *.patch *.gcov + +**/generated_outputs/ diff --git a/Dockerfile b/Dockerfile index 8e97471c3bb..b5b5a600a0a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ RUN apt update && apt install -y \ libglib2.0-dev \ llvm \ git \ - python3 \ + python3.11 \ python3-pip \ cmake \ wget \ diff --git a/tests/symqemu/test.py b/tests/symqemu/test.py index 447722acece..026946cb780 100644 --- a/tests/symqemu/test.py +++ b/tests/symqemu/test.py @@ -1,10 +1,10 @@ -import filecmp import pathlib import shutil import unittest import tempfile import subprocess import os +import hashlib import util @@ -19,33 +19,20 @@ def run_symqemu_and_assert_correct_result(self, binary_name): util.run_symqemu_on_test_binary(binary_name=binary_name, generated_test_cases_output_dir=symqemu_gen_output_dir) - # `filecmp.dircmp` does a "shallow" comparison, but this is not a problem here because - # the timestamps should always be different, so the actual content of the files will be compared. - # See https://docs.python.org/3/library/filecmp.html#filecmp.dircmp - expected_vs_actual_output_comparison = filecmp.dircmp(symqemu_gen_output_dir, symqemu_ref_output_dir) + expected_hashes = set() + for ref_file in symqemu_ref_output_dir.iterdir(): + with open(ref_file, 'rb', buffering=0) as f: + expected_hashes.add(hashlib.file_digest(f, "sha256").hexdigest()) - for diff_file in expected_vs_actual_output_comparison.diff_files: - ref_file = symqemu_ref_output_dir / diff_file - gen_file = symqemu_gen_output_dir / diff_file + testcase_not_found = False + for gen_file in symqemu_gen_output_dir.iterdir(): + with open(gen_file, 'rb', buffering=0) as f: + f_hash = hashlib.file_digest(f, "sha256").hexdigest() + if not f_hash in expected_hashes: + print(f"Error: content of file {gen_file} not found in expected testcases."); + testcase_not_found = True - tmp_ref = tempfile.NamedTemporaryFile("w+") - subprocess.run(["xxd", f"{ref_file}"], stdout=tmp_ref, check=True) - - tmp_gen = tempfile.NamedTemporaryFile("w+") - subprocess.run(["xxd", f"{gen_file}"], stdout=tmp_gen, check=True) - - wdiff = subprocess.run(["wdiff", f"{tmp_ref.name}", f"{tmp_gen.name}"], capture_output=True) - colordiff = subprocess.run(["colordiff"], input=wdiff.stdout, capture_output=True) - - print(f"===== Diff found in {diff_file} ======") - print(f"{colordiff.stdout.decode('utf-8').strip()}") - print(f"=================================") - print() - - self.assertEqual(expected_vs_actual_output_comparison.diff_files, []) - self.assertEqual(expected_vs_actual_output_comparison.left_only, []) - self.assertEqual(expected_vs_actual_output_comparison.right_only, []) - self.assertEqual(expected_vs_actual_output_comparison.funny_files, []) + self.assertFalse(testcase_not_found) def test_simple(self): self.run_symqemu_and_assert_correct_result('simple') From f62c7afdeee37830d30781d182abf554e5cd8ef6 Mon Sep 17 00:00:00 2001 From: Romain Malmain Date: Fri, 2 May 2025 20:25:19 +0200 Subject: [PATCH 2/2] add warning --- tests/symqemu/test.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/symqemu/test.py b/tests/symqemu/test.py index 026946cb780..a7034728b32 100644 --- a/tests/symqemu/test.py +++ b/tests/symqemu/test.py @@ -19,19 +19,29 @@ def run_symqemu_and_assert_correct_result(self, binary_name): util.run_symqemu_on_test_binary(binary_name=binary_name, generated_test_cases_output_dir=symqemu_gen_output_dir) - expected_hashes = set() + expected_hashes = {} for ref_file in symqemu_ref_output_dir.iterdir(): with open(ref_file, 'rb', buffering=0) as f: - expected_hashes.add(hashlib.file_digest(f, "sha256").hexdigest()) + f_hash = hashlib.file_digest(f, "sha256").hexdigest() + expected_hashes[f_hash] = [False, ref_file] testcase_not_found = False for gen_file in symqemu_gen_output_dir.iterdir(): with open(gen_file, 'rb', buffering=0) as f: f_hash = hashlib.file_digest(f, "sha256").hexdigest() - if not f_hash in expected_hashes: + ret = expected_hashes.get(f_hash) + if ret is not None: + ret[0] = True + expected_hashes[f_hash] = ret + else: print(f"Error: content of file {gen_file} not found in expected testcases."); testcase_not_found = True + for (is_found, fname) in expected_hashes.values(): + # (is_found, fname) = tuple(ret) + if not is_found: + print(f"Warning: expected testcase {fname} has not been generated.") + self.assertFalse(testcase_not_found) def test_simple(self):