tests: use the Rust test runner instead of our manually generated one

This way, things like `#[cfg]` and `#[should_panic]` work.
This seems simpler than trying to parse the `#[cfg]`s from Python
and keeping the old way.
This generates a much simpler `lib.rs` instead of a `main.rs`
that tries to be its own simple test runner.

Rust's test runner doesn't distinguish between expected and unexpected failures
(it should counts an expected failure as a success),
so we lose this output information, but an expected failure really is a success so this should be fine.

Author: kkysen

scripts/test_translator.py

@@ -213,6 +213,8 @@ def __init__(self, path: str, test_functions: Optional[List[TestFunction]] = Non
 
 
 class TestDirectory:
+    rs_test_files: list[TestFile]
+
     def __init__(self, full_path: str, files: 're.Pattern', keep: List[str], log_level: str) -> None:
         self.c_files = []
         self.rs_test_files = []
@@ -269,9 +271,8 @@ def __init__(self, full_path: str, files: 're.Pattern', keep: List[str], log_lev
                 elif (filename.startswith("test_") and ext == ".rs" and
                       files.search(filename)):
                     rs_test_file = self._read_rust_test_file(path)
-
                     self.rs_test_files.append(rs_test_file)
-
+    
     def _read_c_file(self, path: str) -> Optional[CFile]:
         file_config = None
         file_flags = set()
@@ -451,7 +452,6 @@ def run(self) -> List[TestOutcome]:
             rust_file_builder.add_mod(RustMod(extensionless_rust_file,
                                               RustVisibility.Public))
 
-        match_arms = []
         rustc_extra_args = ["-C", "target-cpu=native"]
 
         # Build one binary that can call all the tests
@@ -462,6 +462,8 @@ def run(self) -> List[TestOutcome]:
 
             _, file_name = os.path.split(test_file.path)
             extensionless_file_name, _ = os.path.splitext(file_name)
+            if test_file.pass_expected:
+                rust_file_builder.add_mod(RustMod(extensionless_file_name, RustVisibility.Private))
 
             if not test_file.pass_expected:
                 try:
@@ -484,32 +486,11 @@ def run(self) -> List[TestOutcome]:
 
                 continue
 
-            for test_function in test_file.test_functions:
-                rust_file_builder.add_mod(RustMod(extensionless_file_name,
-                                                  RustVisibility.Public))
-                left = "Some(\"{}::{}\")".format(extensionless_file_name,
-                                                 test_function.name)
-                right = "{}::{}()".format(extensionless_file_name,
-                                          test_function.name)
-                match_arms.append((left, right))
-
-        match_arms.append(("e",
-                           "panic!(\"Tried to run unknown test: {:?}\", e)"))
-
-        test_main_body = [
-            RustMatch("std::env::args().nth(1).as_ref().map(AsRef::<str>::as_ref)", match_arms),
-        ]
-        test_main = RustFunction("main",
-                                 visibility=RustVisibility.Public,
-                                 body=[str(stmt) for stmt in test_main_body])
-
-        rust_file_builder.add_function(test_main)
-
-        main_file = rust_file_builder.build(self.full_path + "/src/main.rs")
+        lib_file = rust_file_builder.build(self.full_path + "/src/lib.rs")
 
-        self.generated_files["rust_src"].append(main_file)
+        self.generated_files["rust_src"].append(lib_file)
 
-        # Try and build test binary
+        # Build
         with pb.local.cwd(self.full_path):
             args = ["build"]
 
@@ -522,65 +503,45 @@ def run(self) -> List[TestOutcome]:
             retcode, stdout, stderr = cargo[args].run(retcode=None)
 
         if retcode != 0:
-            _, main_file_path_short = os.path.split(main_file.path)
+            _, lib_file_path_short = os.path.split(lib_file.path)
 
-            self.print_status(Colors.FAIL, "FAILED", "compile {}".format(main_file_path_short))
+            self.print_status(Colors.FAIL, "FAILED", "compile {}".format(lib_file_path_short))
             sys.stdout.write('\n')
             sys.stdout.write(stderr)
 
             outcomes.append(TestOutcome.UnexpectedFailure)
 
             return outcomes
 
-        for test_file in self.rs_test_files:
-            if not test_file.pass_expected:
-                continue
-
-            _, file_name = os.path.split(test_file.path)
-            extensionless_file_name, _ = os.path.splitext(file_name)
-
-            for test_function in test_file.test_functions:
-                args = ["run", "{}::{}".format(extensionless_file_name, test_function.name)]
-
-                if c.BUILD_TYPE == 'release':
-                    args.append('--release')
-
-                with pb.local.cwd(self.full_path):
-                    retcode, stdout, stderr = cargo[args].run(retcode=None)
-
-                logging.debug("stdout:%s\n", stdout)
-
-                test_str = file_name + ' - ' + test_function.name
-
-                if retcode == 0:
-                    if test_function.pass_expected:
-                        self.print_status(Colors.OKGREEN, "OK", "    test " + test_str)
-                        sys.stdout.write('\n')
-
-                        outcomes.append(TestOutcome.Success)
-                    else:
-                        self.print_status(Colors.FAIL, "FAILED", "test " + test_str)
-                        sys.stdout.write('\n')
+        # Test
+        with pb.local.cwd(self.full_path):
+            args = ["test"]
 
-                        outcomes.append(TestOutcome.UnexpectedSuccess)
+            if c.BUILD_TYPE == 'release':
+                args.append('--release')
 
-                elif retcode != 0:
-                    if test_function.pass_expected:
-                        self.print_status(Colors.FAIL, "FAILED", "test " + test_str)
-                        sys.stdout.write('\n')
-                        sys.stdout.write(stderr)
+            if self.target:
+                args += ["--target", self.target]
 
-                        outcomes.append(TestOutcome.UnexpectedFailure)
-                    else:
-                        self.print_status(Colors.OKBLUE, "FAILED", "test " + test_str)
-                        sys.stdout.write('\n')
+            retcode, stdout, stderr = cargo[args].run(retcode=None)
+        
+        if retcode != 0:
+            _, lib_file_path_short = os.path.split(lib_file.path)
 
-                        outcomes.append(TestOutcome.Failure)
+            self.print_status(Colors.FAIL, "FAILED", "test {}".format(lib_file_path_short))
+            sys.stdout.write('\n')
+            sys.stdout.write(stdout)
+        else:
+            sys.stdout.write("\n".join(line for line in stdout.split("\n") if "... ok" in line or "... FAILED" in line))
+        
+        # Don't distinguish between expected and unexpected failures.
+        # `#[should_panic]` is used for that instead of `// xfail` now.
+        # Also, `cargo test -- --format json` is unstable, so it's easier to just parse very simply.
+        for _ in range(stdout.count("... ok")):
+            outcomes.append(TestOutcome.Success)
+        for _ in range(stdout.count("... FAILED")):
+            outcomes.append(TestOutcome.UnexpectedFailure)
 
-        if not outcomes:
-            display_text = "   No rust file(s) matching " + self.files.pattern
-            display_text += " within this folder\n"
-            self.print_status(Colors.OKBLUE, "N/A", display_text)
         return outcomes
 
     def cleanup(self) -> None:

tests/README.md

@@ -23,6 +23,7 @@ extern "C" {
 // The length can be any value
 const BUFFER_SIZE: usize = 1024;
 
+#[test]
 pub fn test_example() {
     let mut buffer = [0; BUFFER_SIZE];
     let mut rust_buffer = [0; BUFFER_SIZE];
@@ -42,7 +43,8 @@ The C code can do one of two things: modify some sort of buffer or return a valu
 
 To completely skip the translation of a C file, you must add the comment `//! skip_translation` at the top of the file. That will prevent the case from showing up as red in the console output.
 
-You can also mark a Rust file as unexpected to compile, by adding `//! xfail` to the top of the file, or just expect an individual test function to fail to run by adding `// xfail` prior to the function definition.
+You can also mark a Rust file as unexpected to compile by adding `//! xfail` to the top of the file.
+For an individual test function, use the normal Rust `#[should_panic]` for `#[test]`s.
 
 Adding `//! extern_crate_X` to the top of a test file will ensure `extern crate X;` gets added to the main binary driver.
 

tests/arrays/src/test_arrays.rs

@@ -27,18 +27,21 @@ const BUFFER_SIZE: usize = 49;
 const BUFFER_SIZE2: usize = 2;
 const BUFFER_SIZEV: usize = 88;
 
+#[test]
 pub fn test_sized_array_impls() {
     unsafe {
         assert_eq!(rust_test_sized_array(), test_sized_array());
     }
 }
 
+#[test]
 pub fn test_global_incomplete_array() {
     unsafe {
         assert_eq!(rust_check_some_ints(), check_some_ints());
     }
 }
 
+#[test]
 pub fn test_buffer() {
     let mut buffer = [0; BUFFER_SIZE];
     let mut rust_buffer = [0; BUFFER_SIZE];
@@ -59,6 +62,7 @@ pub fn test_buffer() {
     }
 }
 
+#[test]
 pub fn test_buffer2() {
     let mut buffer = [0; BUFFER_SIZE2];
     let mut rust_buffer = [0; BUFFER_SIZE2];
@@ -73,6 +77,7 @@ pub fn test_buffer2() {
     assert_eq!(buffer, expected_buffer);
 }
 
+#[test]
 pub fn test_variable_arrays() {
     let mut buffer = [0; BUFFER_SIZEV];
     let mut rust_buffer = [0; BUFFER_SIZEV];
@@ -93,6 +98,7 @@ pub fn test_variable_arrays() {
     }
 }
 
+#[test]
 pub fn test_alloca_arrays() {
     let mut buffer = [0; BUFFER_SIZEV];
     let mut rust_buffer = [0; BUFFER_SIZEV];

tests/asm.aarch64/src/test_asm.rs

@@ -10,6 +10,7 @@ extern "C" {
 
 const BUFFER_SIZE: usize = 6;
 
+#[test]
 pub fn test_buffer() {
     let mut buffer = [0; BUFFER_SIZE];
     let mut rust_buffer = [0; BUFFER_SIZE];

tests/asm.arm/src/test_asm.rs

@@ -8,6 +8,7 @@ extern "C" {
 
 const BUFFER_SIZE: usize = 1;
 
+#[test]
 pub fn test_buffer() {
     let mut buffer = [0; BUFFER_SIZE];
     let mut rust_buffer = [0; BUFFER_SIZE];

tests/asm.x86_64/src/test_asm.rs

@@ -10,6 +10,7 @@ extern "C" {
 
 const BUFFER_SIZE: usize = 6;
 
+#[test]
 pub fn test_buffer() {
     let mut buffer = [0; BUFFER_SIZE];
     let mut rust_buffer = [0; BUFFER_SIZE];

tests/builtins/src/test_builtins.rs

@@ -21,6 +21,7 @@ extern "C" {
 const BUFFER_SIZE: usize = 1024;
 const BUFFER_SIZE2: usize = 10;
 
+#[test]
 pub fn test_atomics() {
     let mut buffer = [0; BUFFER_SIZE];
     let mut rust_buffer = [0; BUFFER_SIZE];
@@ -35,6 +36,7 @@ pub fn test_atomics() {
     }
 }
 
+#[test]
 pub fn test_new_atomics() {
     let mut buffer = [0; BUFFER_SIZE];
     let mut rust_buffer = [0; BUFFER_SIZE];
@@ -50,6 +52,7 @@ pub fn test_new_atomics() {
     }
 }
 
+#[test]
 pub fn test_mem_fns() {
     let const_string = "I am ten!\0";
     let mut buffer = [0; BUFFER_SIZE2];
@@ -66,6 +69,7 @@ pub fn test_mem_fns() {
     }
 }
 
+#[test]
 pub fn test_ffs() {
     for i in 0..256 {
         let ffs_ret = unsafe { ffs(i) };
@@ -85,6 +89,7 @@ pub fn test_ffs() {
     }
 }
 
+#[test]
 pub fn test_clang9_intrinsics() {
     let pinf = 1.0 / 0.0;
     let ninf = -1.0 / 0.0;
@@ -121,6 +126,7 @@ pub fn test_clang9_intrinsics() {
     }
 }
 
+#[test]
 pub fn test_assume_aligned() {
     let null = std::ptr::null_mut();
 

tests/casts/src/test_casts.rs

@@ -18,13 +18,15 @@ extern "C" {
 
 const BUFFER_SIZE: usize = 1;
 
+#[test]
 pub fn test_compiles() {
     unsafe {
         cast_stuff();
         rust_cast_stuff();
     }
 }
 
+#[test]
 pub fn test_buffer() {
     let mut buffer = [0; BUFFER_SIZE];
     let mut rust_buffer = [0; BUFFER_SIZE];
@@ -37,6 +39,7 @@ pub fn test_buffer() {
     assert_eq!(buffer, rust_buffer);
 }
 
+#[test]
 pub fn test_identity() {
     for i in 0..10 {
         let id = unsafe { identity(i) };

tests/comments/src/test_comments.rs

@@ -1,5 +1,6 @@
 use crate::comments::{rust_test_fn, CONSTANT, CONSTANT1};
 
+#[test]
 pub fn test_comments() {
     let val = unsafe { rust_test_fn() };
     assert_eq!(6, val);

tests/conditionals/src/test_conditionals.rs

@@ -26,6 +26,7 @@ const BUFFER_SIZE: usize = 4;
 const BUFFER_SIZE2: usize = 30;
 const BUFFER_SIZE3: usize = 6;
 
+#[test]
 pub fn test_buffer() {
     let mut buffer = [0; BUFFER_SIZE];
     let mut rust_buffer = [0; BUFFER_SIZE];
@@ -40,6 +41,7 @@ pub fn test_buffer() {
     assert_eq!(buffer, expected_buffer);
 }
 
+#[test]
 pub fn test_buffer2() {
     let mut buffer = [0; BUFFER_SIZE2];
     let mut rust_buffer = [0; BUFFER_SIZE2];
@@ -56,6 +58,7 @@ pub fn test_buffer2() {
     assert_eq!(buffer, expected_buffer);
 }
 
+#[test]
 pub fn test_binary_conditionals() {
     let mut buffer = [0; BUFFER_SIZE3];
     let mut rust_buffer = [0; BUFFER_SIZE3];
@@ -70,6 +73,7 @@ pub fn test_binary_conditionals() {
     assert_eq!(buffer, expected_buffer);
 }
 
+#[test]
 pub fn test_unused_conditional() {
     unsafe {
         assert_eq!(unused_conditional1(), rust_unused_conditional1());
@@ -78,6 +82,7 @@ pub fn test_unused_conditional() {
     }
 }
 
+#[test]
 pub fn test_else_if_chain(){
     unsafe {
         assert_eq!(entry4(0) , rust_entry4(0));