wamr-test-suites: Save .wasm/.aot file to the report dir if test failed (#1937)

Author: no1wudi

tests/wamr-test-suites/spec-test-script/all.py

@@ -111,6 +111,7 @@ def test_case(
     verbose_flag=True,
     qemu_flag=False,
     qemu_firmware='',
+    log='',
 ):
     case_path = pathlib.Path(case_path).resolve()
     case_name = case_path.stem
@@ -170,6 +171,10 @@ def test_case(
     if not clean_up_flag:
         CMD.append("--no_cleanup")
 
+    if log != '':
+        CMD.append("--log-dir")
+        CMD.append(log)
+
     CMD.append(case_path)
     print(f"============> run {case_name} ", end="")
     with subprocess.Popen(
@@ -228,7 +233,8 @@ def test_suite(
     verbose_flag=True,
     parl_flag=False,
     qemu_flag=False,
-    qemu_firmware=''
+    qemu_firmware='',
+    log='',
 ):
     suite_path = pathlib.Path(SPEC_TEST_DIR).resolve()
     if not suite_path.exists():
@@ -264,6 +270,7 @@ def test_suite(
                         verbose_flag,
                         qemu_flag,
                         qemu_firmware,
+                        log,
                     ],
                 )
 
@@ -299,6 +306,7 @@ def test_suite(
                     verbose_flag,
                     qemu_flag,
                     qemu_firmware,
+                    log,
                 )
                 successful_case += 1
             except Exception as e:
@@ -393,6 +401,12 @@ def main():
         dest="qemu_firmware",
         help="Firmware required by qemu",
     )
+    parser.add_argument(
+        "--log",
+        default='',
+        dest="log",
+        help="Log directory",
+    )
     parser.add_argument(
         "--quiet",
         action="store_false",
@@ -435,6 +449,7 @@ def main():
             options.parl_flag,
             options.qemu_flag,
             options.qemu_firmware,
+            options.log,
         )
         end = time.time_ns()
         print(
@@ -455,7 +470,8 @@ def main():
                     options.clean_up_flag,
                     options.verbose_flag,
                     options.qemu_flag,
-                    options.qemu_firmware
+                    options.qemu_firmware,
+                    options.log
                 )
             else:
                 ret = True

tests/wamr-test-suites/spec-test-script/runtest.py

@@ -200,6 +200,8 @@ def assert_prompt(runner, prompts, timeout, is_need_execute_result):
         help="Use direct pipes instead of pseudo-tty")
 parser.add_argument('--log-file', type=str,
         help="Write messages to the named file in addition the screen")
+parser.add_argument('--log-dir', type=str,
+        help="The log directory to save the case file if test failed")
 parser.add_argument('--debug-file', type=str,
         help="Write all test interaction the named file")
 
@@ -1092,6 +1094,7 @@ def test_assert_with_exception(form, wast_tempfile, wasm_tempfile, aot_tempfile,
 
 if __name__ == "__main__":
     opts = parser.parse_args(sys.argv[1:])
+    print('Input param :',opts)
 
     if opts.aot: test_aot = True
     # default x86_64
@@ -1271,12 +1274,16 @@ def test_assert_with_exception(form, wast_tempfile, wasm_tempfile, aot_tempfile,
         print("THE FINAL EXCEPTION IS {}".format(e))
         ret_code = 101
 
+        shutil.copyfile(wasm_tempfile, os.path.join(opts.log_dir, os.path.basename(wasm_tempfile)))
+
         if opts.aot or opts.xip:
+            shutil.copyfile(aot_tempfile, os.path.join(opts.log_dir,os.path.basename(aot_tempfile)))
             if "indirect-mode" in str(e):
                 compile_wasm_to_aot(wasm_tempfile, aot_tempfile, None, opts, None, "object")
+                shutil.copyfile(aot_tempfile, os.path.join(opts.log_dir,os.path.basename(aot_tempfile)+'.o'))
                 subprocess.check_call(["llvm-objdump", "-r", aot_tempfile])
             compile_wasm_to_aot(wasm_tempfile, aot_tempfile, None, opts, None, "ir")
-            subprocess.check_call(["cat", aot_tempfile])
+            shutil.copyfile(aot_tempfile, os.path.join(opts.log_dir,os.path.basename(aot_tempfile)+".ir"))
 
     else:
         ret_code = 0

tests/wamr-test-suites/test_wamr.sh

@@ -442,9 +442,12 @@ function spec_test()
 
     if [[ ${ENABLE_QEMU} == 1 ]]; then
         ARGS_FOR_SPEC_TEST+="--qemu "
-        ARGS_FOR_SPEC_TEST+="--qemu-firmware ${QEMU_FIRMWARE}"
+        ARGS_FOR_SPEC_TEST+="--qemu-firmware ${QEMU_FIRMWARE} "
     fi
 
+    # set log directory
+    ARGS_FOR_SPEC_TEST+="--log ${REPORT_DIR}"
+
     cd ${WORK_DIR}
     echo "python3 ./all.py ${ARGS_FOR_SPEC_TEST} | tee -a ${REPORT_DIR}/spec_test_report.txt"
     python3 ./all.py ${ARGS_FOR_SPEC_TEST} | tee -a ${REPORT_DIR}/spec_test_report.txt