Provide optional bundled io atol/rtol as parameters

Change-Id: I80148fffc4b25e5470b53b559e184ac08630426a

Author: mansnils

backends/arm/scripts/run_vkml.sh

@@ -19,6 +19,7 @@ _setup_msg="please refer to ${et_root_dir}/examples/arm/setup.sh to properly ins
 
 
 model=""
+opt_flags=""
 build_path="cmake-out-vkml"
 converter="model-converter"
 
@@ -33,6 +34,7 @@ help() {
 for arg in "$@"; do
     case $arg in
       -h|--help) help ;;
+      --optional_flags=*) opt_flags="${arg#*=}";;
       --model=*) model="${arg#*=}";;
       --build_path=*) build_path="${arg#*=}";;
       *)
@@ -59,7 +61,7 @@ runner=$(find ${build_path} -name executor_runner -type f)
 
 
 echo "--------------------------------------------------------------------------------"
-echo "Running ${model} with ${runner}"
+echo "Running ${model} with ${runner} ${opt_flags}"
 echo "WARNING: The VK_ML layer driver will not provide accurate performance information"
 echo "--------------------------------------------------------------------------------"
 
@@ -75,7 +77,7 @@ fi
 log_file=$(mktemp)
 
 
-${nobuf} ${runner} -model_path ${model} | tee ${log_file}
+${nobuf} ${runner} -model_path ${model} ${opt_flags} | tee ${log_file}
 echo "[${BASH_SOURCE[0]}] execution complete, $?"
 
 # Most of these can happen for bare metal or linx executor_runner runs.

backends/arm/test/test_arm_baremetal.sh

@@ -254,8 +254,8 @@ test_models_vkml() { # End to End model tests using model_test.py
 
     # VKML
     echo "${TEST_SUITE_NAME}: Test target VKML"
-    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=vgf --model=resnet18 --extra_flags="-DET_BUNDLE_IO_ATOL=0.2 -DET_BUNDLE_IO_RTOL=0.2"
-    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=vgf --model=resnet50 --extra_flags="-DET_BUNDLE_IO_ATOL=0.2 -DET_BUNDLE_IO_RTOL=0.2"
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=vgf --model=resnet18 --extra_runtime_flags="--bundleio_atol=0.2 --bundleio_rtol=0.2"
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=vgf --model=resnet50 --extra_runtime_flags="--bundleio_atol=0.2 --bundleio_rtol=0.2"
 
     echo "${TEST_SUITE_NAME}: PASS"
 }

backends/arm/test/test_model.py

@@ -67,9 +67,15 @@ def get_args():
     parser.add_argument(
         "--extra_flags",
         required=False,
-        default=None,
+        default="",
         help="Extra cmake flags to pass the when building the executor_runner",
     )
+    parser.add_argument(
+        "--extra_runtime_flags",
+        required=False,
+        default="",
+        help="Extra runtime flags to pass the final runner/executable",
+    )
     parser.add_argument(
         "--timeout",
         required=False,
@@ -228,13 +234,14 @@ def build_vkml_runtime(
     return runner
 
 
-def run_vkml(script_path: str, pte_file: str, runner_build_path: str):
+def run_vkml(script_path: str, pte_file: str, runner_build_path: str, extra_flags: str):
     run_external_cmd(
         [
             "bash",
             os.path.join(script_path, "run_vkml.sh"),
             f"--model={pte_file}",
             f"--build_path={runner_build_path}",
+            f"--optional_flags={extra_flags}",
         ]
     )
 
@@ -297,7 +304,7 @@ def run_vkml(script_path: str, pte_file: str, runner_build_path: str):
             )
 
             start_time = time.perf_counter()
-            run_vkml(script_path, pte_file, build_path)
+            run_vkml(script_path, pte_file, build_path, args.extra_runtime_flags)
             end_time = time.perf_counter()
             print(
                 f"[Test model: {end_time - start_time:.2f} s] Tested VKML runner: {vkml_runner}"

examples/portable/executor_runner/executor_runner.cpp

@@ -83,6 +83,11 @@ DEFINE_int32(
     -1,
     "Number of CPU threads for inference. Defaults to -1, which implies we'll use a heuristic to derive the # of performant cores for a specific device.");
 
+#ifdef ET_BUNDLE_IO_ENABLED
+DEFINE_double(bundleio_rtol, 0.01, "Relative tolerance for bundled IO.");
+DEFINE_double(bundleio_atol, 0.01, "Absolute tolerance for bundled IO.");
+#endif
+
 using executorch::aten::ScalarType;
 using executorch::aten::Tensor;
 #ifdef ET_BUNDLE_IO_ENABLED
@@ -108,19 +113,6 @@ using executorch::runtime::Span;
 using executorch::runtime::Tag;
 using executorch::runtime::TensorInfo;
 
-#ifdef ET_BUNDLE_IO_ENABLED
-#if defined(ET_BUNDLE_IO_ATOL)
-constexpr float bundleio_atol = ET_BUNDLE_IO_ATOL;
-#else
-constexpr float bundleio_atol = 0.01;
-#endif
-#if defined(ET_BUNDLE_IO_RTOL)
-constexpr float bundleio_rtol = ET_BUNDLE_IO_RTOL;
-#else
-constexpr float bundleio_rtol = 0.01;
-#endif
-#endif
-
 /// Helper to manage resources for ETDump generation
 class EventTraceManager {
  public:
@@ -604,7 +596,11 @@ int main(int argc, char** argv) {
     }
 
     Error status = verify_method_outputs(
-        *method, model_pte, testset_idx, bundleio_rtol, bundleio_atol);
+        *method,
+        model_pte,
+        testset_idx,
+        FLAGS_bundleio_rtol,
+        FLAGS_bundleio_atol);
     if (status == Error::Ok) {
       ET_LOG(Info, "Model output match expected BundleIO bpte ref data.");
       ET_LOG(Info, "TEST: BundleIO index[%zu] Test_result: PASS", testset_idx);
@@ -613,8 +609,8 @@ int main(int argc, char** argv) {
       ET_LOG(
           Error,
           "Model output don't match expected BundleIO bpte ref data. rtol=%f atol=%f",
-          bundleio_rtol,
-          bundleio_atol);
+          FLAGS_bundleio_rtol,
+          FLAGS_bundleio_atol);
       ET_LOG(Error, "TEST: BundleIO index[%zu] Test_result: FAIL", testset_idx);
       ET_LOG(
           Error,