diff --git a/backends/arm/test/test_arm_baremetal.sh b/backends/arm/test/test_arm_baremetal.sh index af3f4bea501..f1be263013f 100755 --- a/backends/arm/test/test_arm_baremetal.sh +++ b/backends/arm/test/test_arm_baremetal.sh @@ -17,7 +17,6 @@ _setup_msg="please refer to ${et_root_dir}/examples/arm/setup.sh to properly ins TEST_SUITE=$1 -TOSA_VERSION="${2:-TOSA-1.0+INT}" # Source the tools # This should be prepared by the setup.sh @@ -157,17 +156,23 @@ test_run_ethosu_fvp() { # End to End model tests using run.sh # TOSA quantized echo "${TEST_SUITE_NAME}: Test ethos-u target TOSA" - examples/arm/run.sh --et_build_root=arm_test/test_run --target=${TOSA_VERSION} --model_name=add - examples/arm/run.sh --et_build_root=arm_test/test_run --target=${TOSA_VERSION} --model_name=mul + examples/arm/run.sh --et_build_root=arm_test/test_run --target=TOSA-1.0+INT --model_name=add + examples/arm/run.sh --et_build_root=arm_test/test_run --target=TOSA-1.0+INT --model_name=mul # Ethos-U55 echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U55" examples/arm/run.sh --et_build_root=arm_test/test_run --target=ethos-u55-128 --model_name=add + examples/arm/run.sh --et_build_root=arm_test/test_run --target=ethos-u55-128 --model_name=add --bundleio + examples/arm/run.sh --et_build_root=arm_test/test_run --target=ethos-u55-128 --model_name=add --bundleio --etdump + examples/arm/run.sh --et_build_root=arm_test/test_run --target=ethos-u55-128 --model_name=add --etdump examples/arm/run.sh --et_build_root=arm_test/test_run --target=ethos-u55-128 --model_name=mul # Ethos-U85 echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U85" examples/arm/run.sh --et_build_root=arm_test/test_run --target=ethos-u85-128 --model_name=add + examples/arm/run.sh --et_build_root=arm_test/test_run --target=ethos-u85-128 --model_name=add --bundleio + examples/arm/run.sh --et_build_root=arm_test/test_run --target=ethos-u85-128 --model_name=add --bundleio --etdump + examples/arm/run.sh --et_build_root=arm_test/test_run --target=ethos-u85-128 --model_name=add --etdump examples/arm/run.sh --et_build_root=arm_test/test_run --target=ethos-u85-128 --model_name=mul # Cortex-M op tests @@ -187,17 +192,17 @@ test_models_tosa() { # End to End model tests using model_test.py # TOSA quantized echo "${TEST_SUITE_NAME}: Test ethos-u target TOSA" - python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=${TOSA_VERSION} --model=mv2 - python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=${TOSA_VERSION} --model=mv3 - python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=${TOSA_VERSION} --model=lstm - python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=${TOSA_VERSION} --model=edsr - # python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=${TOSA_VERSION} --model=emformer_transcribe # Takes long time to run - # python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=${TOSA_VERSION} --model=emformer_join # Takes long time to run - python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=${TOSA_VERSION} --model=w2l - python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=${TOSA_VERSION} --model=ic3 - python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=${TOSA_VERSION} --model=ic4 - python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=${TOSA_VERSION} --model=resnet18 - python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=${TOSA_VERSION} --model=resnet50 + python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA-1.0+INT --model=mv2 + python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA-1.0+INT --model=mv3 + python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA-1.0+INT --model=lstm + python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA-1.0+INT --model=edsr + # python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA-1.0+INT --model=emformer_transcribe # Takes long time to run + # python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA-1.0+INT --model=emformer_join # Takes long time to run + python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA-1.0+INT --model=w2l + python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA-1.0+INT --model=ic3 + python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA-1.0+INT --model=ic4 + python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA-1.0+INT --model=resnet18 + python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA-1.0+INT --model=resnet50 echo "${TEST_SUITE_NAME}: PASS" } diff --git a/examples/arm/aot_arm_compiler.py b/examples/arm/aot_arm_compiler.py index d6a1eab3205..c7a9dde2b1b 100644 --- a/examples/arm/aot_arm_compiler.py +++ b/examples/arm/aot_arm_compiler.py @@ -8,6 +8,7 @@ # Example script for exporting simple models to flatbuffer import argparse +import copy import json import logging import os @@ -45,6 +46,7 @@ from executorch.backends.cortex_m.passes.replace_quant_nodes_pass import ( ReplaceQuantNodesPass, ) +from executorch.devtools import generate_etrecord from executorch.devtools.backend_debug import get_delegation_info from executorch.devtools.bundled_program.config import MethodTestCase, MethodTestSuite @@ -505,6 +507,13 @@ def get_args(): default=False, help="Flag for producing BundleIO bpte file with input/output test/ref data.", ) + parser.add_argument( + "--etrecord", + action="store_true", + required=False, + default=False, + help="Flag for producing a etrecord file.", + ) parser.add_argument( "-t", "--target", @@ -822,6 +831,8 @@ def transform_for_cortex_m_backend(edge): dump_delegation_info(edge, args.intermediates) + edge_program_manager_copy = copy.deepcopy(edge) + try: exec_prog = edge.to_executorch( config=ExecutorchBackendConfig(extract_delegate_segments=False) @@ -843,9 +854,9 @@ def transform_for_cortex_m_backend(edge): ) if args.bundleio: - output_name = f"{output_name}.bpte" + output_file_name = f"{output_name}.bpte" else: - output_name = f"{output_name}.pte" + output_file_name = f"{output_name}.pte" if args.output is not None: if args.output.endswith(".pte") or args.output.endswith(".bpte"): @@ -858,19 +869,25 @@ def transform_for_cortex_m_backend(edge): raise RuntimeError( f"When not using --bundleio a .bpte file should not be use as --output {args.output}" ) - output_name = args.output + output_file_name = args.output else: # --output is a folder - output_name = os.path.join(args.output, output_name) + output_file_name = os.path.join(args.output, output_file_name) + + if args.bundleio or args.etrecord: + etrecord_file_name = os.path.splitext(output_file_name)[0] + "_etrecord.bin" + # Generate ETRecord + generate_etrecord(etrecord_file_name, edge_program_manager_copy, exec_prog) + print(f"ETRecord saved as {etrecord_file_name}") if args.bundleio: # Realize the quantization impact on numerics when generating reference output reference_model = original_model if not model_int8 else model_int8 - save_bpte_program(exec_prog, reference_model, output_name) - print(f"Bundle PTE file saved as {output_name}") + save_bpte_program(exec_prog, reference_model, output_file_name) + print(f"Bundle PTE file saved as {output_file_name}") else: - save_pte_program(exec_prog, output_name) - print(f"PTE file saved as {output_name}") + save_pte_program(exec_prog, output_file_name) + print(f"PTE file saved as {output_file_name}") if args.evaluate: evaluate_model( diff --git a/examples/arm/run.sh b/examples/arm/run.sh index 60fa0896aba..e93ba6f4572 100755 --- a/examples/arm/run.sh +++ b/examples/arm/run.sh @@ -184,9 +184,11 @@ fi cd $et_root_dir devtools_flag="" bundleio_flag="" +etrecord_flag="" et_dump_flag="" if [ "$build_with_etdump" = true ] ; then et_dump_flag="--etdump" + etrecord_flag="--etrecord" fi if [ "$bundleio" = true ] ; then @@ -263,7 +265,7 @@ for i in "${!test_model[@]}"; do model_compiler_flags="${model_compiler_flags} --model_input=${model_input}" fi - ARM_AOT_CMD="python3 -m examples.arm.aot_arm_compiler --model_name=${model} --target=${target} ${model_compiler_flags} --intermediate=${output_folder} --output=${pte_file} --system_config=${system_config} --memory_mode=${memory_mode} $bundleio_flag --config=${config}" + ARM_AOT_CMD="python3 -m examples.arm.aot_arm_compiler --model_name=${model} --target=${target} ${model_compiler_flags} --intermediate=${output_folder} --output=${pte_file} --system_config=${system_config} --memory_mode=${memory_mode} $bundleio_flag ${etrecord_flag} --config=${config}" echo "CALL ${ARM_AOT_CMD}" >&2 ${ARM_AOT_CMD} 1>&2