diff --git a/backends/arm/test/models/test_mobilenet_v2_arm.py b/backends/arm/test/models/test_mobilenet_v2_arm.py index f9d408c1bae..a50e2732f15 100644 --- a/backends/arm/test/models/test_mobilenet_v2_arm.py +++ b/backends/arm/test/models/test_mobilenet_v2_arm.py @@ -100,11 +100,11 @@ def test_mv2_u55_BI(self): ) if common.is_option_enabled("corstone300"): tester.run_method_and_compare_outputs( - atol=1.0, qtol=1, inputs=self.model_inputs + atol=1.0, qtol=1, inputs=self.model_inputs, target_board="corstone-300" ) def test_mv2_u85_BI(self): - ( + tester = ( ArmTester( self.mv2, example_inputs=self.model_inputs, @@ -116,4 +116,9 @@ def test_mv2_u85_BI(self): .check(list(self.operators_after_quantization)) .partition() .to_executorch() + .serialize() ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs( + atol=1.0, qtol=1, inputs=self.model_inputs, target_board="corstone-320" + ) diff --git a/backends/arm/test/ops/test_add.py b/backends/arm/test/ops/test_add.py index cff8af11654..e3eeb187da3 100644 --- a/backends/arm/test/ops/test_add.py +++ b/backends/arm/test/ops/test_add.py @@ -137,16 +137,22 @@ def test_add_u55_BI(self, test_data: torch.Tensor): test_data, ) if common.is_option_enabled("corstone300"): - tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) + tester.run_method_and_compare_outputs( + qtol=1, inputs=test_data, target_board="corstone-300" + ) @parameterized.expand(Add.test_parameters) def test_add_u85_BI(self, test_data: torch.Tensor): test_data = (test_data,) - self._test_add_ethos_BI_pipeline( + tester = self._test_add_ethos_BI_pipeline( self.Add(), common.get_u85_compile_spec(permute_memory_to_nhwc=True), test_data, ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs( + qtol=1, inputs=test_data, target_board="corstone-320" + ) @parameterized.expand(Add2.test_parameters) def test_add2_tosa_MI(self, operand1: torch.Tensor, operand2: torch.Tensor): @@ -165,11 +171,17 @@ def test_add2_u55_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): self.Add2(), common.get_u55_compile_spec(), test_data ) if common.is_option_enabled("corstone300"): - tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) + tester.run_method_and_compare_outputs( + qtol=1, inputs=test_data, target_board="corstone-300" + ) @parameterized.expand(Add2.test_parameters) def test_add2_u85_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): test_data = (operand1, operand2) - self._test_add_ethos_BI_pipeline( + tester = self._test_add_ethos_BI_pipeline( self.Add2(), common.get_u85_compile_spec(), test_data ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs( + qtol=1, inputs=test_data, target_board="corstone-320" + ) diff --git a/backends/arm/test/runner_utils.py b/backends/arm/test/runner_utils.py index 2935a2e13ef..0a0143e14c6 100644 --- a/backends/arm/test/runner_utils.py +++ b/backends/arm/test/runner_utils.py @@ -177,6 +177,7 @@ def __init__( self.qp_input: list[QuantizationParams] = None self.qp_output: QuantizationParams = None self.timeout = 120 + self.target_board: str = None self._has_init_run = False @@ -185,11 +186,17 @@ def init_run( exported_program: ExportedProgram, edge_program: ExportedProgram, is_quantized: bool, + target_board: str, ): + + if target_board not in ["corstone-300", "corstone-320"]: + raise RuntimeError(f"Unknown target board: {target_board}") + self.input_names = _get_input_names(edge_program) self.output_node = _get_output_node(exported_program) self.output_name = self.output_node.name self.is_quantized = is_quantized + self.target_board = target_board if is_quantized: self.qp_input = _get_input_quantization_params(exported_program) @@ -205,7 +212,7 @@ def init_run( def set_timeout(self, timeout: int): self.timeout = timeout - def run_corstone300( + def run_corstone( self, inputs: Tuple[torch.Tensor], ) -> list[torch.Tensor]: @@ -231,7 +238,7 @@ def run_corstone300( ) elf_path = os.path.join( "cmake-out", - "arm_semihosting_executor_runner_corstone-300", + f"arm_semihosting_executor_runner_{self.target_board}", "arm_executor_runner", ) assert os.path.exists( @@ -242,32 +249,66 @@ def run_corstone300( for input_path in input_paths: cmd_line += f" -i {input_path}" - command_args = [ - "FVP_Corstone_SSE-300_Ethos-U55", - "-C", - "ethosu.num_macs=128", - "-C", - "mps3_board.visualisation.disable-visualisation=1", - "-C", - "mps3_board.telnetterminal0.start_telnet=0", - "-C", - "mps3_board.uart0.out_file='-'", - "-C", - "cpu0.CFGITCMSZ=11", - "-C", - "cpu0.semihosting-enable=1", - "-C", - "cpu0.semihosting-stack_base=0", - "-C", - "cpu0.semihosting-heap_limit=0", - "-C", - f"cpu0.semihosting-cmd_line='{cmd_line}'", - "-a", - elf_path, - "--timelimit", - f"{self.timeout}", - ] - result = _run_cmd(command_args, check=False) + command_args = { + "corstone-300": [ + "FVP_Corstone_SSE-300_Ethos-U55", + "-C", + "ethosu.num_macs=128", + "-C", + "mps3_board.visualisation.disable-visualisation=1", + "-C", + "mps3_board.telnetterminal0.start_telnet=0", + "-C", + "mps3_board.uart0.out_file='-'", + "-C", + "cpu0.CFGITCMSZ=11", + "-C", + "cpu0.semihosting-enable=1", + "-C", + "cpu0.semihosting-stack_base=0", + "-C", + "cpu0.semihosting-heap_limit=0", + "-C", + f"cpu0.semihosting-cmd_line='{cmd_line}'", + "-a", + elf_path, + "--timelimit", + f"{self.timeout}", + ], + "corstone-320": [ + "FVP_Corstone_SSE-320", + "-C", + "mps4_board.subsystem.ethosu.num_macs=128", + "-C", + "mps4_board.visualisation.disable-visualisation=1", + "-C", + "mps4_board.telnetterminal0.start_telnet=0", + "-C", + "mps4_board.uart0.out_file='-'", + "-C", + "mps4_board.uart0.unbuffered_output=1", + "-C", + "mps4_board.uart0.shutdown_on_eot=1", + "-C", + "mps4_board.subsystem.cpu0.semihosting-enable=1", + "-C", + "mps4_board.subsystem.cpu0.semihosting-stack_base=0", + "-C", + "mps4_board.subsystem.cpu0.semihosting-heap_limit=0", + "-C", + f"mps4_board.subsystem.cpu0.semihosting-cmd_line='{cmd_line}'", + "-a", + elf_path, + "--timelimit", + f"{self.timeout}", + ], + } + + result = _run_cmd(command_args[self.target_board], check=False) + if result.returncode != 0: + raise RuntimeError( + f"Failed to run {command_args[self.target_board]}\nError: {result.stderr.decode()}" + ) result_stdout = result.stdout.decode() error_regex = r"(^[EF][: ].*$)|(^.*Hard fault.*$)|(^.*Assertion.*$)" @@ -276,10 +317,8 @@ def run_corstone300( # regex to check for error or fault messages in stdout from FVP if re.compile(error_regex, re.MULTILINE).search(result_stdout): raise RuntimeError( - f"Corstone simulation failed, log: \n {result_stdout}\n{result.stderr.decode()}" + f"Corstone simulation failed:\ncmd: {command_args[self.target_board]}\n, log: \n {result_stdout}\n{result.stderr.decode()}" ) - elif "E [" in result_stdout: - logger.error(result_stdout) tosa_ref_output = np.fromfile(out_path_with_suffix, dtype=np.float32) output_shape = self.output_node.args[0][0].meta["val"].shape diff --git a/backends/arm/test/tester/arm_tester.py b/backends/arm/test/tester/arm_tester.py index 7e8a1198ad0..eb52f4b2070 100644 --- a/backends/arm/test/tester/arm_tester.py +++ b/backends/arm/test/tester/arm_tester.py @@ -98,7 +98,7 @@ def __init__(self, runner_util: RunnerUtil, timeout: int = 1): self.runner.set_timeout(timeout) def run_artifact(self, inputs): - return self.runner.run_corstone300(inputs) + return self.runner.run_corstone(inputs) def dump_artifact(self, path_to_dump: Optional[str]): if not path_to_dump: @@ -226,6 +226,7 @@ def run_method_and_compare_outputs( self, inputs: Optional[Tuple[torch.Tensor]] = None, stage: Optional[str] = None, + target_board: Optional[str] = "corstone-300", num_runs=1, atol=1e-03, rtol=1e-03, @@ -260,7 +261,12 @@ def run_method_and_compare_outputs( edge_program = self.stages[ self.stage_name(tester.ToEdge) ].artifact.exported_program() - self.runner_util.init_run(exported_program, edge_program, is_quantized) + self.runner_util.init_run( + exported_program, + edge_program, + is_quantized, + target_board, + ) if is_quantized: reference_stage = self.stages[self.stage_name(tester.Quantize)] diff --git a/examples/arm/setup.sh b/examples/arm/setup.sh index 73d552cb268..20224b9e9c3 100755 --- a/examples/arm/setup.sh +++ b/examples/arm/setup.sh @@ -45,20 +45,28 @@ function verify_md5() { script_dir=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) if [[ "${ARCH}" == "x86_64" ]]; then - # FVP - fvp_url="https://developer.arm.com/-/media/Arm%20Developer%20Community/Downloads/OSS/FVP/Corstone-300/FVP_Corstone_SSE-300_11.22_20_Linux64.tgz?rev=018659bd574f4e7b95fa647e7836ccf4&hash=22A79103C6FA5FFA7AFF3BE0447F3FF9" - fvp_model_dir="Linux64_GCC-9.3" - fvp_md5_checksum="98e93b949d0fbac977292d8668d34523" + # FVPs + corstone300_url="https://developer.arm.com/-/media/Arm%20Developer%20Community/Downloads/OSS/FVP/Corstone-300/FVP_Corstone_SSE-300_11.22_20_Linux64.tgz?rev=018659bd574f4e7b95fa647e7836ccf4&hash=22A79103C6FA5FFA7AFF3BE0447F3FF9" + corstone300_model_dir="Linux64_GCC-9.3" + corstone300_md5_checksum="98e93b949d0fbac977292d8668d34523" + + corstone320_url="https://developer.arm.com/-/media/Arm%20Developer%20Community/Downloads/OSS/FVP/Corstone-320/FVP_Corstone_SSE-320_11.27_25_Linux64.tgz?rev=a507bffc219a4d5792f1192ab7002d89&hash=D9A824AA8227D2E679C9B9787FF4E8B6FBE3D7C6" + corstone320_model_dir="Linux64_GCC-9.3" + corstone320_md5_checksum="3deb3c68f9b2d145833f15374203514d" # toochain toolchain_url="https://armkeil.blob.core.windows.net/developer/Files/downloads/gnu/12.3.rel1/binrel/arm-gnu-toolchain-12.3.rel1-x86_64-arm-none-eabi.tar.xz" toolchain_dir="arm-gnu-toolchain-12.3.rel1-x86_64-arm-none-eabi" toolchain_md5_checksum="00ebb1b70b1f88906c61206457eacb61" elif [[ "${ARCH}" == "aarch64" ]] || [[ "${ARCH}" == "arm64" ]]; then - # FVP - fvp_url="https://developer.arm.com/-/media/Arm%20Developer%20Community/Downloads/OSS/FVP/Corstone-300/FVP_Corstone_SSE-300_11.22_20_Linux64_armv8l.tgz?rev=9cc6e9a32bb947ca9b21fa162144cb01&hash=7657A4CF27D42E892E3F08D452AAB073" - fvp_model_dir="Linux64_armv8l_GCC-9.3" - fvp_md5_checksum="cbbabbe39b07939cff7a3738e1492ef1" + # FVPs + corstone300_url="https://developer.arm.com/-/media/Arm%20Developer%20Community/Downloads/OSS/FVP/Corstone-300/FVP_Corstone_SSE-300_11.22_20_Linux64_armv8l.tgz?rev=9cc6e9a32bb947ca9b21fa162144cb01&hash=7657A4CF27D42E892E3F08D452AAB073" + corstone300_model_dir="Linux64_armv8l_GCC-9.3" + corstone300_md5_checksum="cbbabbe39b07939cff7a3738e1492ef1" + + corstone320_url="https://developer.arm.com/-/media/Arm%20Developer%20Community/Downloads/OSS/FVP/Corstone-320/FVP_Corstone_SSE-320_11.27_25_Linux64_armv8l.tgz?rev=b6ebe0923cb84f739e017385fd3c333c&hash=8965C4B98E2FF7F792A099B08831FE3CB6120493" + corstone320_model_dir="Linux64_armv8l_GCC-9.3" + corstone320_md5_checksum="3889f1d80a6d9861ea4aa6f1c88dd0ae" # toochain if [[ "${OS}" == "Darwin" ]]; then @@ -105,26 +113,50 @@ function setup_fvp() { fi # Download and install the Corstone 300 FVP simulator platform - cd "${root_dir}" - if [[ ! -e FVP_cs300.tgz ]]; then - echo "[${FUNCNAME[0]}] Downloading FVP ..." - curl --output FVP_cs300.tgz "${fvp_url}" - verify_md5 ${fvp_md5_checksum} FVP_cs300.tgz - fi - - echo "[${FUNCNAME[0]}] Installing FVP ..." - rm -rf FVP - mkdir -p FVP - cd FVP - tar xf ../FVP_cs300.tgz - ./FVP_Corstone_SSE-300.sh --i-agree-to-the-contained-eula --force --destination ./ --quiet --no-interactive - - fvp_bin_path="$(cd models/${fvp_model_dir} && pwd)" - export PATH=${PATH}:${fvp_bin_path} - - hash FVP_Corstone_SSE-300_Ethos-U55 - echo "export PATH=\${PATH}:${fvp_bin_path}" >> ${setup_path_script} - + fvps=("corstone300" "corstone320") + + for fvp in "${fvps[@]}"; do + cd "${root_dir}" + if [[ ! -e "FVP_${fvp}.tgz" ]]; then + echo "[${FUNCNAME[0]}] Downloading FVP ${fvp}..." + url_variable=${fvp}_url + fvp_url=${!url_variable} + curl --output "FVP_${fvp}.tgz" "${fvp_url}" + md5_variable=${fvp}_md5_checksum + fvp_md5_checksum=${!md5_variable} + verify_md5 ${fvp_md5_checksum} FVP_${fvp}.tgz + fi + + echo "[${FUNCNAME[0]}] Installing FVP ${fvp}..." + rm -rf FVP-${fvp} + mkdir -p FVP-${fvp} + cd FVP-${fvp} + tar xf ../FVP_${fvp}.tgz + + # Install the FVP + case ${fvp} in + corstone300) + ./FVP_Corstone_SSE-300.sh --i-agree-to-the-contained-eula --force --destination ./ --quiet --no-interactive + ;; + corstone320) + ./FVP_Corstone_SSE-320.sh --i-agree-to-the-contained-eula --force --destination ./ --quiet --no-interactive + ;; + *) + echo "[${FUNCNAME[0]}] Error: Unknown FVP model ${fvp}. Exiting." + exit 1 + ;; + esac + + model_dir_variable=${fvp}_model_dir + fvp_model_dir=${!model_dir_variable} + fvp_bin_path="$(cd models/${fvp_model_dir} && pwd)" + export PATH=${PATH}:${fvp_bin_path} + + echo "export PATH=\${PATH}:${fvp_bin_path}" >> ${setup_path_script} + done + + # Fixup for Corstone-320 python dependency + echo "export LD_LIBRARY_PATH=${root_dir}/FVP-corstone320/python/lib/" >> ${setup_path_script} } function setup_toolchain() {