Corstone-320 download and tests (pytorch#5787)

Summary:
Handle downloading of Corstone-320 FVP and add support for running unit tests on the platform.

Pull Request resolved: pytorch#5787

Reviewed By: digantdesai

Differential Revision: D63762487

Pulled By: mergennachin

fbshipit-source-id: 286b80ff9bf38e8faf87f3668f5c3884ea82fa26

Author: per

backends/arm/test/models/test_mobilenet_v2_arm.py

@@ -100,11 +100,11 @@ def test_mv2_u55_BI(self):
         )
         if common.is_option_enabled("corstone300"):
             tester.run_method_and_compare_outputs(
-                atol=1.0, qtol=1, inputs=self.model_inputs
+                atol=1.0, qtol=1, inputs=self.model_inputs, target_board="corstone-300"
             )
 
     def test_mv2_u85_BI(self):
-        (
+        tester = (
             ArmTester(
                 self.mv2,
                 example_inputs=self.model_inputs,
@@ -116,4 +116,9 @@ def test_mv2_u85_BI(self):
             .check(list(self.operators_after_quantization))
             .partition()
             .to_executorch()
+            .serialize()
         )
+        if common.is_option_enabled("corstone300"):
+            tester.run_method_and_compare_outputs(
+                atol=1.0, qtol=1, inputs=self.model_inputs, target_board="corstone-320"
+            )

backends/arm/test/ops/test_add.py

@@ -137,16 +137,22 @@ def test_add_u55_BI(self, test_data: torch.Tensor):
             test_data,
         )
         if common.is_option_enabled("corstone300"):
-            tester.run_method_and_compare_outputs(qtol=1, inputs=test_data)
+            tester.run_method_and_compare_outputs(
+                qtol=1, inputs=test_data, target_board="corstone-300"
+            )
 
     @parameterized.expand(Add.test_parameters)
     def test_add_u85_BI(self, test_data: torch.Tensor):
         test_data = (test_data,)
-        self._test_add_ethos_BI_pipeline(
+        tester = self._test_add_ethos_BI_pipeline(
             self.Add(),
             common.get_u85_compile_spec(permute_memory_to_nhwc=True),
             test_data,
         )
+        if common.is_option_enabled("corstone300"):
+            tester.run_method_and_compare_outputs(
+                qtol=1, inputs=test_data, target_board="corstone-320"
+            )
 
     @parameterized.expand(Add2.test_parameters)
     def test_add2_tosa_MI(self, operand1: torch.Tensor, operand2: torch.Tensor):
@@ -165,11 +171,17 @@ def test_add2_u55_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
             self.Add2(), common.get_u55_compile_spec(), test_data
         )
         if common.is_option_enabled("corstone300"):
-            tester.run_method_and_compare_outputs(qtol=1, inputs=test_data)
+            tester.run_method_and_compare_outputs(
+                qtol=1, inputs=test_data, target_board="corstone-300"
+            )
 
     @parameterized.expand(Add2.test_parameters)
     def test_add2_u85_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
         test_data = (operand1, operand2)
-        self._test_add_ethos_BI_pipeline(
+        tester = self._test_add_ethos_BI_pipeline(
             self.Add2(), common.get_u85_compile_spec(), test_data
         )
+        if common.is_option_enabled("corstone300"):
+            tester.run_method_and_compare_outputs(
+                qtol=1, inputs=test_data, target_board="corstone-320"
+            )

backends/arm/test/runner_utils.py

@@ -177,6 +177,7 @@ def __init__(
         self.qp_input: list[QuantizationParams] = None
         self.qp_output: QuantizationParams = None
         self.timeout = 120
+        self.target_board: str = None
 
         self._has_init_run = False
 
@@ -185,11 +186,17 @@ def init_run(
         exported_program: ExportedProgram,
         edge_program: ExportedProgram,
         is_quantized: bool,
+        target_board: str,
     ):
+
+        if target_board not in ["corstone-300", "corstone-320"]:
+            raise RuntimeError(f"Unknown target board: {target_board}")
+
         self.input_names = _get_input_names(edge_program)
         self.output_node = _get_output_node(exported_program)
         self.output_name = self.output_node.name
         self.is_quantized = is_quantized
+        self.target_board = target_board
 
         if is_quantized:
             self.qp_input = _get_input_quantization_params(exported_program)
@@ -205,7 +212,7 @@ def init_run(
     def set_timeout(self, timeout: int):
         self.timeout = timeout
 
-    def run_corstone300(
+    def run_corstone(
         self,
         inputs: Tuple[torch.Tensor],
     ) -> list[torch.Tensor]:
@@ -231,7 +238,7 @@ def run_corstone300(
             )
         elf_path = os.path.join(
             "cmake-out",
-            "arm_semihosting_executor_runner_corstone-300",
+            f"arm_semihosting_executor_runner_{self.target_board}",
             "arm_executor_runner",
         )
         assert os.path.exists(
@@ -242,32 +249,66 @@ def run_corstone300(
         for input_path in input_paths:
             cmd_line += f" -i {input_path}"
 
-        command_args = [
-            "FVP_Corstone_SSE-300_Ethos-U55",
-            "-C",
-            "ethosu.num_macs=128",
-            "-C",
-            "mps3_board.visualisation.disable-visualisation=1",
-            "-C",
-            "mps3_board.telnetterminal0.start_telnet=0",
-            "-C",
-            "mps3_board.uart0.out_file='-'",
-            "-C",
-            "cpu0.CFGITCMSZ=11",
-            "-C",
-            "cpu0.semihosting-enable=1",
-            "-C",
-            "cpu0.semihosting-stack_base=0",
-            "-C",
-            "cpu0.semihosting-heap_limit=0",
-            "-C",
-            f"cpu0.semihosting-cmd_line='{cmd_line}'",
-            "-a",
-            elf_path,
-            "--timelimit",
-            f"{self.timeout}",
-        ]
-        result = _run_cmd(command_args, check=False)
+        command_args = {
+            "corstone-300": [
+                "FVP_Corstone_SSE-300_Ethos-U55",
+                "-C",
+                "ethosu.num_macs=128",
+                "-C",
+                "mps3_board.visualisation.disable-visualisation=1",
+                "-C",
+                "mps3_board.telnetterminal0.start_telnet=0",
+                "-C",
+                "mps3_board.uart0.out_file='-'",
+                "-C",
+                "cpu0.CFGITCMSZ=11",
+                "-C",
+                "cpu0.semihosting-enable=1",
+                "-C",
+                "cpu0.semihosting-stack_base=0",
+                "-C",
+                "cpu0.semihosting-heap_limit=0",
+                "-C",
+                f"cpu0.semihosting-cmd_line='{cmd_line}'",
+                "-a",
+                elf_path,
+                "--timelimit",
+                f"{self.timeout}",
+            ],
+            "corstone-320": [
+                "FVP_Corstone_SSE-320",
+                "-C",
+                "mps4_board.subsystem.ethosu.num_macs=128",
+                "-C",
+                "mps4_board.visualisation.disable-visualisation=1",
+                "-C",
+                "mps4_board.telnetterminal0.start_telnet=0",
+                "-C",
+                "mps4_board.uart0.out_file='-'",
+                "-C",
+                "mps4_board.uart0.unbuffered_output=1",
+                "-C",
+                "mps4_board.uart0.shutdown_on_eot=1",
+                "-C",
+                "mps4_board.subsystem.cpu0.semihosting-enable=1",
+                "-C",
+                "mps4_board.subsystem.cpu0.semihosting-stack_base=0",
+                "-C",
+                "mps4_board.subsystem.cpu0.semihosting-heap_limit=0",
+                "-C",
+                f"mps4_board.subsystem.cpu0.semihosting-cmd_line='{cmd_line}'",
+                "-a",
+                elf_path,
+                "--timelimit",
+                f"{self.timeout}",
+            ],
+        }
+
+        result = _run_cmd(command_args[self.target_board], check=False)
+        if result.returncode != 0:
+            raise RuntimeError(
+                f"Failed to run {command_args[self.target_board]}\nError: {result.stderr.decode()}"
+            )
         result_stdout = result.stdout.decode()
 
         error_regex = r"(^[EF][: ].*$)|(^.*Hard fault.*$)|(^.*Assertion.*$)"
@@ -276,10 +317,8 @@ def run_corstone300(
         # regex to check for error or fault messages in stdout from FVP
         if re.compile(error_regex, re.MULTILINE).search(result_stdout):
             raise RuntimeError(
-                f"Corstone simulation failed, log: \n {result_stdout}\n{result.stderr.decode()}"
+                f"Corstone simulation failed:\ncmd: {command_args[self.target_board]}\n, log: \n {result_stdout}\n{result.stderr.decode()}"
             )
-        elif "E [" in result_stdout:
-            logger.error(result_stdout)
 
         tosa_ref_output = np.fromfile(out_path_with_suffix, dtype=np.float32)
         output_shape = self.output_node.args[0][0].meta["val"].shape

backends/arm/test/tester/arm_tester.py

@@ -98,7 +98,7 @@ def __init__(self, runner_util: RunnerUtil, timeout: int = 1):
         self.runner.set_timeout(timeout)
 
     def run_artifact(self, inputs):
-        return self.runner.run_corstone300(inputs)
+        return self.runner.run_corstone(inputs)
 
     def dump_artifact(self, path_to_dump: Optional[str]):
         if not path_to_dump:
@@ -226,6 +226,7 @@ def run_method_and_compare_outputs(
         self,
         inputs: Optional[Tuple[torch.Tensor]] = None,
         stage: Optional[str] = None,
+        target_board: Optional[str] = "corstone-300",
         num_runs=1,
         atol=1e-03,
         rtol=1e-03,
@@ -260,7 +261,12 @@ def run_method_and_compare_outputs(
         edge_program = self.stages[
             self.stage_name(tester.ToEdge)
         ].artifact.exported_program()
-        self.runner_util.init_run(exported_program, edge_program, is_quantized)
+        self.runner_util.init_run(
+            exported_program,
+            edge_program,
+            is_quantized,
+            target_board,
+        )
 
         if is_quantized:
             reference_stage = self.stages[self.stage_name(tester.Quantize)]

examples/arm/setup.sh

@@ -45,20 +45,28 @@ function verify_md5() {
 script_dir=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
 
 if [[ "${ARCH}" == "x86_64" ]]; then
-    # FVP
-    fvp_url="https://developer.arm.com/-/media/Arm%20Developer%20Community/Downloads/OSS/FVP/Corstone-300/FVP_Corstone_SSE-300_11.22_20_Linux64.tgz?rev=018659bd574f4e7b95fa647e7836ccf4&hash=22A79103C6FA5FFA7AFF3BE0447F3FF9"
-    fvp_model_dir="Linux64_GCC-9.3"
-    fvp_md5_checksum="98e93b949d0fbac977292d8668d34523"
+    # FVPs
+    corstone300_url="https://developer.arm.com/-/media/Arm%20Developer%20Community/Downloads/OSS/FVP/Corstone-300/FVP_Corstone_SSE-300_11.22_20_Linux64.tgz?rev=018659bd574f4e7b95fa647e7836ccf4&hash=22A79103C6FA5FFA7AFF3BE0447F3FF9"
+    corstone300_model_dir="Linux64_GCC-9.3"
+    corstone300_md5_checksum="98e93b949d0fbac977292d8668d34523"
+
+    corstone320_url="https://developer.arm.com/-/media/Arm%20Developer%20Community/Downloads/OSS/FVP/Corstone-320/FVP_Corstone_SSE-320_11.27_25_Linux64.tgz?rev=a507bffc219a4d5792f1192ab7002d89&hash=D9A824AA8227D2E679C9B9787FF4E8B6FBE3D7C6"
+    corstone320_model_dir="Linux64_GCC-9.3"
+    corstone320_md5_checksum="3deb3c68f9b2d145833f15374203514d"
 
     # toochain
     toolchain_url="https://armkeil.blob.core.windows.net/developer/Files/downloads/gnu/12.3.rel1/binrel/arm-gnu-toolchain-12.3.rel1-x86_64-arm-none-eabi.tar.xz"
     toolchain_dir="arm-gnu-toolchain-12.3.rel1-x86_64-arm-none-eabi"
     toolchain_md5_checksum="00ebb1b70b1f88906c61206457eacb61"
 elif [[ "${ARCH}" == "aarch64" ]] || [[ "${ARCH}" == "arm64" ]]; then
-    # FVP
-    fvp_url="https://developer.arm.com/-/media/Arm%20Developer%20Community/Downloads/OSS/FVP/Corstone-300/FVP_Corstone_SSE-300_11.22_20_Linux64_armv8l.tgz?rev=9cc6e9a32bb947ca9b21fa162144cb01&hash=7657A4CF27D42E892E3F08D452AAB073"
-    fvp_model_dir="Linux64_armv8l_GCC-9.3"
-    fvp_md5_checksum="cbbabbe39b07939cff7a3738e1492ef1"
+    # FVPs
+    corstone300_url="https://developer.arm.com/-/media/Arm%20Developer%20Community/Downloads/OSS/FVP/Corstone-300/FVP_Corstone_SSE-300_11.22_20_Linux64_armv8l.tgz?rev=9cc6e9a32bb947ca9b21fa162144cb01&hash=7657A4CF27D42E892E3F08D452AAB073"
+    corstone300_model_dir="Linux64_armv8l_GCC-9.3"
+    corstone300_md5_checksum="cbbabbe39b07939cff7a3738e1492ef1"
+
+    corstone320_url="https://developer.arm.com/-/media/Arm%20Developer%20Community/Downloads/OSS/FVP/Corstone-320/FVP_Corstone_SSE-320_11.27_25_Linux64_armv8l.tgz?rev=b6ebe0923cb84f739e017385fd3c333c&hash=8965C4B98E2FF7F792A099B08831FE3CB6120493"
+    corstone320_model_dir="Linux64_armv8l_GCC-9.3"
+    corstone320_md5_checksum="3889f1d80a6d9861ea4aa6f1c88dd0ae"
 
     # toochain
     if [[ "${OS}" == "Darwin" ]]; then
@@ -105,26 +113,50 @@ function setup_fvp() {
     fi
 
     # Download and install the Corstone 300 FVP simulator platform
-    cd "${root_dir}"
-    if [[ ! -e FVP_cs300.tgz ]]; then
-        echo "[${FUNCNAME[0]}] Downloading FVP ..."
-        curl --output FVP_cs300.tgz "${fvp_url}"
-        verify_md5 ${fvp_md5_checksum} FVP_cs300.tgz
-    fi
-
-    echo "[${FUNCNAME[0]}] Installing FVP ..."
-    rm -rf FVP
-    mkdir -p FVP
-    cd FVP
-    tar xf ../FVP_cs300.tgz
-    ./FVP_Corstone_SSE-300.sh --i-agree-to-the-contained-eula --force --destination ./ --quiet --no-interactive
-
-    fvp_bin_path="$(cd models/${fvp_model_dir} && pwd)"
-    export PATH=${PATH}:${fvp_bin_path}
-
-    hash FVP_Corstone_SSE-300_Ethos-U55
-    echo "export PATH=\${PATH}:${fvp_bin_path}" >> ${setup_path_script}
-
+    fvps=("corstone300" "corstone320")
+
+    for fvp in "${fvps[@]}"; do
+        cd "${root_dir}"
+        if [[ ! -e "FVP_${fvp}.tgz" ]]; then
+            echo "[${FUNCNAME[0]}] Downloading FVP ${fvp}..."
+            url_variable=${fvp}_url
+            fvp_url=${!url_variable}
+            curl --output "FVP_${fvp}.tgz" "${fvp_url}"
+            md5_variable=${fvp}_md5_checksum
+            fvp_md5_checksum=${!md5_variable}
+            verify_md5 ${fvp_md5_checksum} FVP_${fvp}.tgz
+        fi
+
+        echo "[${FUNCNAME[0]}] Installing FVP ${fvp}..."
+        rm -rf FVP-${fvp}
+        mkdir -p FVP-${fvp}
+        cd FVP-${fvp}
+        tar xf ../FVP_${fvp}.tgz
+
+        # Install the FVP
+        case ${fvp} in
+            corstone300)
+                ./FVP_Corstone_SSE-300.sh --i-agree-to-the-contained-eula --force --destination ./ --quiet --no-interactive
+                ;;
+            corstone320)
+                ./FVP_Corstone_SSE-320.sh --i-agree-to-the-contained-eula --force --destination ./ --quiet --no-interactive
+                ;;
+            *)
+                echo "[${FUNCNAME[0]}] Error: Unknown FVP model ${fvp}. Exiting."
+                exit 1
+                ;;
+        esac
+
+        model_dir_variable=${fvp}_model_dir
+        fvp_model_dir=${!model_dir_variable}
+        fvp_bin_path="$(cd models/${fvp_model_dir} && pwd)"
+        export PATH=${PATH}:${fvp_bin_path}
+
+        echo "export PATH=\${PATH}:${fvp_bin_path}" >> ${setup_path_script}
+    done
+
+    # Fixup for Corstone-320 python dependency
+    echo "export LD_LIBRARY_PATH=${root_dir}/FVP-corstone320/python/lib/" >> ${setup_path_script}
 }
 
 function setup_toolchain() {