Run on device and compare outputs

For op test, add run_method_and_compare_outputs to make the test complete.
The Exynos-backend-specific PTE file will be executed on device, and its outputs will be checked.

Co-authored-by: chong-chen <[email protected]>

Author: Jiseong-oh

backends/samsung/test/ops/test_add.py

@@ -47,6 +47,7 @@ def _test(self, module: torch.nn.Module, inputs):
             .check_not(["executorch_exir_dialects_edge__ops_aten_add_Tensor"])
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
+            .run_method_and_compare_outputs(inputs=inputs)
         )
 
     def test_fp32_simple_add(self):

backends/samsung/test/ops/test_avg_pool2d.py

@@ -32,19 +32,15 @@ def __init__(
             ceil_mode=False,
         ).to(torch.float)
 
-    def get_example_inputs(self) -> tuple[torch.Tensor]:
-        input_1 = torch.randn(1, 16, 24, 24)
-        return (input_1,)
-
     def forward(self, x: torch.Tensor) -> torch.Tensor:
         return self.avg_pool(x)
 
 
 class TestAvgPool2d(unittest.TestCase):
-    def _test(self, module: torch.nn.Module):
+    def _test(self, module: torch.nn.Module, inputs):
         tester = SamsungTester(
             module,
-            module.get_example_inputs(),
+            inputs,
             [gen_samsung_backend_compile_spec("E9955")],
         )
         (
@@ -54,13 +50,17 @@ def _test(self, module: torch.nn.Module):
             .check_not(["executorch_exir_dialects_edge__ops_aten_avg_pool2d_default"])
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
+            .run_method_and_compare_outputs(inputs=inputs)
         )
 
     def test_fp32_avg_pool2d(self):
-        self._test(AvgPool2d())
+        inputs = (torch.randn(1, 16, 24, 24),)
+        self._test(AvgPool2d(), inputs)
 
     def test_fp32_avg_pool2d_with_stride(self):
-        self._test(AvgPool2d(stride=2))
+        inputs = (torch.randn(1, 16, 24, 24),)
+        self._test(AvgPool2d(stride=2), inputs)
 
     def test_fp32_avg_pool2d_with_kernel_size(self):
-        self._test(AvgPool2d(kernel_size=4))
+        inputs = (torch.randn(1, 16, 24, 24),)
+        self._test(AvgPool2d(kernel_size=4), inputs)

backends/samsung/test/ops/test_batch_norm.py

@@ -43,6 +43,7 @@ def _test(self, module: torch.nn.Module, inputs):
             )
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
+            .run_method_and_compare_outputs(inputs=inputs)
         )
 
     def test_fp32_batch_norm(self):

backends/samsung/test/ops/test_bmm.py

@@ -31,6 +31,7 @@ def forward(self, x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
 
 class TestBatchMatmul(unittest.TestCase):
     def _test(self, module: torch.nn.Module):
+        torch.manual_seed(8)
         inputs = module.get_example_inputs()
         tester = SamsungTester(
             module, inputs, [gen_samsung_backend_compile_spec("E9955")]
@@ -42,6 +43,7 @@ def _test(self, module: torch.nn.Module):
             .check_not(["executorch_exir_dialects_edge__ops_aten_bmm_default"])
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
+            .run_method_and_compare_outputs(inputs=inputs, atol=0.005, rtol=0.005)
         )
 
     def test_fp32_bmm(self):

backends/samsung/test/ops/test_cat.py

@@ -37,6 +37,7 @@ def _test(self, module: torch.nn.Module, inputs):
             .check_not(["executorch_exir_dialects_edge__ops_aten_cat_default"])
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
+            .run_method_and_compare_outputs(inputs=inputs)
         )
 
     def test_fp32_concat_on_axis1(self):

backends/samsung/test/ops/test_clamp.py

@@ -42,6 +42,7 @@ def _test(self, module: torch.nn.Module, inputs):
             .check_not(["executorch_exir_dialects_edge__ops_aten_clamp_default"])
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
+            .run_method_and_compare_outputs(inputs=inputs)
         )
 
     def test_fp32_clamp(self):

backends/samsung/test/ops/test_constant_pad_nd.py

@@ -38,6 +38,7 @@ def _test(self, module: torch.nn.Module, inputs):
             )
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
+            .run_method_and_compare_outputs(inputs=inputs)
         )
 
     def test_fp32_constant_pad_nd(self):

backends/samsung/test/ops/test_conv2d.py

@@ -41,10 +41,6 @@ def __init__(
 
         self.in_channels = in_channels
 
-    def get_example_inputs(self) -> tuple[torch.Tensor]:
-        input_1 = torch.randn(1, self.in_channels, 24, 24)
-        return (input_1,)
-
     def forward(self, x: torch.Tensor) -> torch.Tensor:
         return self.conv(x)
 
@@ -62,19 +58,15 @@ def __init__(self) -> None:
             bias=True,
         )
 
-    def get_example_inputs(self) -> tuple[torch.Tensor]:
-        input_1 = torch.randn(1, 32, 24, 24)
-        return (input_1,)
-
     def forward(self, x: torch.Tensor) -> torch.Tensor:
         return self.conv(x)
 
 
 class TestConv2d(unittest.TestCase):
-    def _test(self, module: torch.nn.Module):
+    def _test(self, module: torch.nn.Module, inputs):
         tester = SamsungTester(
             module,
-            module.get_example_inputs(),
+            inputs,
             [gen_samsung_backend_compile_spec("E9955")],
         )
         (
@@ -83,16 +75,21 @@ def _test(self, module: torch.nn.Module):
             .check_not(["executorch_exir_dialects_edge__ops_aten_convolution_default"])
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
+            .run_method_and_compare_outputs(inputs=inputs)
         )
 
     def test_fp32_conv2d_without_bias(self):
-        self._test(Conv2d(bias=False))
+        inputs = (torch.randn(1, 3, 24, 24),)
+        self._test(Conv2d(bias=False), inputs)
 
     def test_fp32_conv2d_with_bias(self):
-        self._test(Conv2d(bias=True))
+        inputs = (torch.randn(1, 3, 24, 24),)
+        self._test(Conv2d(bias=True), inputs)
 
     def test_fp32_depthwise_conv2d(self):
-        self._test(Conv2d(in_channels=8, out_channels=8, groups=8))
+        inputs = (torch.randn(1, 8, 24, 24),)
+        self._test(Conv2d(in_channels=8, out_channels=8, groups=8), inputs)
 
     def test_fp32_transpose_conv2d(self):
-        self._test(TransposeConv2d())
+        inputs = (torch.randn(1, 32, 24, 24),)
+        self._test(TransposeConv2d(), inputs)

backends/samsung/test/ops/test_div.py

@@ -38,6 +38,7 @@ def _test(self, module: torch.nn.Module, inputs):
             .check_not(["executorch_exir_dialects_edge__ops_aten_div_Tensor"])
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
+            .run_method_and_compare_outputs(inputs=inputs)
         )
 
     def test_fp32_simple_div(self):

backends/samsung/test/ops/test_embedding.py

@@ -37,9 +37,10 @@ def _test(self, module: torch.nn.Module, inputs):
             .check_not(["executorch_exir_dialects_edge__ops_aten_embedding_default"])
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
+            .run_method_and_compare_outputs(inputs=inputs)
         )
 
     def test_fp32_embedding(self):
         num_embeddings = 2048
-        inputs = (torch.randint(0, num_embeddings, (1, 64), dtype=torch.int32),)
+        inputs = (torch.randint(0, 12, (1, 64), dtype=torch.int32),)
         self._test(Embedding(num_embeddings=num_embeddings), inputs)