Support Conv3d and TransposeConv3d

Author: winskuo-quic

backends/qualcomm/builders/README.md

@@ -365,7 +365,7 @@ Please help update following table if you are contributing new operators:
 + 🚫 = Deprecated, supported with other QNN Ops
 
 
-| Operators | HTP - 90/116 Enabled |
+| Operators | HTP - 92/116 Enabled |
 |-----------|---------|
 | Argmax | ✓ |
 | Argmin | ✓ |
@@ -375,7 +375,7 @@ Please help update following table if you are contributing new operators:
 | ChannelShuffle | ✗ |
 | Concat | ✓ |
 | Conv2d | ✓ |
-| Conv3d | ✗ |
+| Conv3d | ✓ |
 | Convert | ✓ |
 | CreateSparse | ✗ |
 | CumulativeSum | ✓ |
@@ -481,7 +481,7 @@ Please help update following table if you are contributing new operators:
 | TopK | ✓ |
 | TransPose | ✓ |
 | TransPoseConv2d | ✓ |
-| TransPoseConv3d | ✗ |
+| TransPoseConv3d | ✓ |
 | Unpack | ✓ |
 
 ## Issues

backends/qualcomm/builders/__init__.py

@@ -24,7 +24,7 @@
     op_cat,
     op_ceil,
     op_clamp,
-    op_conv2d,
+    op_conv,
     op_copy,
     op_cos,
     op_cum_sum,
@@ -129,7 +129,7 @@
     op_cat,
     op_ceil,
     op_clamp,
-    op_conv2d,
+    op_conv,
     op_copy,
     op_cos,
     op_cum_sum,

backends/qualcomm/builders/op_conv2d.py backends/qualcomm/builders/op_conv.pybackends/qualcomm/builders/op_conv2d.py renamed to backends/qualcomm/builders/op_conv.py

@@ -4,10 +4,10 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+import warnings
 from typing import cast, Dict, List
 
 import executorch.backends.qualcomm.python.PyQnnWrapperAdaptor as PyQnnWrapper
-
 import numpy as np
 import torch
 from executorch.backends.qualcomm.utils.constants import QCOM_DATA
@@ -16,8 +16,10 @@
 from .node_visitor_manager import register_node_visitor
 from .qnn_constants import (
     OpConv2d,
+    OpConv3d,
     OpDepthWiseConv2d,
     OpTransposeConv2d,
+    OpTransposeConv3d,
     QNN_OP_PACKAGE_NAME_QTI_AISW,
 )
 from .utils import get_parameter
@@ -66,7 +68,7 @@ def _add_conv_op_parameter(
             len(padding_shape),
             padding_shape,
             np.array(
-                [[padding[0], padding[0]], [padding[1], padding[1]]],
+                padding,
                 dtype=np.uint32,
             ),
             True,
@@ -108,8 +110,14 @@ def define_node(
         input_node = self.get_node(node.args[0])
         input_tensor = self.get_tensor(input_node, node)
         assert (
-            input_tensor.dim() == 4
+            input_tensor.dim() != 3
         ), "All Conv1D should be converted to Conv2D in CanonicalizeConv,"
+        assert input_tensor.dim() in {
+            4,
+            5,
+        }, "Only Conv2d and Conv3d is supported in conv builder,"
+
+        is_conv2d = input_tensor.dim() == 4
         input_tensor_wrapper = self.define_tensor(
             input_node,
             node,
@@ -122,7 +130,12 @@ def define_node(
         filter_tensor = get_parameter(filter_node, self.edge_program)
         # weight of pytorch OIHW(conv2d) | IOHW(conv_transpose2d), yet QNN is HWIO
         is_transpose_conv = cast(bool, node.args[6])
-        filter_axis_order = (2, 3, 0, 1) if is_transpose_conv else (2, 3, 1, 0)
+        if is_conv2d:
+            filter_axis_order = (2, 3, 0, 1) if is_transpose_conv else (2, 3, 1, 0)
+        else:
+            filter_axis_order = (
+                (2, 3, 4, 0, 1) if is_transpose_conv else (2, 3, 4, 1, 0)
+            )
         filter_tensor = filter_tensor.permute(dims=filter_axis_order).contiguous()
         filter_tensor_wrapper = self.define_tensor(
             filter_node,
@@ -132,7 +145,6 @@ def define_node(
             nodes_to_wrappers,
         )
         conv_input_tensors = [input_tensor_wrapper, filter_tensor_wrapper]
-
         if node.args[2] is not None:
             bias_node = self.get_node(node.args[2])
             bias_tensor = get_parameter(bias_node, self.edge_program)
@@ -159,11 +171,10 @@ def define_node(
         padding = cast(List[int], node.args[4])
         dilation = cast(List[int], node.args[5])
         output_padding = cast(List[int], node.args[7])
-
         groups = cast(int, node.args[8])
-        # Qnn filter tensor is (H, W, Cin, Cout)
-        group_input_channels = filter_tensor.shape[2]
-        group_output_channels = int(filter_tensor.shape[3] / groups)
+        # Qnn filter tensor is (H, W, Cin, Cout) or (D, H, W, Cin, Cout)
+        group_input_channels = filter_tensor.shape[-2]
+        group_output_channels = int(filter_tensor.shape[-1] / groups)
         # 1) groups = input_channels (i.e. group_input_channels = 1)
         # 2) output_channels is a positive integer multiple of input channels
         # TODO: Currently, negative results will be zero with Depthwise conv2d when input_channel == groups == 1
@@ -175,18 +186,36 @@ def define_node(
         )
         if len(padding) == 1:
             padding = padding + padding
+        padding = [[x, x] for x in padding]
 
         stride_shape = [len(stride)]
-        padding_shape = [2, 2]
+        padding_shape = [len(padding), len(padding[0])]
         dilation_shape = [len(dilation)]
         output_padding_shape = [len(output_padding)]
 
+        # Some transpose_conv3d restrictions by QNN
+        if not is_conv2d and is_transpose_conv:
+            if dilation[0] != 1:
+
+                warnings.warn(
+                    "[QNN Delegate Op Builder]: As of QNN 2.37, TransposeConv3d only supports dilation of 1 in the depth dimension.",
+                    stacklevel=1,
+                )
+
+            # If using dilation, then stride height and width must = 1
+            if any(x != 1 for x in dilation) and (stride[1] != 1 or stride[2] != 1):
+                warnings.warn(
+                    "[QNN Delegate Op Builder]: As of QNN 2.37, TransposeConv3d, only supports dilation if stride in height and width = 1.",
+                    stacklevel=1,
+                )
+
         if is_depthwise_conv:
+            assert is_conv2d, "DepthWise only supports Conv2d"
             op_class = OpDepthWiseConv2d
         elif is_transpose_conv:
-            op_class = OpTransposeConv2d
+            op_class = OpTransposeConv2d if is_conv2d else OpTransposeConv3d
         else:
-            op_class = OpConv2d
+            op_class = OpConv2d if is_conv2d else OpConv3d
 
         conv_op = PyQnnWrapper.PyQnnOpWrapper(
             node.name,

backends/qualcomm/builders/qnn_constants.py

@@ -59,6 +59,15 @@ class OpConv2d:
     param_dilation: str = "dilation"
 
 
+@dataclass(init=False, frozen=True)
+class OpConv3d:
+    op_name: str = "Conv3d"
+    param_stride: str = "stride"
+    param_pad_amount: str = "pad_amount"
+    param_group: str = "group"
+    param_dilation: str = "dilation"
+
+
 @dataclass(init=False, frozen=True)
 class OpConvert:
     op_name: str = "Convert"
@@ -573,6 +582,15 @@ class OpTransposeConv2d:
     param_output_padding: str = "output_padding"
 
 
+@dataclass(init=False, frozen=True)
+class OpTransposeConv3d:
+    op_name: str = "TransposeConv3d"
+    param_stride: str = "stride"
+    param_pad_amount: str = "pad_amount"
+    param_group: str = "group"
+    param_output_padding: str = "output_padding"
+
+
 @dataclass(init=False, frozen=True)
 class OpUnpack:
     op_name: str = "UnPack"

backends/qualcomm/quantizer/annotators.py

@@ -1094,11 +1094,13 @@ def annotate_cdist(node: Node, quantization_config: QuantizationConfig) -> None:
 
 @register_annotator(
     [
+        torch.ops.aten.conv1d.default,
         torch.ops.aten.conv2d.default,
         torch.ops.aten.conv2d.padding,
-        torch.ops.aten.conv1d.default,
-        torch.ops.aten.conv_transpose2d.input,
+        torch.ops.aten.conv3d.default,
         torch.ops.aten.conv_transpose1d.default,
+        torch.ops.aten.conv_transpose2d.input,
+        torch.ops.aten.conv_transpose3d.input,
         torch.ops.aten.convolution.default,
     ]
 )

backends/qualcomm/tests/models.py

@@ -597,6 +597,28 @@ def forward(self, x):
         return self.second(self.first(x))
 
 
+class Conv3dSequential(torch.nn.Module):
+    def __init__(self, bias=True):
+        super().__init__()
+        self.first = torch.nn.Conv3d(
+            in_channels=1,
+            out_channels=3,
+            kernel_size=(3, 3, 3),
+            padding=1,
+            bias=bias,
+        )
+        self.second = torch.nn.Conv3d(
+            in_channels=3,
+            out_channels=2,
+            kernel_size=(3, 3, 3),
+            padding=1,
+            bias=bias,
+        )
+
+    def forward(self, x):
+        return self.second(self.first(x))
+
+
 class Conv2dSingle(torch.nn.Module):
     def __init__(
         self,
@@ -619,40 +641,6 @@ def forward(self, x):
         return self.conv(x)
 
 
-class ConvTranspose1dSingle(torch.nn.Module):
-    def __init__(self, bias=True, dilation=1):
-        super().__init__()
-        self.conv_transpose = torch.nn.ConvTranspose1d(
-            in_channels=1,
-            out_channels=3,
-            kernel_size=3,
-            stride=2,
-            padding=1,
-            dilation=dilation,
-            bias=bias,
-        )
-
-    def forward(self, x):
-        return self.conv_transpose(x)
-
-
-class ConvTranspose2dSingle(torch.nn.Module):
-    def __init__(self, bias=True, dilation=1):
-        super().__init__()
-        self.conv_transpose = torch.nn.ConvTranspose2d(
-            in_channels=1,
-            out_channels=3,
-            kernel_size=3,
-            stride=2,
-            padding=1,
-            dilation=dilation,
-            bias=bias,
-        )
-
-    def forward(self, x):
-        return self.conv_transpose(x)
-
-
 class Conv2dDownUpSample(torch.nn.Module):
     def __init__(self, bias=True):
         super().__init__()
@@ -737,6 +725,57 @@ def forward(self, x):
         return topk_values
 
 
+class ConvTranspose1dSingle(torch.nn.Module):
+    def __init__(self, bias=True, dilation=1):
+        super().__init__()
+        self.conv_transpose = torch.nn.ConvTranspose1d(
+            in_channels=1,
+            out_channels=3,
+            kernel_size=3,
+            stride=2,
+            padding=1,
+            dilation=dilation,
+            bias=bias,
+        )
+
+    def forward(self, x):
+        return self.conv_transpose(x)
+
+
+class ConvTranspose2dSingle(torch.nn.Module):
+    def __init__(self, bias=True, dilation=1):
+        super().__init__()
+        self.conv_transpose = torch.nn.ConvTranspose2d(
+            in_channels=1,
+            out_channels=3,
+            kernel_size=3,
+            stride=2,
+            padding=1,
+            dilation=dilation,
+            bias=bias,
+        )
+
+    def forward(self, x):
+        return self.conv_transpose(x)
+
+
+class ConvTranspose3dSingle(torch.nn.Module):
+    def __init__(self, bias=True, dilation=1):
+        super().__init__()
+        self.conv_transpose = torch.nn.ConvTranspose3d(
+            in_channels=1,
+            out_channels=3,
+            kernel_size=1,
+            stride=(1, 1, 1),
+            padding=1,
+            dilation=dilation,
+            bias=bias,
+        )
+
+    def forward(self, x):
+        return self.conv_transpose(x)
+
+
 class Cos(torch.nn.Module):
     def __init__(self):
         super().__init__()