[ET-VK][Ops] dequantize_per_tensor.tensor variant

Pull Request resolved: #12209

# Context

We need a tensor variant for dequantize/quantize operators since that is the expected output of choose_qparams.

# Changes

This extends the logic that currently exists to support a tensor variant for scales and zeros.
ghstack-source-id: 294163235
@exported-using-ghexport

Differential Revision: [D77746135](https://our.internmc.facebook.com/intern/diff/D77746135/)

Author: morelos

backends/vulkan/runtime/graph/ops/glsl/dequantize_buffer.glsl

@@ -27,9 +27,14 @@ ${layout_declare_tensor(B, "w", "t_out", OUT_DTYPE, "buffer")}
 ${layout_declare_tensor(B, "r", "t_in", IN_DTYPE, "buffer")}
 
 $if MODE == "per_tensor":
+  $if SHAPE == "tensor":
+    ${layout_declare_tensor(B, "r", "t_scale", "float", "buffer")}
+    ${layout_declare_tensor(B, "r", "t_zero_point", "int", "buffer")}
+
   layout(push_constant) uniform restrict Block {
-    float scale;
-    int zero_point;
+    $if SHAPE == "scalar":
+      float scale;
+      int zero_point;
     int quant_min;
     int quant_max;
   };
@@ -146,7 +151,10 @@ void dequantize_per_tensor() {
   const int in_bufi = tidx_to_bufi(out_tidx, t_in_strides);
 
   IN_T qvalue = t_in[in_bufi];
-  OUT_T value = dequantize_val(qvalue, scale, zero_point);
+  $if SHAPE == "scalar":
+    OUT_T value = dequantize_val(qvalue, scale, zero_point);
+  $if SHAPE == "tensor":
+    OUT_T value = dequantize_val(qvalue, t_scale[0], t_zero_point[0]);
 
   t_out[out_bufi] = value;
 }

backends/vulkan/runtime/graph/ops/glsl/dequantize_buffer.yaml

@@ -3,6 +3,7 @@ dequantize_buffer:
     IN_DTYPE: int32
     OUT_DTYPE: float
     MODE: per_tensor
+    SHAPE: tensor
   generate_variant_forall:
     IN_DTYPE:
       - VALUE: uint8
@@ -15,6 +16,9 @@ dequantize_buffer:
   shader_variants:
     - NAME: dequantize_per_tensor_buffer
       MODE: per_tensor
+      SHAPE: scalar
+    - NAME: dequantize_per_tensor_tensor_buffer
+      MODE: per_tensor
     - NAME: dequantize_per_token_buffer
       MODE: per_token
     - NAME: dequantize_per_channel_buffer

backends/vulkan/runtime/graph/ops/glsl/dequantize_texture.glsl

@@ -30,9 +30,14 @@ ${layout_declare_tensor(B, "w", "t_out", OUT_DTYPE, "texture3d")}
 ${layout_declare_tensor(B, "r", "t_in", IN_DTYPE, "texture3d")}
 
 $if MODE == "per_tensor":
+  $if SHAPE == "tensor":
+    ${layout_declare_tensor(B, "r", "t_scale", "float", "buffer")}
+    ${layout_declare_tensor(B, "r", "t_zero_point", "int", "buffer")}
+
   layout(push_constant) uniform restrict Block {
-    float scale;
-    int zero_point;
+    $if SHAPE == "scalar":
+      float scale;
+      int zero_point;
     int quant_min;
     int quant_max;
   };
@@ -148,7 +153,11 @@ void dequantize_per_tensor() {
 
   [[unroll]] for (int i = 0; i < 4; ++i) {
     IN_T qvalue = IN_T(intex[i]);
-    OUT_T value = dequantize_val(qvalue, scale, zero_point);
+    $if SHAPE == "scalar":
+      OUT_T value = dequantize_val(qvalue, scale, zero_point);
+    $if SHAPE == "tensor":
+      OUT_T value = dequantize_val(qvalue, t_scale[0], t_zero_point[0]);
+
     $if OUT_DTYPE == "double":
       outtex[i] = float(value);
     $else:

backends/vulkan/runtime/graph/ops/glsl/dequantize_texture.yaml

@@ -3,6 +3,7 @@ dequantize_texture:
     IN_DTYPE: int32
     OUT_DTYPE: float
     MODE: per_tensor
+    SHAPE: tensor
   generate_variant_forall:
     IN_DTYPE:
       - VALUE: uint8
@@ -15,6 +16,9 @@ dequantize_texture:
   shader_variants:
     - NAME: dequantize_per_tensor_texture3d
       MODE: per_tensor
+      SHAPE: scalar
+    - NAME: dequantize_per_tensor_tensor_texture3d
+      MODE: per_tensor
     - NAME: dequantize_per_token_texture3d
       MODE: per_token
     - NAME: dequantize_per_channel_texture3d

backends/vulkan/runtime/graph/ops/impl/Dequantize.cpp

@@ -80,13 +80,23 @@ void add_dequantize_per_tensor_node(
     const ValueRef& quant_min,
     const ValueRef& quant_max,
     const ValueRef& output) {
+  const bool is_tensor_scale_zp =
+      graph.val_is_tensor(scale) && graph.val_is_tensor(zero_point);
+
   std::string kernel_name("dequantize_per_tensor");
+  if (is_tensor_scale_zp) {
+    kernel_name += "_tensor";
+  }
   add_storage_type_suffix(kernel_name, graph.storage_type_of(input));
   add_dtype_suffix(kernel_name, graph.dtype_of(input));
   add_dtype_suffix(kernel_name, graph.dtype_of(output));
 
-  float scale_val = static_cast<float>(graph.get_double(scale));
-  int zero_point_val = static_cast<int>(graph.get_int(zero_point));
+  float scale_val = 1.0;
+  int zero_point_val = 0;
+  if (!is_tensor_scale_zp) {
+    scale_val = static_cast<float>(graph.get_double(scale));
+    zero_point_val = static_cast<int>(graph.get_int(zero_point));
+  }
   int quant_min_val = static_cast<int>(graph.get_int(quant_min));
   int quant_max_val = static_cast<int>(graph.get_int(quant_max));
 
@@ -100,15 +110,17 @@ void add_dequantize_per_tensor_node(
         graph.strides_ubo(input),
         graph.sizes_ubo(output),
         graph.strides_ubo(output)};
+  } else {
+    param_ubos = {
+        graph.logical_limits_ubo(input), graph.logical_limits_ubo(output)};
+  }
+
+  if (is_tensor_scale_zp) {
     push_constants = {
-        PushConstantDataInfo(&scale_val, sizeof(float)),
-        PushConstantDataInfo(&zero_point_val, sizeof(int)),
         PushConstantDataInfo(&quant_min_val, sizeof(int)),
         PushConstantDataInfo(&quant_max_val, sizeof(int)),
     };
   } else {
-    param_ubos = {
-        graph.logical_limits_ubo(input), graph.logical_limits_ubo(output)};
     push_constants = {
         PushConstantDataInfo(&scale_val, sizeof(float)),
         PushConstantDataInfo(&zero_point_val, sizeof(int)),
@@ -122,13 +134,20 @@ void add_dequantize_per_tensor_node(
       graph.hashed_layout_of(input),
   };
 
+  std::vector<ArgGroup> inputs_and_outputs = {
+      {output, vkapi::kWrite}, {input, vkapi::kRead}};
+  if (is_tensor_scale_zp) {
+    inputs_and_outputs.emplace_back(
+        ArgGroup{{scale, zero_point}, vkapi::kRead});
+  }
+
   graph.execute_nodes().emplace_back(new DynamicDispatchNode(
       graph,
       VK_KERNEL_FROM_STR(kernel_name),
       default_pick_global_wg_size,
       default_pick_local_wg_size,
       // Inputs and Outputs
-      {{output, vkapi::kWrite}, {input, vkapi::kRead}},
+      inputs_and_outputs,
       // Shader param buffers
       param_ubos,
       // Push Constants
@@ -519,6 +538,9 @@ REGISTER_OPERATORS {
   VK_REGISTER_OP(
       quantized_decomposed.dequantize_per_tensor.default,
       dequantize_per_tensor_impl);
+  VK_REGISTER_OP(
+      quantized_decomposed.dequantize_per_tensor.tensor,
+      dequantize_per_tensor_impl);
   VK_REGISTER_OP(
       quantized_decomposed.dequantize_per_token.default,
       dequantize_per_token_impl);