Only support int8 and quant dtypes for quant operators

Author: aaron-ang

backends/xnnpack/partition/config/xnnpack_config.py

@@ -10,12 +10,17 @@
 from typing import List, Optional
 
 import torch
+from torch.export import ExportedProgram
+from executorch.backends.xnnpack.utils.quant_utils import (
+    is_quant,
+    is_dequant,
+    is_qparam,
+)
 from executorch.exir.backend.canonical_partitioners.config_partitioner import (
     format_target_name,
     PartitionerConfig,
 )
 from executorch.exir.backend.utils import WhyNoPartition
-from torch.export import ExportedProgram
 
 logger = logging.getLogger(__name__)
 why = WhyNoPartition(logger=logger)
@@ -220,9 +225,18 @@ def _check_node_has_valid_dtype(self, node):
         valid_dtypes = {
             torch.float32,
             torch.float16,
-            torch.int8,
-            torch.qint8,
         }
+        # Only allow int8 and quant dtypes for quant operations
+        if is_quant(node) or is_dequant(node) or is_qparam(node):
+            valid_dtypes.update(
+                {
+                    torch.qint32,
+                    torch.qint8,
+                    torch.quint8,
+                    torch.int8,
+                }
+            )
+
         if (
             node.op != "placeholder"
             and node.op != "call_function"