Revert "Arm backend: Match fp32->int32 cast between pytorch and TOSA's CAST (pytorch#12243)"

This reverts commit 1315388.

Author: psiddh

backends/arm/_passes/__init__.py

@@ -51,7 +51,6 @@
 from .decompose_sqrt_pass import DecomposeSqrtPass  # noqa
 from .decompose_sum_pass import DecomposeSumPass  # noqa
 from .decompose_var_pass import DecomposeVarPass  # noqa
-from .decorate_fp32_to_int32_casting_pass import DecorateFp32toInt32CastingPass  # noqa
 from .fold_qdq_with_annotated_qparams_pass import (  # noqa
     FoldAndAnnotateQParamsPass,
     QuantizeOperatorArguments,

backends/arm/_passes/arm_pass_manager.py

@@ -56,7 +56,6 @@
     DecomposeSqrtPass,
     DecomposeSumPass,
     DecomposeVarPass,
-    DecorateFp32toInt32CastingPass,
     FoldAndAnnotateQParamsPass,
     FuseBatchnorm2DPass,
     FuseConstantArgsPass,
@@ -201,9 +200,6 @@ def _tosa_080_MI_pipeline(self, exported_program: ExportedProgram) -> GraphModul
         self.add_pass(MatchArgRanksPass(exported_program))
         self.add_pass(DecomposeAdaptiveAvgPool2dPass())
         self.add_pass(DecomposeAvgPool2d())
-        self.add_pass(
-            DecorateFp32toInt32CastingPass()
-        )  # Require that no new fp32->int32 is introduced after this pass
         self.add_pass(ComputeConstantOpsAOT(exported_program))
 
         self.add_pass(DecomposeGroupedConv())